HW2_q1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "2ef5dc3a",
   "metadata": {},
   "source": [
    "# libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "929dbac0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: rarfile in /home/erfan_wsl/.local/lib/python3.10/site-packages (4.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "# pip install rarfile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d2cdb6ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "# from zipfile import ZipFile, Path\n",
    "from rarfile import RarFile, Path\n",
    "from PIL import Image\n",
    "import numpy as np\n",
    "# import fnmatch"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8bd8662",
   "metadata": {},
   "source": [
    "# Creating dataframes and saving them as csv files"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "92bd2ccf",
   "metadata": {},
   "source": [
    "### if you have already run these cells, you can skip this section"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "b6e8ce49",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#reading csv files from the zip file and concatenating them as a dataframe also creating related class numbers dataframe\n",
    "DF_labels= pd.read_csv(\"trainLabels.csv\") \n",
    "data=[]\n",
    "class_df = []\n",
    "with RarFile(\"ASM_Features.rar\") as zipfiles:\n",
    "\n",
    "    #the last entry is the zipfile name\n",
    "    #we'll skip it\n",
    "    filelist = zipfiles.namelist()[:-1]\n",
    "        \n",
    "    for file_name in filelist:\n",
    "        filelist = zipfiles.namelist()[:-1]\n",
    "\n",
    "        data.append(pd.read_csv(zipfiles.open(file_name)))\n",
    "        class_df.append( DF_labels[DF_labels[\"Id\"] == file_name[13:-4] ][\"Class\"].values[0] )\n",
    "\n",
    "        \n",
    "df = pd.concat(data)\n",
    "class_df = pd.DataFrame(class_df,columns=[\"Class\",])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "4f617331",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10863</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10864</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10865</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10866</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10867</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10868 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Class\n",
       "0          9\n",
       "1          2\n",
       "2          9\n",
       "3          1\n",
       "4          8\n",
       "...      ...\n",
       "10863      4\n",
       "10864      4\n",
       "10865      4\n",
       "10866      4\n",
       "10867      4\n",
       "\n",
       "[10868 rows x 1 columns]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_df "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "67b26893",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>OPcodes</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>db</td>\n",
       "      <td>1366877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>mov</td>\n",
       "      <td>5295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>push</td>\n",
       "      <td>3208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>call</td>\n",
       "      <td>1533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>jmp</td>\n",
       "      <td>730</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>64</td>\n",
       "      <td>movzx</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>65</td>\n",
       "      <td>fst</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>66</td>\n",
       "      <td>mul</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>67</td>\n",
       "      <td>jge</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>68</td>\n",
       "      <td>ja</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>507499 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0 OPcodes    count\n",
       "0           0      db  1366877\n",
       "1           1     mov     5295\n",
       "2           2    push     3208\n",
       "3           3    call     1533\n",
       "4           4     jmp      730\n",
       "..        ...     ...      ...\n",
       "64         64   movzx        1\n",
       "65         65     fst        1\n",
       "66         66     mul        1\n",
       "67         67     jge        1\n",
       "68         68      ja        1\n",
       "\n",
       "[507499 rows x 3 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "4a0aa889",
   "metadata": {},
   "outputs": [],
   "source": [
    "#extracting all the unique OPcodes from all the files\n",
    "col = df['OPcodes'].unique()\n",
    "# d = pd.DataFrame(0, index=np.arange(1), columns=col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "id": "f3cbdbef",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['db',\n",
       " 'mov',\n",
       " 'push',\n",
       " 'call',\n",
       " 'jmp',\n",
       " 'test',\n",
       " 'cmp',\n",
       " 'sub',\n",
       " 'add',\n",
       " 'jz',\n",
       " 'pop',\n",
       " 'align',\n",
       " 'dd',\n",
       " 'xor',\n",
       " 'jge',\n",
       " 'lea',\n",
       " 'jle',\n",
       " 'retn',\n",
       " 'or',\n",
       " 'shl',\n",
       " 'and',\n",
       " 'jnz',\n",
       " 'imul',\n",
       " 'sar',\n",
       " 'jl',\n",
       " 'jg',\n",
       " 'neg',\n",
       " 'dec',\n",
       " 'inc',\n",
       " 'shr',\n",
       " 'stmxcsr',\n",
       " 'setnl',\n",
       " 'adc',\n",
       " 'dw',\n",
       " 'jb',\n",
       " 'unicode',\n",
       " 'not',\n",
       " 'movzx',\n",
       " 'mul',\n",
       " 'jbe',\n",
       " 'jnb',\n",
       " 'ja',\n",
       " 'div',\n",
       " 'sbb',\n",
       " 'lock',\n",
       " 'rcr',\n",
       " 'rcl',\n",
       " 'rep',\n",
       " 'les',\n",
       " 'movlps',\n",
       " 'int',\n",
       " 'xchg',\n",
       " 'retf',\n",
       " 'arpl',\n",
       " 'out',\n",
       " 'ror',\n",
       " 'in',\n",
       " 'repne',\n",
       " 'enter',\n",
       " 'lds',\n",
       " 'jnp',\n",
       " 'loop',\n",
       " 'fcomp',\n",
       " 'fimul',\n",
       " 'js',\n",
       " 'loopne',\n",
       " 'jo',\n",
       " 'jns',\n",
       " 'movsx',\n",
       " 'fcom',\n",
       " 'bound',\n",
       " 'fistp',\n",
       " 'fdiv',\n",
       " 'fsub',\n",
       " 'jp',\n",
       " 'sal',\n",
       " 'aad',\n",
       " 'fsubr',\n",
       " 'rol',\n",
       " 'fld',\n",
       " 'fstp',\n",
       " 'pinsrw',\n",
       " 'setz',\n",
       " 'seto',\n",
       " 'idiv',\n",
       " 'setnz',\n",
       " 'rclsid',\n",
       " 'fnstcw',\n",
       " 'riid',\n",
       " 'jno',\n",
       " 'fild',\n",
       " 'extrn',\n",
       " 'vmovhps',\n",
       " 's',\n",
       " 'fnstsw',\n",
       " 'fadd',\n",
       " 'fst',\n",
       " 'fmul',\n",
       " 'fdivr',\n",
       " 'fucomp',\n",
       " 'fucom',\n",
       " 'fxch',\n",
       " 'faddp',\n",
       " 'fmulp',\n",
       " 'aam',\n",
       " 'retnw',\n",
       " 'fisttp',\n",
       " 'fisub',\n",
       " 'jecxz',\n",
       " 'ficom',\n",
       " 'fidivr',\n",
       " 'ficomp',\n",
       " 'fidiv',\n",
       " 'fbstp',\n",
       " 'fldenv',\n",
       " 'fldcw',\n",
       " 'fsubp',\n",
       " 'movs',\n",
       " 'lfs',\n",
       " 'frstor',\n",
       " 'fcmovne',\n",
       " 'xabort',\n",
       " 'fnstenv',\n",
       " 'fcomip',\n",
       " 'xorps',\n",
       " 'fbld',\n",
       " 'fcmovnu',\n",
       " 'fiadd',\n",
       " 'cmovg',\n",
       " 'ffreep',\n",
       " 'paddb',\n",
       " 'fisubr',\n",
       " 'fnsave',\n",
       " 'repe',\n",
       " 'offset',\n",
       " 'fcmovnb',\n",
       " 'ht',\n",
       " 'loope',\n",
       " 'fist',\n",
       " 'fstsw',\n",
       " 'fdivrp',\n",
       " 'fsave',\n",
       " 'fdivp',\n",
       " 'shrd',\n",
       " 'fcomi',\n",
       " 'packsswb',\n",
       " 'punpckhdq',\n",
       " 'movups',\n",
       " 'scas',\n",
       " 'vpcmpeqb',\n",
       " 'fsubrp',\n",
       " 'vpermq',\n",
       " 'vmovsd',\n",
       " 'orps',\n",
       " 'ffree',\n",
       " 'fcmovb',\n",
       " 'setl',\n",
       " 'shld',\n",
       " 'fstcw',\n",
       " 'setnle',\n",
       " 'pclsid',\n",
       " 'pqit',\n",
       " 'pv',\n",
       " 'ppenumfmtetc',\n",
       " 'lgs',\n",
       " 'setle',\n",
       " 'dwhkl',\n",
       " 'ldmxcsr',\n",
       " 'name',\n",
       " 'psubq',\n",
       " 'bsr',\n",
       " 'bsf',\n",
       " 'cmovnz',\n",
       " 'cmovb',\n",
       " 'subps',\n",
       " 'fucomip',\n",
       " 'por',\n",
       " 'cmps',\n",
       " 'retfw',\n",
       " 'rcpps',\n",
       " 'wait',\n",
       " 'setbe',\n",
       " 'fucomi',\n",
       " 'fcmovnbe',\n",
       " 'outs',\n",
       " 'movdqa',\n",
       " 'bt',\n",
       " 'bts',\n",
       " 'pxor',\n",
       " 'movapd',\n",
       " 'movd',\n",
       " 'paddq',\n",
       " 'bswap',\n",
       " 'pmuludq',\n",
       " 'movq',\n",
       " 'pshufd',\n",
       " 'pand',\n",
       " 'psrlq',\n",
       " 'punpckldq',\n",
       " 'pcmpeqd',\n",
       " 'bit',\n",
       " 'psrldq',\n",
       " 'cmovz',\n",
       " 'deterministic',\n",
       " 'bits',\n",
       " 'cmpxchg',\n",
       " 'andnps',\n",
       " 'paddusw',\n",
       " 'psubsb',\n",
       " 'setb',\n",
       " 'fcmovu',\n",
       " 'andpd',\n",
       " 'psllq',\n",
       " 'cmpltpd',\n",
       " 'ucomisd',\n",
       " 'psubd',\n",
       " 'cmpnlepd',\n",
       " 'subsd',\n",
       " 'orpd',\n",
       " 'addsd',\n",
       " 'cmova',\n",
       " 'cmovbe',\n",
       " 'punpcklwd',\n",
       " 'btr',\n",
       " 'setnb',\n",
       " 'h',\n",
       " 'edge',\n",
       " 'hhk',\n",
       " 'setnbe',\n",
       " 'psrlw',\n",
       " 'punpcklbw',\n",
       " 'psllw',\n",
       " 'pmullw',\n",
       " 'psubsw',\n",
       " 'paddw',\n",
       " 'packuswb',\n",
       " 'punpckhbw',\n",
       " 'punpckhwd',\n",
       " 'paddsw',\n",
       " 'psubw',\n",
       " 'pmulhw',\n",
       " 'xadd',\n",
       " 'clsid',\n",
       " 'landing',\n",
       " 'pcmpgtd',\n",
       " 'nop',\n",
       " 'dir',\n",
       " 'movlpd',\n",
       " 'mulsd',\n",
       " 'pextrw',\n",
       " 'mulpd',\n",
       " 'movsd',\n",
       " 'xorpd',\n",
       " 'pmovmskb',\n",
       " 'addpd',\n",
       " 'unpcklpd',\n",
       " 'pmaxsw',\n",
       " 'divsd',\n",
       " 'cdq',\n",
       " 'fcmove',\n",
       " 'pcmpgtb',\n",
       " 'vdivps',\n",
       " 'vmread',\n",
       " 'btc',\n",
       " 'set',\n",
       " 'clear',\n",
       " 'pavgb',\n",
       " 'pavgusb',\n",
       " 'pmaddwd',\n",
       " 'psubusb',\n",
       " 'psraw',\n",
       " 'pcmpeqb',\n",
       " 'paddd',\n",
       " 'packssdw',\n",
       " 'pfadd',\n",
       " 'psubb',\n",
       " 'psrad',\n",
       " 'movntq',\n",
       " 'pfmul',\n",
       " 'paddusb',\n",
       " 'pminub',\n",
       " 'pmaxub',\n",
       " 'pmulhuw',\n",
       " 'movaps',\n",
       " 'prefetchnta',\n",
       " 'pfsub',\n",
       " 'cmovns',\n",
       " 'cmovl',\n",
       " 'pcmpgtw',\n",
       " 'prefetch',\n",
       " 'pshufw',\n",
       " 'cmovle',\n",
       " 'pswapd',\n",
       " 'shufps',\n",
       " 'pandn',\n",
       " 'psrld',\n",
       " 'psubusw',\n",
       " 'cmovge',\n",
       " 'cmovs',\n",
       " 'addps',\n",
       " 'pfacc',\n",
       " 'mulps',\n",
       " 'pfsubr',\n",
       " 'pslld',\n",
       " 'pminsw',\n",
       " 'movntps',\n",
       " 'unpcklps',\n",
       " 'movss',\n",
       " 'sets',\n",
       " 'paddsb',\n",
       " 'pcmpeqw',\n",
       " 'cmovnb',\n",
       " 'psadbw',\n",
       " 'fcmovbe',\n",
       " 'sqrtps',\n",
       " 'setnp',\n",
       " 'xlat',\n",
       " 'vsqrtps',\n",
       " 'svts',\n",
       " 'sldt',\n",
       " 'iid',\n",
       " 'sz',\n",
       " 'psz',\n",
       " 'lods',\n",
       " 'vrcpss',\n",
       " 'signature',\n",
       " 'cmpeqps',\n",
       " 'vpsrad',\n",
       " 'vpcmpgtw',\n",
       " 'proto',\n",
       " 'addr',\n",
       " 'minps',\n",
       " 'vpunpckldq',\n",
       " 'unpckhps',\n",
       " 'stos',\n",
       " 'values',\n",
       " 'ucomiss',\n",
       " 'cmovp',\n",
       " 'lar',\n",
       " 'setns',\n",
       " 'andps',\n",
       " 'scasb',\n",
       " 'hnt',\n",
       " 'clc',\n",
       " 'comiss',\n",
       " 'rguid',\n",
       " 'pperrinfo',\n",
       " 'lgdt',\n",
       " 'vminsd',\n",
       " 'vmovdqu',\n",
       " 'fldz',\n",
       " 'cmovnp',\n",
       " 'vmptrld',\n",
       " 'enterw',\n",
       " 'vmovdqa',\n",
       " 'cmpneqpd',\n",
       " 'jcxz',\n",
       " 'vpunpcklqdq',\n",
       " 'cmovo',\n",
       " 'maxps',\n",
       " 'setno',\n",
       " 'plpal',\n",
       " 'table',\n",
       " 'ppv',\n",
       " 'ho',\n",
       " 'cy',\n",
       " 'invalid',\n",
       " 'vpcmpeqw',\n",
       " 'capabilities',\n",
       " 'rsts',\n",
       " 'hwinsta',\n",
       " 'rsqrtps',\n",
       " 'hwnd',\n",
       " 'leave',\n",
       " 'format',\n",
       " 'dy',\n",
       " 'vpsrld',\n",
       " 'ppshf',\n",
       " 'pvarg',\n",
       " 'ppidl',\n",
       " 'sgdt',\n",
       " 'vpsllq',\n",
       " 'maskmovq',\n",
       " 'color',\n",
       " 'pslldq',\n",
       " 'movdqu',\n",
       " 'pshuflw',\n",
       " 'pshufhw',\n",
       " 'punpcklqdq',\n",
       " 'divps',\n",
       " 'punpckhqdq',\n",
       " 'item',\n",
       " 'mulss',\n",
       " 'subss',\n",
       " 'divss',\n",
       " 'setp',\n",
       " 'addss',\n",
       " 'ins',\n",
       " 'dict',\n",
       " 'rop',\n",
       " 'cmovno',\n",
       " 'is',\n",
       " 'cp',\n",
       " 'seconds',\n",
       " 'rsldt',\n",
       " 'cmpps',\n",
       " 'vpmulhuw',\n",
       " 'lp',\n",
       " 'image',\n",
       " 'movmskps',\n",
       " 'vpacksswb',\n",
       " 'movlhps',\n",
       " 'vmwrite',\n",
       " 'vpsubb',\n",
       " 'phmx',\n",
       " 'lpfn',\n",
       " 'hmod',\n",
       " 'vdivsd',\n",
       " 'vmovups',\n",
       " 'certificates',\n",
       " 'movhps',\n",
       " 'svldt',\n",
       " 'lss',\n",
       " 'psizl',\n",
       " 'vsqrtsd',\n",
       " 'pavgw',\n",
       " 'vaddsd',\n",
       " 'vaddss',\n",
       " 'rc',\n",
       " 'movhlps',\n",
       " 'certificate',\n",
       " 'cert',\n",
       " 'clflush',\n",
       " 'unpckhpd',\n",
       " 'cmpeqsd',\n",
       " 'subpd',\n",
       " 'fstenv',\n",
       " 'char',\n",
       " 'chars',\n",
       " 'pptlib',\n",
       " 'loopw',\n",
       " 'vaddps',\n",
       " 'vpsllw',\n",
       " 'fps',\n",
       " 'movnti',\n",
       " 'comisd',\n",
       " 'shufpd',\n",
       " 'hdc',\n",
       " 'insertps',\n",
       " 'pextrd',\n",
       " 'extractps',\n",
       " 'pinsrd',\n",
       " 'minss',\n",
       " 'maxss',\n",
       " 'packusdw',\n",
       " 'pblendvb',\n",
       " 'pextrb',\n",
       " 'roundps',\n",
       " 'cmpltps',\n",
       " 'pminuw',\n",
       " 'pmovzxwd',\n",
       " 'pshufb',\n",
       " 'pmovzxbd',\n",
       " 'ptest',\n",
       " 'cmpleps',\n",
       " 'pminud',\n",
       " 'cmpeqss',\n",
       " 'fxsave',\n",
       " 'hbm',\n",
       " 'xbegin',\n",
       " 'sqrtsd',\n",
       " 'sqrtss',\n",
       " 'andnpd',\n",
       " 'minsd',\n",
       " 'cmpltsd',\n",
       " 'cmpnlesd',\n",
       " 'rcpss',\n",
       " 'cmplesd',\n",
       " 'movmskpd',\n",
       " 'pterm',\n",
       " 'pinit',\n",
       " 'bstr',\n",
       " 'x',\n",
       " 'y',\n",
       " 'vpaddusb',\n",
       " 'popa',\n",
       " 'buf',\n",
       " 'hugexp',\n",
       " 'vals',\n",
       " 'valid',\n",
       " 'wctab',\n",
       " 'pcinit',\n",
       " 'npows',\n",
       " 'initlocks',\n",
       " 'init',\n",
       " 'mods',\n",
       " 'nomem',\n",
       " 'vpshufhw',\n",
       " 'vpshuflw',\n",
       " 'accept',\n",
       " 'sendto',\n",
       " 'recv',\n",
       " 'select',\n",
       " 'send',\n",
       " 'connect',\n",
       " 'socket',\n",
       " 'recvfrom',\n",
       " 'vpunpckhbw',\n",
       " 'optval',\n",
       " 'himl',\n",
       " 'vorpd',\n",
       " 'fn',\n",
       " 'vdivpd',\n",
       " 'vpsubusb',\n",
       " 'vsubsd',\n",
       " 'vmovapd',\n",
       " 'retries',\n",
       " 'vpmaddwd',\n",
       " 'cmpsw',\n",
       " 'vandnpd',\n",
       " 'vmovddup',\n",
       " 'vaddsubpd',\n",
       " 'cmc',\n",
       " 'pabsw',\n",
       " 'vpaddsb',\n",
       " 'vmovntsd',\n",
       " 'vpaddusw',\n",
       " 'type',\n",
       " 'charset',\n",
       " 'hostshort',\n",
       " 'fd',\n",
       " 'fptc',\n",
       " 'lpsz',\n",
       " 'pvar',\n",
       " 'lpraststat',\n",
       " 'prgbq',\n",
       " 'hpal',\n",
       " 'ui',\n",
       " 'presbits',\n",
       " 'loopwe',\n",
       " 'pusha',\n",
       " 'vmovd',\n",
       " 'vsqrtpd',\n",
       " 'vpunpckhdq',\n",
       " 'vpextrw',\n",
       " 'prefetchw',\n",
       " 'to',\n",
       " 'audio',\n",
       " 'sample',\n",
       " 'vmptrst',\n",
       " 'pfrcp',\n",
       " 'pfcmpge',\n",
       " 'pfrsqrt',\n",
       " 'pfcmpgt',\n",
       " 'rsqrtss',\n",
       " 'femms',\n",
       " 'pfpnacc',\n",
       " 'pfnacc',\n",
       " 'pfcmpeq',\n",
       " 'pfmax',\n",
       " 'cmpneqps',\n",
       " 'pfmin',\n",
       " 'dq',\n",
       " 'lidt',\n",
       " 'movntdq',\n",
       " 'vmulps',\n",
       " 'vpcext',\n",
       " 'hwo',\n",
       " 'hwi',\n",
       " 'vpsrlq',\n",
       " 'vcomisd',\n",
       " 'vpunpckhwd',\n",
       " 'vmovsldup',\n",
       " 'plimit',\n",
       " 'vpaddd',\n",
       " 'minutes',\n",
       " 'parms',\n",
       " 'addsubpd',\n",
       " 'seems',\n",
       " 'vucomiss',\n",
       " 'arglist',\n",
       " 'vpaddw',\n",
       " 'xsaveopt',\n",
       " 'loopwne',\n",
       " 'vmovss',\n",
       " 'vmovntps',\n",
       " 'will',\n",
       " 'timeout',\n",
       " 'bytes',\n",
       " 'logarithm',\n",
       " 'vpunpckhqdq',\n",
       " 'vpcmpgtd',\n",
       " 'str',\n",
       " 'lddqu',\n",
       " 'phaddd',\n",
       " 'phaddw',\n",
       " 'allows',\n",
       " 'does',\n",
       " 'shall',\n",
       " 'palignr',\n",
       " 'lldt',\n",
       " 'verw',\n",
       " 'vcmppd',\n",
       " 'vpclmulqdq',\n",
       " 'points',\n",
       " 'pwh',\n",
       " 'lsl',\n",
       " 'separaters',\n",
       " 'evenexp',\n",
       " 'vhsubps',\n",
       " 'data',\n",
       " 'program',\n",
       " 'block',\n",
       " 'memotected',\n",
       " 'popf',\n",
       " 'vpackuswb',\n",
       " 'start',\n",
       " 'sec',\n",
       " 'vmulss',\n",
       " 'vshufpd',\n",
       " 'vpandn',\n",
       " 'pb',\n",
       " 'vpunpcklwd',\n",
       " 'pc',\n",
       " 'pptl',\n",
       " 'being',\n",
       " 'lcid',\n",
       " 'vpackssdw',\n",
       " 'ans',\n",
       " 'years',\n",
       " 'ptlib',\n",
       " 'vmovupd',\n",
       " 'vpcmpgtb',\n",
       " 'for',\n",
       " 'rect',\n",
       " 'cld',\n",
       " 'phsubd',\n",
       " 'bmp',\n",
       " 'ucb',\n",
       " 'vmaxsd',\n",
       " 'fcompp',\n",
       " 'fucompp',\n",
       " 'file',\n",
       " 'vmaxpd',\n",
       " 'gain',\n",
       " 'setcachedevice',\n",
       " 'under',\n",
       " 'vpinsrw',\n",
       " 'drive',\n",
       " 'button',\n",
       " 'degrees',\n",
       " 'vminss',\n",
       " 'vmovaps',\n",
       " 'vsubps',\n",
       " 'pmaddubsw',\n",
       " 'vshufps',\n",
       " 'vxorps',\n",
       " 'vunpcklps',\n",
       " 'vunpckhps',\n",
       " 'vpmullw',\n",
       " 'vblendps',\n",
       " 'vpermilps',\n",
       " 'pmulhrsw',\n",
       " 'movhpd',\n",
       " 'psignw',\n",
       " 'outsd',\n",
       " 'aaa',\n",
       " 'sidt',\n",
       " 'vpmulhw',\n",
       " 'vmovntpd',\n",
       " 'vpmaxub',\n",
       " 'sysexit',\n",
       " 'lpx',\n",
       " 'hbr',\n",
       " 'vrsqrtss',\n",
       " 'rdrand',\n",
       " 'profile',\n",
       " 'vpaddq',\n",
       " 'syskind',\n",
       " 'xrstor',\n",
       " 'pdm',\n",
       " 'buttons',\n",
       " 'device',\n",
       " 'kbps',\n",
       " 'power',\n",
       " 'end',\n",
       " 'proxy',\n",
       " 'vsubpd',\n",
       " 'hcur',\n",
       " 'uint',\n",
       " 'container',\n",
       " 'string',\n",
       " 'vhsubpd',\n",
       " 'log',\n",
       " 'vpaddb',\n",
       " 'vcmpps',\n",
       " 'vpsadbw',\n",
       " 'vdivss',\n",
       " 'vmovlhps',\n",
       " 'vpunpcklbw',\n",
       " 'vmaxss',\n",
       " 'topic',\n",
       " 'dfd',\n",
       " 'vmovntdq',\n",
       " 'vpsubd',\n",
       " 'vpaddsw',\n",
       " 'vunpcklpd',\n",
       " 'options',\n",
       " 'pt',\n",
       " 'pmxcd',\n",
       " 'lprect',\n",
       " 'vcmpss',\n",
       " 'mpsadbw',\n",
       " 'phminposuw',\n",
       " 'vpxor',\n",
       " 'vpcmpeqd',\n",
       " 'daa',\n",
       " 'vpsrlw',\n",
       " 'stosd']"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "col.tolist() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "1f54c125",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out67\n",
      "out2\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>db</th>\n",
       "      <th>mov</th>\n",
       "      <th>push</th>\n",
       "      <th>call</th>\n",
       "      <th>jmp</th>\n",
       "      <th>test</th>\n",
       "      <th>cmp</th>\n",
       "      <th>sub</th>\n",
       "      <th>add</th>\n",
       "      <th>jz</th>\n",
       "      <th>...</th>\n",
       "      <th>pmxcd</th>\n",
       "      <th>lprect</th>\n",
       "      <th>vcmpss</th>\n",
       "      <th>mpsadbw</th>\n",
       "      <th>phminposuw</th>\n",
       "      <th>vpxor</th>\n",
       "      <th>vpcmpeqd</th>\n",
       "      <th>daa</th>\n",
       "      <th>vpsrlw</th>\n",
       "      <th>stosd</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1366877</td>\n",
       "      <td>5295</td>\n",
       "      <td>3208</td>\n",
       "      <td>1533</td>\n",
       "      <td>730</td>\n",
       "      <td>674</td>\n",
       "      <td>672</td>\n",
       "      <td>648</td>\n",
       "      <td>622</td>\n",
       "      <td>557</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63354</td>\n",
       "      <td>2190</td>\n",
       "      <td>569</td>\n",
       "      <td>333</td>\n",
       "      <td>47</td>\n",
       "      <td>48</td>\n",
       "      <td>218</td>\n",
       "      <td>216</td>\n",
       "      <td>323</td>\n",
       "      <td>54</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1226</td>\n",
       "      <td>31</td>\n",
       "      <td>442</td>\n",
       "      <td>125</td>\n",
       "      <td>1</td>\n",
       "      <td>73</td>\n",
       "      <td>127</td>\n",
       "      <td>91</td>\n",
       "      <td>32</td>\n",
       "      <td>44</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>105</td>\n",
       "      <td>89</td>\n",
       "      <td>81</td>\n",
       "      <td>53</td>\n",
       "      <td>12</td>\n",
       "      <td>14</td>\n",
       "      <td>15</td>\n",
       "      <td>5</td>\n",
       "      <td>11</td>\n",
       "      <td>13</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5752</td>\n",
       "      <td>2535</td>\n",
       "      <td>402</td>\n",
       "      <td>194</td>\n",
       "      <td>72</td>\n",
       "      <td>6</td>\n",
       "      <td>182</td>\n",
       "      <td>363</td>\n",
       "      <td>427</td>\n",
       "      <td>78</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 719 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        db   mov  push  call  jmp test  cmp  sub  add   jz  ... pmxcd lprect  \\\n",
       "0  1366877  5295  3208  1533  730  674  672  648  622  557  ...     0      0   \n",
       "0    63354  2190   569   333   47   48  218  216  323   54  ...     0      0   \n",
       "0     1226    31   442   125    1   73  127   91   32   44  ...     0      0   \n",
       "0      105    89    81    53   12   14   15    5   11   13  ...     0      0   \n",
       "0     5752  2535   402   194   72    6  182  363  427   78  ...     0      0   \n",
       "\n",
       "  vcmpss mpsadbw phminposuw vpxor vpcmpeqd daa vpsrlw stosd  \n",
       "0      0       0          0     0        0   0      0     0  \n",
       "0      0       0          0     0        0   0      0     0  \n",
       "0      0       0          0     0        0   0      0     0  \n",
       "0      0       0          0     0        0   0      0     0  \n",
       "0      0       0          0     0        0   0      0     0  \n",
       "\n",
       "[5 rows x 719 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# creating a data frame which it's columns are all the unique OPcodes and each row contains the count of OPcodes for\n",
    "# an specific file \n",
    "\n",
    "data2=[]\n",
    "\n",
    "with RarFile(\"ASM_Features.rar\") as zipfiles:\n",
    "    \n",
    "    #the last entry is the zipfile name\n",
    "    #we'll skip it\n",
    "    filelist = zipfiles.namelist()[:-1]\n",
    "    for count,file_name in enumerate(filelist):\n",
    "\n",
    "        temp = pd.read_csv(zipfiles.open(file_name))\n",
    "        tempt = temp.drop('Unnamed: 0', axis=1).transpose()\n",
    "        tempt.columns= tempt[0:1].values[0]\n",
    "        tempt = tempt.drop([\"OPcodes\"])            \n",
    "        DATA_FRAME= pd.DataFrame(0, index=np.arange(1), columns=col)\n",
    "        DATA_FRAME[tempt.columns]=tempt.values\n",
    "        data2.append(DATA_FRAME)\n",
    "        print(count,end=\"\\r\")\n",
    "\n",
    "print(\"out\")\n",
    "DATA_FRAME = pd.concat(data2)\n",
    "print(\"out2\")\n",
    "\n",
    "DATA_FRAME.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "id": "367b41bb",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>db</th>\n",
       "      <th>mov</th>\n",
       "      <th>push</th>\n",
       "      <th>call</th>\n",
       "      <th>jmp</th>\n",
       "      <th>test</th>\n",
       "      <th>cmp</th>\n",
       "      <th>sub</th>\n",
       "      <th>add</th>\n",
       "      <th>jz</th>\n",
       "      <th>...</th>\n",
       "      <th>pmxcd</th>\n",
       "      <th>lprect</th>\n",
       "      <th>vcmpss</th>\n",
       "      <th>mpsadbw</th>\n",
       "      <th>phminposuw</th>\n",
       "      <th>vpxor</th>\n",
       "      <th>vpcmpeqd</th>\n",
       "      <th>daa</th>\n",
       "      <th>vpsrlw</th>\n",
       "      <th>stosd</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1366877</td>\n",
       "      <td>5295</td>\n",
       "      <td>3208</td>\n",
       "      <td>1533</td>\n",
       "      <td>730</td>\n",
       "      <td>674</td>\n",
       "      <td>672</td>\n",
       "      <td>648</td>\n",
       "      <td>622</td>\n",
       "      <td>557</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63354</td>\n",
       "      <td>2190</td>\n",
       "      <td>569</td>\n",
       "      <td>333</td>\n",
       "      <td>47</td>\n",
       "      <td>48</td>\n",
       "      <td>218</td>\n",
       "      <td>216</td>\n",
       "      <td>323</td>\n",
       "      <td>54</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1226</td>\n",
       "      <td>31</td>\n",
       "      <td>442</td>\n",
       "      <td>125</td>\n",
       "      <td>1</td>\n",
       "      <td>73</td>\n",
       "      <td>127</td>\n",
       "      <td>91</td>\n",
       "      <td>32</td>\n",
       "      <td>44</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>105</td>\n",
       "      <td>89</td>\n",
       "      <td>81</td>\n",
       "      <td>53</td>\n",
       "      <td>12</td>\n",
       "      <td>14</td>\n",
       "      <td>15</td>\n",
       "      <td>5</td>\n",
       "      <td>11</td>\n",
       "      <td>13</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5752</td>\n",
       "      <td>2535</td>\n",
       "      <td>402</td>\n",
       "      <td>194</td>\n",
       "      <td>72</td>\n",
       "      <td>6</td>\n",
       "      <td>182</td>\n",
       "      <td>363</td>\n",
       "      <td>427</td>\n",
       "      <td>78</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>264480</td>\n",
       "      <td>152</td>\n",
       "      <td>284</td>\n",
       "      <td>91</td>\n",
       "      <td>2</td>\n",
       "      <td>24</td>\n",
       "      <td>39</td>\n",
       "      <td>46</td>\n",
       "      <td>126</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>41915</td>\n",
       "      <td>281</td>\n",
       "      <td>462</td>\n",
       "      <td>112</td>\n",
       "      <td>286</td>\n",
       "      <td>40</td>\n",
       "      <td>78</td>\n",
       "      <td>100</td>\n",
       "      <td>158</td>\n",
       "      <td>36</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>44679</td>\n",
       "      <td>134</td>\n",
       "      <td>248</td>\n",
       "      <td>35</td>\n",
       "      <td>149</td>\n",
       "      <td>3</td>\n",
       "      <td>45</td>\n",
       "      <td>21</td>\n",
       "      <td>71</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>75982</td>\n",
       "      <td>512</td>\n",
       "      <td>879</td>\n",
       "      <td>33</td>\n",
       "      <td>1360</td>\n",
       "      <td>78</td>\n",
       "      <td>107</td>\n",
       "      <td>192</td>\n",
       "      <td>270</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>67654</td>\n",
       "      <td>474</td>\n",
       "      <td>826</td>\n",
       "      <td>33</td>\n",
       "      <td>1287</td>\n",
       "      <td>72</td>\n",
       "      <td>112</td>\n",
       "      <td>165</td>\n",
       "      <td>238</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10868 rows × 719 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         db   mov  push  call   jmp test  cmp  sub  add   jz  ... pmxcd  \\\n",
       "0   1366877  5295  3208  1533   730  674  672  648  622  557  ...     0   \n",
       "0     63354  2190   569   333    47   48  218  216  323   54  ...     0   \n",
       "0      1226    31   442   125     1   73  127   91   32   44  ...     0   \n",
       "0       105    89    81    53    12   14   15    5   11   13  ...     0   \n",
       "0      5752  2535   402   194    72    6  182  363  427   78  ...     0   \n",
       "..      ...   ...   ...   ...   ...  ...  ...  ...  ...  ...  ...   ...   \n",
       "0    264480   152   284    91     2   24   39   46  126   11  ...     0   \n",
       "0     41915   281   462   112   286   40   78  100  158   36  ...     0   \n",
       "0     44679   134   248    35   149    3   45   21   71    4  ...     0   \n",
       "0     75982   512   879    33  1360   78  107  192  270   17  ...     0   \n",
       "0     67654   474   826    33  1287   72  112  165  238   17  ...     0   \n",
       "\n",
       "   lprect vcmpss mpsadbw phminposuw vpxor vpcmpeqd daa vpsrlw stosd  \n",
       "0       0      0       0          0     0        0   0      0     0  \n",
       "0       0      0       0          0     0        0   0      0     0  \n",
       "0       0      0       0          0     0        0   0      0     0  \n",
       "0       0      0       0          0     0        0   0      0     0  \n",
       "0       0      0       0          0     0        0   0      0     0  \n",
       "..    ...    ...     ...        ...   ...      ...  ..    ...   ...  \n",
       "0       0      0       0          0     0        0   0      0     0  \n",
       "0       0      0       0          0     0        0   0      0     0  \n",
       "0       0      0       0          0     0        0   0      0     0  \n",
       "0       0      0       0          0     0        0   0      0     0  \n",
       "0       0      0       0          0     0        0   0      0     0  \n",
       "\n",
       "[10868 rows x 719 columns]"
      ]
     },
     "execution_count": 222,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DATA_FRAME"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "3e567de2",
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_FRAME.index = np.arange(DATA_FRAME.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "1b1e9eb3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>db</th>\n",
       "      <th>mov</th>\n",
       "      <th>push</th>\n",
       "      <th>call</th>\n",
       "      <th>jmp</th>\n",
       "      <th>test</th>\n",
       "      <th>cmp</th>\n",
       "      <th>sub</th>\n",
       "      <th>add</th>\n",
       "      <th>jz</th>\n",
       "      <th>...</th>\n",
       "      <th>pmxcd</th>\n",
       "      <th>lprect</th>\n",
       "      <th>vcmpss</th>\n",
       "      <th>mpsadbw</th>\n",
       "      <th>phminposuw</th>\n",
       "      <th>vpxor</th>\n",
       "      <th>vpcmpeqd</th>\n",
       "      <th>daa</th>\n",
       "      <th>vpsrlw</th>\n",
       "      <th>stosd</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1366877</td>\n",
       "      <td>5295</td>\n",
       "      <td>3208</td>\n",
       "      <td>1533</td>\n",
       "      <td>730</td>\n",
       "      <td>674</td>\n",
       "      <td>672</td>\n",
       "      <td>648</td>\n",
       "      <td>622</td>\n",
       "      <td>557</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>63354</td>\n",
       "      <td>2190</td>\n",
       "      <td>569</td>\n",
       "      <td>333</td>\n",
       "      <td>47</td>\n",
       "      <td>48</td>\n",
       "      <td>218</td>\n",
       "      <td>216</td>\n",
       "      <td>323</td>\n",
       "      <td>54</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1226</td>\n",
       "      <td>31</td>\n",
       "      <td>442</td>\n",
       "      <td>125</td>\n",
       "      <td>1</td>\n",
       "      <td>73</td>\n",
       "      <td>127</td>\n",
       "      <td>91</td>\n",
       "      <td>32</td>\n",
       "      <td>44</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>105</td>\n",
       "      <td>89</td>\n",
       "      <td>81</td>\n",
       "      <td>53</td>\n",
       "      <td>12</td>\n",
       "      <td>14</td>\n",
       "      <td>15</td>\n",
       "      <td>5</td>\n",
       "      <td>11</td>\n",
       "      <td>13</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5752</td>\n",
       "      <td>2535</td>\n",
       "      <td>402</td>\n",
       "      <td>194</td>\n",
       "      <td>72</td>\n",
       "      <td>6</td>\n",
       "      <td>182</td>\n",
       "      <td>363</td>\n",
       "      <td>427</td>\n",
       "      <td>78</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10863</th>\n",
       "      <td>264480</td>\n",
       "      <td>152</td>\n",
       "      <td>284</td>\n",
       "      <td>91</td>\n",
       "      <td>2</td>\n",
       "      <td>24</td>\n",
       "      <td>39</td>\n",
       "      <td>46</td>\n",
       "      <td>126</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10864</th>\n",
       "      <td>41915</td>\n",
       "      <td>281</td>\n",
       "      <td>462</td>\n",
       "      <td>112</td>\n",
       "      <td>286</td>\n",
       "      <td>40</td>\n",
       "      <td>78</td>\n",
       "      <td>100</td>\n",
       "      <td>158</td>\n",
       "      <td>36</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10865</th>\n",
       "      <td>44679</td>\n",
       "      <td>134</td>\n",
       "      <td>248</td>\n",
       "      <td>35</td>\n",
       "      <td>149</td>\n",
       "      <td>3</td>\n",
       "      <td>45</td>\n",
       "      <td>21</td>\n",
       "      <td>71</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10866</th>\n",
       "      <td>75982</td>\n",
       "      <td>512</td>\n",
       "      <td>879</td>\n",
       "      <td>33</td>\n",
       "      <td>1360</td>\n",
       "      <td>78</td>\n",
       "      <td>107</td>\n",
       "      <td>192</td>\n",
       "      <td>270</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10867</th>\n",
       "      <td>67654</td>\n",
       "      <td>474</td>\n",
       "      <td>826</td>\n",
       "      <td>33</td>\n",
       "      <td>1287</td>\n",
       "      <td>72</td>\n",
       "      <td>112</td>\n",
       "      <td>165</td>\n",
       "      <td>238</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10868 rows × 719 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            db   mov  push  call   jmp test  cmp  sub  add   jz  ... pmxcd  \\\n",
       "0      1366877  5295  3208  1533   730  674  672  648  622  557  ...     0   \n",
       "1        63354  2190   569   333    47   48  218  216  323   54  ...     0   \n",
       "2         1226    31   442   125     1   73  127   91   32   44  ...     0   \n",
       "3          105    89    81    53    12   14   15    5   11   13  ...     0   \n",
       "4         5752  2535   402   194    72    6  182  363  427   78  ...     0   \n",
       "...        ...   ...   ...   ...   ...  ...  ...  ...  ...  ...  ...   ...   \n",
       "10863   264480   152   284    91     2   24   39   46  126   11  ...     0   \n",
       "10864    41915   281   462   112   286   40   78  100  158   36  ...     0   \n",
       "10865    44679   134   248    35   149    3   45   21   71    4  ...     0   \n",
       "10866    75982   512   879    33  1360   78  107  192  270   17  ...     0   \n",
       "10867    67654   474   826    33  1287   72  112  165  238   17  ...     0   \n",
       "\n",
       "      lprect vcmpss mpsadbw phminposuw vpxor vpcmpeqd daa vpsrlw stosd  \n",
       "0          0      0       0          0     0        0   0      0     0  \n",
       "1          0      0       0          0     0        0   0      0     0  \n",
       "2          0      0       0          0     0        0   0      0     0  \n",
       "3          0      0       0          0     0        0   0      0     0  \n",
       "4          0      0       0          0     0        0   0      0     0  \n",
       "...      ...    ...     ...        ...   ...      ...  ..    ...   ...  \n",
       "10863      0      0       0          0     0        0   0      0     0  \n",
       "10864      0      0       0          0     0        0   0      0     0  \n",
       "10865      0      0       0          0     0        0   0      0     0  \n",
       "10866      0      0       0          0     0        0   0      0     0  \n",
       "10867      0      0       0          0     0        0   0      0     0  \n",
       "\n",
       "[10868 rows x 719 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DATA_FRAME"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "3a4b0086",
   "metadata": {},
   "outputs": [],
   "source": [
    "#saving the mentioned Dataframe\n",
    "filepath = \"DATA_FRAME.csv\"\n",
    "DATA_FRAME.to_csv(filepath)  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "51bdfa02",
   "metadata": {},
   "outputs": [],
   "source": [
    "#saving Data frame which contains class related to files\n",
    "filepath = \"class_df.csv\"\n",
    "class_df.to_csv(filepath)  \n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4d0afa9",
   "metadata": {},
   "source": [
    "# DATA Final"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e86bc30a",
   "metadata": {},
   "source": [
    "### if you have already run these cells, you can skip this section"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "bc459278",
   "metadata": {},
   "outputs": [],
   "source": [
    "#reading the dataframe of OPcde counts\n",
    "DF_temp = pd.read_csv(\"DATA_FRAME.csv\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "fd443673",
   "metadata": {},
   "outputs": [],
   "source": [
    "#reading dataframe of labels\n",
    "class_df= pd.read_csv(\"class_df.csv\") \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b4d7463",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "b409e078",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>db</th>\n",
       "      <th>mov</th>\n",
       "      <th>push</th>\n",
       "      <th>call</th>\n",
       "      <th>jmp</th>\n",
       "      <th>test</th>\n",
       "      <th>cmp</th>\n",
       "      <th>sub</th>\n",
       "      <th>add</th>\n",
       "      <th>jz</th>\n",
       "      <th>...</th>\n",
       "      <th>lprect</th>\n",
       "      <th>vcmpss</th>\n",
       "      <th>mpsadbw</th>\n",
       "      <th>phminposuw</th>\n",
       "      <th>vpxor</th>\n",
       "      <th>vpcmpeqd</th>\n",
       "      <th>daa</th>\n",
       "      <th>vpsrlw</th>\n",
       "      <th>stosd</th>\n",
       "      <th>Class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1366877</td>\n",
       "      <td>5295</td>\n",
       "      <td>3208</td>\n",
       "      <td>1533</td>\n",
       "      <td>730</td>\n",
       "      <td>674</td>\n",
       "      <td>672</td>\n",
       "      <td>648</td>\n",
       "      <td>622</td>\n",
       "      <td>557</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>63354</td>\n",
       "      <td>2190</td>\n",
       "      <td>569</td>\n",
       "      <td>333</td>\n",
       "      <td>47</td>\n",
       "      <td>48</td>\n",
       "      <td>218</td>\n",
       "      <td>216</td>\n",
       "      <td>323</td>\n",
       "      <td>54</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1226</td>\n",
       "      <td>31</td>\n",
       "      <td>442</td>\n",
       "      <td>125</td>\n",
       "      <td>1</td>\n",
       "      <td>73</td>\n",
       "      <td>127</td>\n",
       "      <td>91</td>\n",
       "      <td>32</td>\n",
       "      <td>44</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>105</td>\n",
       "      <td>89</td>\n",
       "      <td>81</td>\n",
       "      <td>53</td>\n",
       "      <td>12</td>\n",
       "      <td>14</td>\n",
       "      <td>15</td>\n",
       "      <td>5</td>\n",
       "      <td>11</td>\n",
       "      <td>13</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5752</td>\n",
       "      <td>2535</td>\n",
       "      <td>402</td>\n",
       "      <td>194</td>\n",
       "      <td>72</td>\n",
       "      <td>6</td>\n",
       "      <td>182</td>\n",
       "      <td>363</td>\n",
       "      <td>427</td>\n",
       "      <td>78</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10863</th>\n",
       "      <td>264480</td>\n",
       "      <td>152</td>\n",
       "      <td>284</td>\n",
       "      <td>91</td>\n",
       "      <td>2</td>\n",
       "      <td>24</td>\n",
       "      <td>39</td>\n",
       "      <td>46</td>\n",
       "      <td>126</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10864</th>\n",
       "      <td>41915</td>\n",
       "      <td>281</td>\n",
       "      <td>462</td>\n",
       "      <td>112</td>\n",
       "      <td>286</td>\n",
       "      <td>40</td>\n",
       "      <td>78</td>\n",
       "      <td>100</td>\n",
       "      <td>158</td>\n",
       "      <td>36</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10865</th>\n",
       "      <td>44679</td>\n",
       "      <td>134</td>\n",
       "      <td>248</td>\n",
       "      <td>35</td>\n",
       "      <td>149</td>\n",
       "      <td>3</td>\n",
       "      <td>45</td>\n",
       "      <td>21</td>\n",
       "      <td>71</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10866</th>\n",
       "      <td>75982</td>\n",
       "      <td>512</td>\n",
       "      <td>879</td>\n",
       "      <td>33</td>\n",
       "      <td>1360</td>\n",
       "      <td>78</td>\n",
       "      <td>107</td>\n",
       "      <td>192</td>\n",
       "      <td>270</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10867</th>\n",
       "      <td>67654</td>\n",
       "      <td>474</td>\n",
       "      <td>826</td>\n",
       "      <td>33</td>\n",
       "      <td>1287</td>\n",
       "      <td>72</td>\n",
       "      <td>112</td>\n",
       "      <td>165</td>\n",
       "      <td>238</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10868 rows × 720 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            db   mov  push  call   jmp  test  cmp  sub  add   jz  ...  lprect  \\\n",
       "0      1366877  5295  3208  1533   730   674  672  648  622  557  ...       0   \n",
       "1        63354  2190   569   333    47    48  218  216  323   54  ...       0   \n",
       "2         1226    31   442   125     1    73  127   91   32   44  ...       0   \n",
       "3          105    89    81    53    12    14   15    5   11   13  ...       0   \n",
       "4         5752  2535   402   194    72     6  182  363  427   78  ...       0   \n",
       "...        ...   ...   ...   ...   ...   ...  ...  ...  ...  ...  ...     ...   \n",
       "10863   264480   152   284    91     2    24   39   46  126   11  ...       0   \n",
       "10864    41915   281   462   112   286    40   78  100  158   36  ...       0   \n",
       "10865    44679   134   248    35   149     3   45   21   71    4  ...       0   \n",
       "10866    75982   512   879    33  1360    78  107  192  270   17  ...       0   \n",
       "10867    67654   474   826    33  1287    72  112  165  238   17  ...       0   \n",
       "\n",
       "       vcmpss  mpsadbw  phminposuw  vpxor  vpcmpeqd  daa  vpsrlw  stosd  Class  \n",
       "0           0        0           0      0         0    0       0      0      9  \n",
       "1           0        0           0      0         0    0       0      0      2  \n",
       "2           0        0           0      0         0    0       0      0      9  \n",
       "3           0        0           0      0         0    0       0      0      1  \n",
       "4           0        0           0      0         0    0       0      0      8  \n",
       "...       ...      ...         ...    ...       ...  ...     ...    ...    ...  \n",
       "10863       0        0           0      0         0    0       0      0      4  \n",
       "10864       0        0           0      0         0    0       0      0      4  \n",
       "10865       0        0           0      0         0    0       0      0      4  \n",
       "10866       0        0           0      0         0    0       0      0      4  \n",
       "10867       0        0           0      0         0    0       0      0      4  \n",
       "\n",
       "[10868 rows x 720 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# adding labels as last column of the main dataframe\n",
    "DF = pd.concat([DF_temp, class_df], axis=1).drop('Unnamed: 0', axis=1)\n",
    "DF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "50473ec9",
   "metadata": {},
   "outputs": [],
   "source": [
    "filepath = \"DATA_FRAME_Final.csv\"\n",
    "DF.to_csv(filepath)  "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fa1f9bcd",
   "metadata": {},
   "source": [
    "# SPARK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "89f06a75",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 10:33:19 WARN Utils: Your hostname, ERFAN resolves to a loopback address: 127.0.1.1; using 192.168.51.204 instead (on interface eth0)\n",
      "23/01/01 10:33:19 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting default log level to \"WARN\".\n",
      "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 10:33:20 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/erfan_wsl/.local/lib/python3.10/site-packages/pyspark/sql/pandas/conversion.py:474: FutureWarning: iteritems is deprecated and will be removed in a future version. Use .items instead.\n",
      "  for column, series in pdf.iteritems():\n",
      "/home/erfan_wsl/.local/lib/python3.10/site-packages/pyspark/sql/pandas/conversion.py:486: FutureWarning: iteritems is deprecated and will be removed in a future version. Use .items instead.\n",
      "  for column, series in pdf.iteritems():\n"
     ]
    }
   ],
   "source": [
    "#initializing and starting a pyspark session and transform pandas dataframe to spark dataframe\n",
    "\n",
    "from pyspark.sql import SparkSession #Import the pyspark\n",
    "from pyspark.conf import SparkConf #Import the SparkConf\n",
    "from pyspark.context import SparkContext #Import the SparkContext\n",
    "import time\n",
    "from pyspark.sql import SparkSession\n",
    "\n",
    "DF = pd.read_csv(\"DATA_FRAME_Final.csv\") \n",
    "DF = DF.drop('Unnamed: 0', axis=1)\n",
    "conf = SparkConf()\n",
    "conf.setMaster(\"local\").setAppName(\"HW2_1\") \n",
    "sc = SparkContext.getOrCreate(conf=conf)\n",
    "\n",
    "#Create PySpark SparkSession\n",
    "spark= SparkSession.builder.master(\"local\").getOrCreate()\n",
    "\n",
    "#Create PySpark DataFrame from Pandas\n",
    "sparkDF=spark.createDataFrame(DF) \n",
    "# sparkDF.printSchema()\n",
    "# sparkDF.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ded731da",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "            <div>\n",
       "                <p><b>SparkSession - in-memory</b></p>\n",
       "                \n",
       "        <div>\n",
       "            <p><b>SparkContext</b></p>\n",
       "\n",
       "            <p><a href=\"http://192.168.51.204:4040\">Spark UI</a></p>\n",
       "\n",
       "            <dl>\n",
       "              <dt>Version</dt>\n",
       "                <dd><code>v3.3.1</code></dd>\n",
       "              <dt>Master</dt>\n",
       "                <dd><code>local</code></dd>\n",
       "              <dt>AppName</dt>\n",
       "                <dd><code>HW2_1</code></dd>\n",
       "            </dl>\n",
       "        </div>\n",
       "        \n",
       "            </div>\n",
       "        "
      ],
      "text/plain": [
       "<pyspark.sql.session.SparkSession at 0x7fbcf5bb2530>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "6e7c092a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 01:11:06 WARN package: Truncated the string representation of a plan since it was too large. This behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.\n",
      "23/01/01 01:11:07 WARN TaskSetManager: Stage 0 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "sparkDF_pd = sparkDF.select(\"*\").toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "99bea986",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>db</th>\n",
       "      <th>mov</th>\n",
       "      <th>push</th>\n",
       "      <th>call</th>\n",
       "      <th>jmp</th>\n",
       "      <th>test</th>\n",
       "      <th>cmp</th>\n",
       "      <th>sub</th>\n",
       "      <th>add</th>\n",
       "      <th>jz</th>\n",
       "      <th>...</th>\n",
       "      <th>lprect</th>\n",
       "      <th>vcmpss</th>\n",
       "      <th>mpsadbw</th>\n",
       "      <th>phminposuw</th>\n",
       "      <th>vpxor</th>\n",
       "      <th>vpcmpeqd</th>\n",
       "      <th>daa</th>\n",
       "      <th>vpsrlw</th>\n",
       "      <th>stosd</th>\n",
       "      <th>Class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1366877</td>\n",
       "      <td>5295</td>\n",
       "      <td>3208</td>\n",
       "      <td>1533</td>\n",
       "      <td>730</td>\n",
       "      <td>674</td>\n",
       "      <td>672</td>\n",
       "      <td>648</td>\n",
       "      <td>622</td>\n",
       "      <td>557</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>63354</td>\n",
       "      <td>2190</td>\n",
       "      <td>569</td>\n",
       "      <td>333</td>\n",
       "      <td>47</td>\n",
       "      <td>48</td>\n",
       "      <td>218</td>\n",
       "      <td>216</td>\n",
       "      <td>323</td>\n",
       "      <td>54</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1226</td>\n",
       "      <td>31</td>\n",
       "      <td>442</td>\n",
       "      <td>125</td>\n",
       "      <td>1</td>\n",
       "      <td>73</td>\n",
       "      <td>127</td>\n",
       "      <td>91</td>\n",
       "      <td>32</td>\n",
       "      <td>44</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>105</td>\n",
       "      <td>89</td>\n",
       "      <td>81</td>\n",
       "      <td>53</td>\n",
       "      <td>12</td>\n",
       "      <td>14</td>\n",
       "      <td>15</td>\n",
       "      <td>5</td>\n",
       "      <td>11</td>\n",
       "      <td>13</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5752</td>\n",
       "      <td>2535</td>\n",
       "      <td>402</td>\n",
       "      <td>194</td>\n",
       "      <td>72</td>\n",
       "      <td>6</td>\n",
       "      <td>182</td>\n",
       "      <td>363</td>\n",
       "      <td>427</td>\n",
       "      <td>78</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10863</th>\n",
       "      <td>264480</td>\n",
       "      <td>152</td>\n",
       "      <td>284</td>\n",
       "      <td>91</td>\n",
       "      <td>2</td>\n",
       "      <td>24</td>\n",
       "      <td>39</td>\n",
       "      <td>46</td>\n",
       "      <td>126</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10864</th>\n",
       "      <td>41915</td>\n",
       "      <td>281</td>\n",
       "      <td>462</td>\n",
       "      <td>112</td>\n",
       "      <td>286</td>\n",
       "      <td>40</td>\n",
       "      <td>78</td>\n",
       "      <td>100</td>\n",
       "      <td>158</td>\n",
       "      <td>36</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10865</th>\n",
       "      <td>44679</td>\n",
       "      <td>134</td>\n",
       "      <td>248</td>\n",
       "      <td>35</td>\n",
       "      <td>149</td>\n",
       "      <td>3</td>\n",
       "      <td>45</td>\n",
       "      <td>21</td>\n",
       "      <td>71</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10866</th>\n",
       "      <td>75982</td>\n",
       "      <td>512</td>\n",
       "      <td>879</td>\n",
       "      <td>33</td>\n",
       "      <td>1360</td>\n",
       "      <td>78</td>\n",
       "      <td>107</td>\n",
       "      <td>192</td>\n",
       "      <td>270</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10867</th>\n",
       "      <td>67654</td>\n",
       "      <td>474</td>\n",
       "      <td>826</td>\n",
       "      <td>33</td>\n",
       "      <td>1287</td>\n",
       "      <td>72</td>\n",
       "      <td>112</td>\n",
       "      <td>165</td>\n",
       "      <td>238</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10868 rows × 720 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            db   mov  push  call   jmp  test  cmp  sub  add   jz  ...  lprect  \\\n",
       "0      1366877  5295  3208  1533   730   674  672  648  622  557  ...       0   \n",
       "1        63354  2190   569   333    47    48  218  216  323   54  ...       0   \n",
       "2         1226    31   442   125     1    73  127   91   32   44  ...       0   \n",
       "3          105    89    81    53    12    14   15    5   11   13  ...       0   \n",
       "4         5752  2535   402   194    72     6  182  363  427   78  ...       0   \n",
       "...        ...   ...   ...   ...   ...   ...  ...  ...  ...  ...  ...     ...   \n",
       "10863   264480   152   284    91     2    24   39   46  126   11  ...       0   \n",
       "10864    41915   281   462   112   286    40   78  100  158   36  ...       0   \n",
       "10865    44679   134   248    35   149     3   45   21   71    4  ...       0   \n",
       "10866    75982   512   879    33  1360    78  107  192  270   17  ...       0   \n",
       "10867    67654   474   826    33  1287    72  112  165  238   17  ...       0   \n",
       "\n",
       "       vcmpss  mpsadbw  phminposuw  vpxor  vpcmpeqd  daa  vpsrlw  stosd  Class  \n",
       "0           0        0           0      0         0    0       0      0      9  \n",
       "1           0        0           0      0         0    0       0      0      2  \n",
       "2           0        0           0      0         0    0       0      0      9  \n",
       "3           0        0           0      0         0    0       0      0      1  \n",
       "4           0        0           0      0         0    0       0      0      8  \n",
       "...       ...      ...         ...    ...       ...  ...     ...    ...    ...  \n",
       "10863       0        0           0      0         0    0       0      0      4  \n",
       "10864       0        0           0      0         0    0       0      0      4  \n",
       "10865       0        0           0      0         0    0       0      0      4  \n",
       "10866       0        0           0      0         0    0       0      0      4  \n",
       "10867       0        0           0      0         0    0       0      0      4  \n",
       "\n",
       "[10868 rows x 720 columns]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sparkDF_pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "6bb9984f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num Total class  1  :  1541\n",
      "num Total class  2  :  2478\n",
      "num Total class  3  :  2942\n",
      "num Total class  4  :  475\n",
      "num Total class  5  :  42\n",
      "num Total class  6  :  751\n",
      "num Total class  7  :  398\n",
      "num Total class  8  :  1228\n",
      "num Total class  9  :  1013\n"
     ]
    }
   ],
   "source": [
    "#count of each class in the dataframe\n",
    "for i in range(9):\n",
    "    print(\"num Total class \",i+1,\" : \",sum(sparkDF_pd[\"Class\"]==i+1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b283fb65",
   "metadata": {},
   "outputs": [],
   "source": [
    "#spliting the dataframe to TRAIN,TEST,VALID \n",
    "TRAIN,TEST,VALID = sparkDF.randomSplit([0.85,0.1,0.05])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "790ba274",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 10:37:19 WARN package: Truncated the string representation of a plan since it was too large. This behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.\n",
      "23/01/01 10:37:19 WARN TaskSetManager: Stage 0 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num TRAIN class  1  :  1327\n",
      "num TRAIN class  2  :  2126\n",
      "num TRAIN class  3  :  2491\n",
      "num TRAIN class  4  :  414\n",
      "num TRAIN class  5  :  39\n",
      "num TRAIN class  6  :  645\n",
      "num TRAIN class  7  :  330\n",
      "num TRAIN class  8  :  1032\n",
      "num TRAIN class  9  :  858\n"
     ]
    }
   ],
   "source": [
    "#count of each class in the TRAIN dataframe\n",
    "\n",
    "TRAIN_pd = TRAIN.select(\"*\").toPandas()\n",
    "for i in range(9):\n",
    "    print(\"num TRAIN class \",i+1,\" : \",sum(TRAIN_pd[\"Class\"]==i+1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7d0ce931",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 10:37:42 WARN TaskSetManager: Stage 1 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num TEST class  1  :  151\n",
      "num TEST class  2  :  230\n",
      "num TEST class  3  :  293\n",
      "num TEST class  4  :  45\n",
      "num TEST class  5  :  3\n",
      "num TEST class  6  :  72\n",
      "num TEST class  7  :  39\n",
      "num TEST class  8  :  119\n",
      "num TEST class  9  :  113\n"
     ]
    }
   ],
   "source": [
    "#count of each class in the TEST dataframe\n",
    "\n",
    "TEST_pd = TEST.select(\"*\").toPandas()\n",
    "for i in range(9):\n",
    "    print(\"num TEST class \",i+1,\" : \",sum(TEST_pd[\"Class\"]==i+1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "31ab45c4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 10:37:49 WARN TaskSetManager: Stage 2 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num VALID class  1  :  63\n",
      "num VALID class  2  :  122\n",
      "num VALID class  3  :  158\n",
      "num VALID class  4  :  16\n",
      "num VALID class  5  :  0\n",
      "num VALID class  6  :  34\n",
      "num VALID class  7  :  29\n",
      "num VALID class  8  :  77\n",
      "num VALID class  9  :  42\n"
     ]
    }
   ],
   "source": [
    "#count of each class in the VALID dataframe\n",
    "\n",
    "VALID_pd = VALID.select(\"*\").toPandas()\n",
    "for i in range(9):\n",
    "    print(\"num VALID class \",i+1,\" : \",sum(VALID_pd[\"Class\"]==i+1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "e57928c8",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22/12/31 13:58:02 WARN TaskSetManager: Stage 42 contains a task of very large size (15679 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22/12/31 13:58:06 WARN TaskSetManager: Stage 43 contains a task of very large size (15679 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 45:>                                                         (0 + 1) / 1]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(9246, 722)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "# print((TRAIN.count(), len(TRAIN.columns)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "50ce2a3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql.functions import rand \n",
    "# TRAIN,TEST,VALID shuffling\n",
    "TRAIN = TRAIN.orderBy(rand())\n",
    "TEST = TEST.orderBy(rand())\n",
    "VALID = VALID.orderBy(rand())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7a9a03a5",
   "metadata": {},
   "source": [
    "# Preparing inputs for Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "5e761bf6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 10:38:08 WARN TaskSetManager: Stage 3 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 10:38:10 WARN TaskSetManager: Stage 4 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Dataset Count: 9262\n",
      "23/01/01 10:38:14 WARN TaskSetManager: Stage 16 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 10:38:15 WARN TaskSetManager: Stage 17 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Dataset Count: 1065\n",
      "23/01/01 10:38:18 WARN TaskSetManager: Stage 29 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 10:38:19 WARN TaskSetManager: Stage 30 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Dataset Count: 541\n"
     ]
    }
   ],
   "source": [
    "print(\"Training Dataset Count: \" + str(TRAIN.count()))\n",
    "print(\"Test Dataset Count: \" + str(TEST.count()))\n",
    "print(\"Validation Dataset Count: \" + str(VALID.count()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "03071873",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.feature import StringIndexer, VectorAssembler\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ce4edf2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "input_cols=VALID_pd.columns[:-1]\n",
    "output_cols=\"features\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "6f831c4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#adding a features column which contains all the columns as a single list, except labels column, to all dataframes\n",
    "assembler = VectorAssembler(inputCols=input_cols.to_list(), outputCol=output_cols)\n",
    "TRAIN = assembler.transform(TRAIN)\n",
    "TEST = assembler.transform(TEST)\n",
    "VALID = assembler.transform(VALID)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "0ac727d5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 10:39:19 WARN TaskSetManager: Stage 42 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 10:39:20 WARN TaskSetManager: Stage 43 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>db</th>\n",
       "      <th>mov</th>\n",
       "      <th>push</th>\n",
       "      <th>call</th>\n",
       "      <th>jmp</th>\n",
       "      <th>test</th>\n",
       "      <th>cmp</th>\n",
       "      <th>sub</th>\n",
       "      <th>add</th>\n",
       "      <th>jz</th>\n",
       "      <th>...</th>\n",
       "      <th>vcmpss</th>\n",
       "      <th>mpsadbw</th>\n",
       "      <th>phminposuw</th>\n",
       "      <th>vpxor</th>\n",
       "      <th>vpcmpeqd</th>\n",
       "      <th>daa</th>\n",
       "      <th>vpsrlw</th>\n",
       "      <th>stosd</th>\n",
       "      <th>Class</th>\n",
       "      <th>features</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>986</td>\n",
       "      <td>158</td>\n",
       "      <td>367</td>\n",
       "      <td>172</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>51</td>\n",
       "      <td>189</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>(986.0, 158.0, 367.0, 172.0, 1.0, 6.0, 1.0, 51...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>331</td>\n",
       "      <td>66</td>\n",
       "      <td>152</td>\n",
       "      <td>64</td>\n",
       "      <td>19</td>\n",
       "      <td>14</td>\n",
       "      <td>41</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>15</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>(331.0, 66.0, 152.0, 64.0, 19.0, 14.0, 41.0, 5...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3335549</td>\n",
       "      <td>1138</td>\n",
       "      <td>451</td>\n",
       "      <td>209</td>\n",
       "      <td>83</td>\n",
       "      <td>64</td>\n",
       "      <td>147</td>\n",
       "      <td>101</td>\n",
       "      <td>113</td>\n",
       "      <td>58</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>(3335549.0, 1138.0, 451.0, 209.0, 83.0, 64.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>519248</td>\n",
       "      <td>2125</td>\n",
       "      <td>2806</td>\n",
       "      <td>967</td>\n",
       "      <td>217</td>\n",
       "      <td>325</td>\n",
       "      <td>548</td>\n",
       "      <td>102</td>\n",
       "      <td>540</td>\n",
       "      <td>361</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>(519248.0, 2125.0, 2806.0, 967.0, 217.0, 325.0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1826</td>\n",
       "      <td>38</td>\n",
       "      <td>732</td>\n",
       "      <td>208</td>\n",
       "      <td>2</td>\n",
       "      <td>122</td>\n",
       "      <td>204</td>\n",
       "      <td>257</td>\n",
       "      <td>161</td>\n",
       "      <td>89</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>(1826.0, 38.0, 732.0, 208.0, 2.0, 122.0, 204.0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>536</th>\n",
       "      <td>1048199</td>\n",
       "      <td>1712</td>\n",
       "      <td>681</td>\n",
       "      <td>333</td>\n",
       "      <td>139</td>\n",
       "      <td>172</td>\n",
       "      <td>252</td>\n",
       "      <td>178</td>\n",
       "      <td>192</td>\n",
       "      <td>110</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>(1048199.0, 1712.0, 681.0, 333.0, 139.0, 172.0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>537</th>\n",
       "      <td>1043</td>\n",
       "      <td>70</td>\n",
       "      <td>177</td>\n",
       "      <td>78</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>30</td>\n",
       "      <td>45</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>(1043.0, 70.0, 177.0, 78.0, 0.0, 1.0, 3.0, 30....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>538</th>\n",
       "      <td>1046</td>\n",
       "      <td>71</td>\n",
       "      <td>180</td>\n",
       "      <td>78</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "      <td>47</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>(1046.0, 71.0, 180.0, 78.0, 1.0, 4.0, 4.0, 28....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>539</th>\n",
       "      <td>16246</td>\n",
       "      <td>2454</td>\n",
       "      <td>428</td>\n",
       "      <td>136</td>\n",
       "      <td>68</td>\n",
       "      <td>2</td>\n",
       "      <td>209</td>\n",
       "      <td>369</td>\n",
       "      <td>484</td>\n",
       "      <td>77</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>(16246.0, 2454.0, 428.0, 136.0, 68.0, 2.0, 209...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>540</th>\n",
       "      <td>4324</td>\n",
       "      <td>2532</td>\n",
       "      <td>393</td>\n",
       "      <td>204</td>\n",
       "      <td>71</td>\n",
       "      <td>3</td>\n",
       "      <td>178</td>\n",
       "      <td>362</td>\n",
       "      <td>419</td>\n",
       "      <td>78</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>(4324.0, 2532.0, 393.0, 204.0, 71.0, 3.0, 178....</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>541 rows × 721 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          db   mov  push  call  jmp  test  cmp  sub  add   jz  ...  vcmpss  \\\n",
       "0        986   158   367   172    1     6    1   51  189    1  ...       0   \n",
       "1        331    66   152    64   19    14   41    5    7   15  ...       0   \n",
       "2    3335549  1138   451   209   83    64  147  101  113   58  ...       0   \n",
       "3     519248  2125  2806   967  217   325  548  102  540  361  ...       0   \n",
       "4       1826    38   732   208    2   122  204  257  161   89  ...       0   \n",
       "..       ...   ...   ...   ...  ...   ...  ...  ...  ...  ...  ...     ...   \n",
       "536  1048199  1712   681   333  139   172  252  178  192  110  ...       0   \n",
       "537     1043    70   177    78    0     1    3   30   45    1  ...       0   \n",
       "538     1046    71   180    78    1     4    4   28   47    1  ...       0   \n",
       "539    16246  2454   428   136   68     2  209  369  484   77  ...       0   \n",
       "540     4324  2532   393   204   71     3  178  362  419   78  ...       0   \n",
       "\n",
       "     mpsadbw  phminposuw  vpxor  vpcmpeqd  daa  vpsrlw  stosd  Class  \\\n",
       "0          0           0      0         0    0       0      0      3   \n",
       "1          0           0      0         0    0       0      0      6   \n",
       "2          0           0      0         0    0       0      0      2   \n",
       "3          0           0      0         0    0       0      0      6   \n",
       "4          0           0      0         0    0       0      0      9   \n",
       "..       ...         ...    ...       ...  ...     ...    ...    ...   \n",
       "536        0           0      0         0    0       0      0      2   \n",
       "537        0           0      0         0    0       0      0      3   \n",
       "538        0           0      0         0    0       0      0      3   \n",
       "539        0           0      0         0    0       0      0      8   \n",
       "540        0           0      0         0    0       0      0      8   \n",
       "\n",
       "                                              features  \n",
       "0    (986.0, 158.0, 367.0, 172.0, 1.0, 6.0, 1.0, 51...  \n",
       "1    (331.0, 66.0, 152.0, 64.0, 19.0, 14.0, 41.0, 5...  \n",
       "2    (3335549.0, 1138.0, 451.0, 209.0, 83.0, 64.0, ...  \n",
       "3    (519248.0, 2125.0, 2806.0, 967.0, 217.0, 325.0...  \n",
       "4    (1826.0, 38.0, 732.0, 208.0, 2.0, 122.0, 204.0...  \n",
       "..                                                 ...  \n",
       "536  (1048199.0, 1712.0, 681.0, 333.0, 139.0, 172.0...  \n",
       "537  (1043.0, 70.0, 177.0, 78.0, 0.0, 1.0, 3.0, 30....  \n",
       "538  (1046.0, 71.0, 180.0, 78.0, 1.0, 4.0, 4.0, 28....  \n",
       "539  (16246.0, 2454.0, 428.0, 136.0, 68.0, 2.0, 209...  \n",
       "540  (4324.0, 2532.0, 393.0, 204.0, 71.0, 3.0, 178....  \n",
       "\n",
       "[541 rows x 721 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# you can see the last column here\n",
    "VALID.select(\"*\").toPandas()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c59cd1cf",
   "metadata": {},
   "source": [
    "# Random Forest | numTrees=10, maxDepth=10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "3ba80b11",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.classification import RandomForestClassifier\n",
    "import time\n",
    "TIME = []\n",
    "ACC = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "a09dd976",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:01:24 WARN TaskSetManager: Stage 669 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:01:25 WARN TaskSetManager: Stage 670 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:01:51 WARN TaskSetManager: Stage 681 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:01:52 WARN TaskSetManager: Stage 682 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:01:54 WARN TaskSetManager: Stage 694 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:01:55 WARN TaskSetManager: Stage 695 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:02:04 WARN DAGScheduler: Broadcasting large task binary with size 1001.4 KiB\n",
      "23/01/01 12:02:05 WARN DAGScheduler: Broadcasting large task binary with size 1161.2 KiB\n",
      "23/01/01 12:02:06 WARN TaskSetManager: Stage 769 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:02:07 WARN TaskSetManager: Stage 770 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "#building and training a random forest classifier with numTrees=10, maxDepth=10\n",
    "\n",
    "rf = RandomForestClassifier(numTrees=10, maxDepth=10,featuresCol = \"features\", labelCol = \"Class\")\n",
    "s1 = time.time()\n",
    "rfModel = rf.fit(TRAIN)\n",
    "# testing the model\n",
    "predictions = rfModel.transform(TEST)\n",
    "s2 = time.time()\n",
    "\n",
    "# saving the training time in a list for ploting\n",
    "TIME.append(s2-s1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "1dd1553a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# predictions.select(\"*\").toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "f4b48682",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:04:26 WARN TaskSetManager: Stage 781 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:04:27 WARN TaskSetManager: Stage 782 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+----------+\n",
      "|Class|prediction|\n",
      "+-----+----------+\n",
      "|    6|       6.0|\n",
      "|    1|       1.0|\n",
      "|    4|       4.0|\n",
      "|    2|       2.0|\n",
      "|    9|       9.0|\n",
      "|    3|       3.0|\n",
      "|    3|       3.0|\n",
      "|    1|       1.0|\n",
      "|    9|       9.0|\n",
      "|    2|       2.0|\n",
      "+-----+----------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions.select(\"Class\", \"prediction\").show(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "761746fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "6e5b9961",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:04:38 WARN TaskSetManager: Stage 797 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:04:39 WARN TaskSetManager: Stage 798 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.979054122324214\n",
      "Test Error = 0.02094587767578604\n"
     ]
    }
   ],
   "source": [
    "# calculating the accuracy and test error\n",
    "\n",
    "evaluator = MulticlassClassificationEvaluator(labelCol=\"Class\", predictionCol=\"prediction\")\n",
    "accuracy = evaluator.evaluate(predictions)\n",
    "print(\"Accuracy = %s\" % (accuracy))\n",
    "print(\"Test Error = %s\" % (1.0 - accuracy))\n",
    "\n",
    "# saving the accuracy in a list for ploting\n",
    "\n",
    "ACC.append(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "f02eeee0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:04:54 WARN TaskSetManager: Stage 814 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:04:55 WARN TaskSetManager: Stage 815 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/erfan_wsl/.local/lib/python3.10/site-packages/pyspark/sql/context.py:157: FutureWarning: Deprecated in 3.0.0. Use SparkSession.builder.getOrCreate() instead.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[151.   0.   0.   0.   0.   0.   0.   0.   0.]\n",
      " [  3. 227.   0.   0.   0.   0.   0.   0.   0.]\n",
      " [  0.   0. 290.   2.   0.   1.   0.   0.   0.]\n",
      " [  0.   0.   0.  45.   0.   0.   0.   0.   0.]\n",
      " [  0.   1.   0.   0.   1.   1.   0.   0.   0.]\n",
      " [  2.   0.   0.   0.   0.  69.   0.   1.   0.]\n",
      " [  0.   0.   0.   1.   0.   0.  38.   0.   0.]\n",
      " [  6.   0.   0.   2.   0.   0.   0. 111.   0.]\n",
      " [  1.   0.   0.   0.   0.   1.   0.   0. 111.]]\n"
     ]
    }
   ],
   "source": [
    "from pyspark.mllib.evaluation import MulticlassMetrics\n",
    "from pyspark.sql.types import FloatType\n",
    "import pyspark.sql.functions as F\n",
    "\n",
    "# obtaining the confusion matrix\n",
    "\n",
    "preds_and_labels = predictions.select(['prediction','Class']).withColumn('Class', F.col('Class').cast(FloatType())).orderBy('prediction')\n",
    "preds_and_labels = preds_and_labels.select(['prediction','Class'])\n",
    "metrics = MulticlassMetrics(preds_and_labels.rdd.map(tuple))\n",
    "print(metrics.confusionMatrix().toArray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "3fb796ad",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Summary Stats\n",
      "\n",
      "Precision = [0.9263803680981595, 0.9956140350877193, 1.0, 0.9, 1.0, 0.9583333333333334, 1.0, 0.9910714285714286, 1.0]\n",
      "\n",
      "Recall = [1.0, 0.9869565217391304, 0.9897610921501706, 1.0, 0.3333333333333333, 0.9583333333333334, 0.9743589743589743, 0.9327731092436975, 0.9823008849557522]\n",
      "\n",
      "F1 Score = [0.9617834394904459, 0.9912663755458515, 0.9948542024013721, 0.9473684210526316, 0.5, 0.9583333333333334, 0.9870129870129869, 0.961038961038961, 0.9910714285714286]\n",
      "\n",
      "True Positive Rate = [1.0, 0.9869565217391304, 0.9897610921501706, 1.0, 0.3333333333333333, 0.9583333333333334, 0.9743589743589743, 0.9327731092436975, 0.9823008849557522]\n",
      "\n",
      "False Positive Rate = [0.01312910284463895, 0.0011976047904191617, 0.0, 0.004901960784313725, 0.0, 0.0030211480362537764, 0.0, 0.0010570824524312897, 0.0]\n"
     ]
    }
   ],
   "source": [
    "# Overall statistics\n",
    "# calculating precision, recall, f1Score, True Positive Rate, False Positive Rate\n",
    "# mean of these metrics have been calculated just for num_trees=30 becuase it has the best result in accuracy\n",
    "\n",
    "precision = []\n",
    "recall = []\n",
    "f1Score = []\n",
    "tpr = []\n",
    "fpr = []\n",
    "\n",
    "for i in range(9):\n",
    "    precision.append(metrics.precision(i+1.) )\n",
    "    recall.append(metrics.recall(i+1.) )\n",
    "    f1Score.append(metrics.fMeasure(i+1.) )\n",
    "    tpr.append(metrics.truePositiveRate(i+1.) )\n",
    "    fpr.append(metrics.falsePositiveRate(i+1.) )\n",
    "print(\"Summary Stats\")\n",
    "print(\"\\nPrecision = %s\" % precision)\n",
    "print(\"\\nRecall = %s\" % recall)\n",
    "print(\"\\nF1 Score = %s\" % f1Score)\n",
    "print(\"\\nTrue Positive Rate = %s\" % tpr)\n",
    "print(\"\\nFalse Positive Rate = %s\" % fpr)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "37923997",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "73208bc6",
   "metadata": {},
   "source": [
    "# Random Forest | numTrees=20, maxDepth=10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "2b35bf9d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from pyspark.ml.classification import RandomForestClassifier\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "d2259a37",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:09:58 WARN TaskSetManager: Stage 845 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:09:59 WARN TaskSetManager: Stage 846 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:10:26 WARN TaskSetManager: Stage 857 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "[Stage 857:>                                                        (0 + 1) / 1]\r",
      "\r",
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:10:26 WARN TaskSetManager: Stage 858 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:10:28 WARN TaskSetManager: Stage 870 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:10:29 WARN TaskSetManager: Stage 871 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:10:38 WARN DAGScheduler: Broadcasting large task binary with size 1255.0 KiB\n",
      "23/01/01 12:10:39 WARN DAGScheduler: Broadcasting large task binary with size 1560.0 KiB\n",
      "23/01/01 12:10:40 WARN DAGScheduler: Broadcasting large task binary with size 1873.4 KiB\n",
      "23/01/01 12:10:42 WARN TaskSetManager: Stage 945 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:10:43 WARN TaskSetManager: Stage 946 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "rf = RandomForestClassifier(numTrees=20, maxDepth=10,featuresCol = \"features\", labelCol = \"Class\")\n",
    "s1 = time.time()\n",
    "rfModel = rf.fit(TRAIN)\n",
    "predictions = rfModel.transform(TEST)\n",
    "s2 = time.time()\n",
    "TIME.append(s2-s1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "id": "7c8fce81",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22/12/31 15:37:49 WARN TaskSetManager: Stage 180 contains a task of very large size (15647 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22/12/31 15:37:50 WARN TaskSetManager: Stage 181 contains a task of very large size (15647 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>db</th>\n",
       "      <th>mov</th>\n",
       "      <th>push</th>\n",
       "      <th>call</th>\n",
       "      <th>jmp</th>\n",
       "      <th>test</th>\n",
       "      <th>cmp</th>\n",
       "      <th>sub</th>\n",
       "      <th>add</th>\n",
       "      <th>jz</th>\n",
       "      <th>...</th>\n",
       "      <th>vpcmpeqd</th>\n",
       "      <th>daa</th>\n",
       "      <th>vpsrlw</th>\n",
       "      <th>stosd</th>\n",
       "      <th>Id</th>\n",
       "      <th>Class</th>\n",
       "      <th>features</th>\n",
       "      <th>rawPrediction</th>\n",
       "      <th>probability</th>\n",
       "      <th>prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1417101</td>\n",
       "      <td>6224</td>\n",
       "      <td>2370</td>\n",
       "      <td>922</td>\n",
       "      <td>918</td>\n",
       "      <td>827</td>\n",
       "      <td>1006</td>\n",
       "      <td>881</td>\n",
       "      <td>826</td>\n",
       "      <td>697</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>ecd3A46BQyk2SFJ7IMqb</td>\n",
       "      <td>8</td>\n",
       "      <td>(1417101.0, 6224.0, 2370.0, 922.0, 918.0, 827....</td>\n",
       "      <td>[0.0, 2.93246389296825, 4.7026962518154285, 5....</td>\n",
       "      <td>[0.0, 0.14662319464841253, 0.23513481259077146...</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>506008</td>\n",
       "      <td>2092</td>\n",
       "      <td>2741</td>\n",
       "      <td>1041</td>\n",
       "      <td>245</td>\n",
       "      <td>378</td>\n",
       "      <td>546</td>\n",
       "      <td>80</td>\n",
       "      <td>543</td>\n",
       "      <td>462</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>iVAMDtFgjQHusEGmWJCT</td>\n",
       "      <td>2</td>\n",
       "      <td>(506008.0, 2092.0, 2741.0, 1041.0, 245.0, 378....</td>\n",
       "      <td>[0.0, 2.8003635965720712, 4.688393468602078, 5...</td>\n",
       "      <td>[0.0, 0.14001817982860362, 0.23441967343010398...</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>934683</td>\n",
       "      <td>1236</td>\n",
       "      <td>776</td>\n",
       "      <td>386</td>\n",
       "      <td>270</td>\n",
       "      <td>391</td>\n",
       "      <td>285</td>\n",
       "      <td>180</td>\n",
       "      <td>127</td>\n",
       "      <td>315</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>gtiXlab7VqBfK2PYzdx6</td>\n",
       "      <td>3</td>\n",
       "      <td>(934683.0, 1236.0, 776.0, 386.0, 270.0, 391.0,...</td>\n",
       "      <td>[0.0, 2.8349880509799914, 4.624234393584697, 5...</td>\n",
       "      <td>[0.0, 0.14174940254899956, 0.23121171967923485...</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>14518</td>\n",
       "      <td>3087</td>\n",
       "      <td>1919</td>\n",
       "      <td>788</td>\n",
       "      <td>468</td>\n",
       "      <td>480</td>\n",
       "      <td>601</td>\n",
       "      <td>139</td>\n",
       "      <td>694</td>\n",
       "      <td>529</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>E5LXhtYsZMWfA4ebSVUk</td>\n",
       "      <td>3</td>\n",
       "      <td>(14518.0, 3087.0, 1919.0, 788.0, 468.0, 480.0,...</td>\n",
       "      <td>[0.0, 2.8443643749834537, 4.482604228622607, 5...</td>\n",
       "      <td>[0.0, 0.14221821874917268, 0.22413021143113038...</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>76924</td>\n",
       "      <td>489</td>\n",
       "      <td>860</td>\n",
       "      <td>33</td>\n",
       "      <td>1337</td>\n",
       "      <td>69</td>\n",
       "      <td>110</td>\n",
       "      <td>191</td>\n",
       "      <td>266</td>\n",
       "      <td>19</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>I2Rtqw5TSvPfYjFkXGKr</td>\n",
       "      <td>8</td>\n",
       "      <td>(76924.0, 489.0, 860.0, 33.0, 1337.0, 69.0, 11...</td>\n",
       "      <td>[0.0, 2.2545317939040976, 5.491689163622502, 4...</td>\n",
       "      <td>[0.0, 0.1127265896952049, 0.27458445818112515,...</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1063</th>\n",
       "      <td>649</td>\n",
       "      <td>3924</td>\n",
       "      <td>211</td>\n",
       "      <td>305</td>\n",
       "      <td>298</td>\n",
       "      <td>345</td>\n",
       "      <td>367</td>\n",
       "      <td>274</td>\n",
       "      <td>465</td>\n",
       "      <td>253</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6ZO9dMHY5JkCofwTSyvm</td>\n",
       "      <td>1</td>\n",
       "      <td>(649.0, 3924.0, 211.0, 305.0, 298.0, 345.0, 36...</td>\n",
       "      <td>[0.0, 2.7903251843666257, 4.615526141603453, 5...</td>\n",
       "      <td>[0.0, 0.13951625921833127, 0.23077630708017266...</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1064</th>\n",
       "      <td>173150</td>\n",
       "      <td>5899</td>\n",
       "      <td>3472</td>\n",
       "      <td>1341</td>\n",
       "      <td>746</td>\n",
       "      <td>726</td>\n",
       "      <td>728</td>\n",
       "      <td>816</td>\n",
       "      <td>680</td>\n",
       "      <td>566</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Ei92sQqeJWtwHkcx8ouK</td>\n",
       "      <td>8</td>\n",
       "      <td>(173150.0, 5899.0, 3472.0, 1341.0, 746.0, 726....</td>\n",
       "      <td>[0.0, 2.936369598391365, 4.73270969923399, 5.3...</td>\n",
       "      <td>[0.0, 0.14681847991956826, 0.23663548496169948...</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1065</th>\n",
       "      <td>7310</td>\n",
       "      <td>2555</td>\n",
       "      <td>407</td>\n",
       "      <td>204</td>\n",
       "      <td>75</td>\n",
       "      <td>11</td>\n",
       "      <td>190</td>\n",
       "      <td>376</td>\n",
       "      <td>426</td>\n",
       "      <td>78</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5FXu1yi9MvKsJIGPC72c</td>\n",
       "      <td>2</td>\n",
       "      <td>(7310.0, 2555.0, 407.0, 204.0, 75.0, 11.0, 190...</td>\n",
       "      <td>[0.0, 2.8606905950745176, 4.40337991911419, 5....</td>\n",
       "      <td>[0.0, 0.14303452975372585, 0.22016899595570946...</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1066</th>\n",
       "      <td>7905</td>\n",
       "      <td>2295</td>\n",
       "      <td>393</td>\n",
       "      <td>132</td>\n",
       "      <td>71</td>\n",
       "      <td>6</td>\n",
       "      <td>174</td>\n",
       "      <td>327</td>\n",
       "      <td>390</td>\n",
       "      <td>70</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>75IA0ztPUf4HQJiacnoW</td>\n",
       "      <td>3</td>\n",
       "      <td>(7905.0, 2295.0, 393.0, 132.0, 71.0, 6.0, 174....</td>\n",
       "      <td>[0.0, 2.89521599003667, 4.485606767897073, 5.4...</td>\n",
       "      <td>[0.0, 0.1447607995018335, 0.22428033839485365,...</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1067</th>\n",
       "      <td>1564</td>\n",
       "      <td>7571</td>\n",
       "      <td>5004</td>\n",
       "      <td>2377</td>\n",
       "      <td>416</td>\n",
       "      <td>1539</td>\n",
       "      <td>1263</td>\n",
       "      <td>446</td>\n",
       "      <td>2417</td>\n",
       "      <td>1370</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>19BtdpKYGOSyX3oZsirc</td>\n",
       "      <td>3</td>\n",
       "      <td>(1564.0, 7571.0, 5004.0, 2377.0, 416.0, 1539.0...</td>\n",
       "      <td>[0.0, 2.7837651503632332, 4.503392494625092, 5...</td>\n",
       "      <td>[0.0, 0.1391882575181617, 0.22516962473125463,...</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1068 rows × 725 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           db   mov  push  call   jmp  test   cmp  sub   add    jz  ...  \\\n",
       "0     1417101  6224  2370   922   918   827  1006  881   826   697  ...   \n",
       "1      506008  2092  2741  1041   245   378   546   80   543   462  ...   \n",
       "2      934683  1236   776   386   270   391   285  180   127   315  ...   \n",
       "3       14518  3087  1919   788   468   480   601  139   694   529  ...   \n",
       "4       76924   489   860    33  1337    69   110  191   266    19  ...   \n",
       "...       ...   ...   ...   ...   ...   ...   ...  ...   ...   ...  ...   \n",
       "1063      649  3924   211   305   298   345   367  274   465   253  ...   \n",
       "1064   173150  5899  3472  1341   746   726   728  816   680   566  ...   \n",
       "1065     7310  2555   407   204    75    11   190  376   426    78  ...   \n",
       "1066     7905  2295   393   132    71     6   174  327   390    70  ...   \n",
       "1067     1564  7571  5004  2377   416  1539  1263  446  2417  1370  ...   \n",
       "\n",
       "      vpcmpeqd  daa  vpsrlw  stosd                    Id  Class  \\\n",
       "0            0    0       0      0  ecd3A46BQyk2SFJ7IMqb      8   \n",
       "1            0    0       0      0  iVAMDtFgjQHusEGmWJCT      2   \n",
       "2            0    0       0      0  gtiXlab7VqBfK2PYzdx6      3   \n",
       "3            0    0       0      0  E5LXhtYsZMWfA4ebSVUk      3   \n",
       "4            0    0       0      0  I2Rtqw5TSvPfYjFkXGKr      8   \n",
       "...        ...  ...     ...    ...                   ...    ...   \n",
       "1063         0    0       0      0  6ZO9dMHY5JkCofwTSyvm      1   \n",
       "1064         0    0       0      0  Ei92sQqeJWtwHkcx8ouK      8   \n",
       "1065         0    0       0      0  5FXu1yi9MvKsJIGPC72c      2   \n",
       "1066         0    0       0      0  75IA0ztPUf4HQJiacnoW      3   \n",
       "1067         0    0       0      0  19BtdpKYGOSyX3oZsirc      3   \n",
       "\n",
       "                                               features  \\\n",
       "0     (1417101.0, 6224.0, 2370.0, 922.0, 918.0, 827....   \n",
       "1     (506008.0, 2092.0, 2741.0, 1041.0, 245.0, 378....   \n",
       "2     (934683.0, 1236.0, 776.0, 386.0, 270.0, 391.0,...   \n",
       "3     (14518.0, 3087.0, 1919.0, 788.0, 468.0, 480.0,...   \n",
       "4     (76924.0, 489.0, 860.0, 33.0, 1337.0, 69.0, 11...   \n",
       "...                                                 ...   \n",
       "1063  (649.0, 3924.0, 211.0, 305.0, 298.0, 345.0, 36...   \n",
       "1064  (173150.0, 5899.0, 3472.0, 1341.0, 746.0, 726....   \n",
       "1065  (7310.0, 2555.0, 407.0, 204.0, 75.0, 11.0, 190...   \n",
       "1066  (7905.0, 2295.0, 393.0, 132.0, 71.0, 6.0, 174....   \n",
       "1067  (1564.0, 7571.0, 5004.0, 2377.0, 416.0, 1539.0...   \n",
       "\n",
       "                                          rawPrediction  \\\n",
       "0     [0.0, 2.93246389296825, 4.7026962518154285, 5....   \n",
       "1     [0.0, 2.8003635965720712, 4.688393468602078, 5...   \n",
       "2     [0.0, 2.8349880509799914, 4.624234393584697, 5...   \n",
       "3     [0.0, 2.8443643749834537, 4.482604228622607, 5...   \n",
       "4     [0.0, 2.2545317939040976, 5.491689163622502, 4...   \n",
       "...                                                 ...   \n",
       "1063  [0.0, 2.7903251843666257, 4.615526141603453, 5...   \n",
       "1064  [0.0, 2.936369598391365, 4.73270969923399, 5.3...   \n",
       "1065  [0.0, 2.8606905950745176, 4.40337991911419, 5....   \n",
       "1066  [0.0, 2.89521599003667, 4.485606767897073, 5.4...   \n",
       "1067  [0.0, 2.7837651503632332, 4.503392494625092, 5...   \n",
       "\n",
       "                                            probability  prediction  \n",
       "0     [0.0, 0.14662319464841253, 0.23513481259077146...         3.0  \n",
       "1     [0.0, 0.14001817982860362, 0.23441967343010398...         3.0  \n",
       "2     [0.0, 0.14174940254899956, 0.23121171967923485...         3.0  \n",
       "3     [0.0, 0.14221821874917268, 0.22413021143113038...         3.0  \n",
       "4     [0.0, 0.1127265896952049, 0.27458445818112515,...         2.0  \n",
       "...                                                 ...         ...  \n",
       "1063  [0.0, 0.13951625921833127, 0.23077630708017266...         3.0  \n",
       "1064  [0.0, 0.14681847991956826, 0.23663548496169948...         3.0  \n",
       "1065  [0.0, 0.14303452975372585, 0.22016899595570946...         3.0  \n",
       "1066  [0.0, 0.1447607995018335, 0.22428033839485365,...         3.0  \n",
       "1067  [0.0, 0.1391882575181617, 0.22516962473125463,...         3.0  \n",
       "\n",
       "[1068 rows x 725 columns]"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# predictions.select(\"*\").toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "fc106e1c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:10:55 WARN TaskSetManager: Stage 957 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:10:56 WARN TaskSetManager: Stage 958 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:10:58 WARN DAGScheduler: Broadcasting large task binary with size 1478.4 KiB\n",
      "+-----+----------+\n",
      "|Class|prediction|\n",
      "+-----+----------+\n",
      "|    6|       6.0|\n",
      "|    1|       1.0|\n",
      "|    4|       4.0|\n",
      "|    2|       2.0|\n",
      "|    9|       9.0|\n",
      "|    3|       3.0|\n",
      "|    3|       3.0|\n",
      "|    1|       1.0|\n",
      "|    9|       9.0|\n",
      "|    2|       2.0|\n",
      "+-----+----------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions.select(\"Class\", \"prediction\").show(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "dd95a626",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "75f7eec8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:11:13 WARN TaskSetManager: Stage 973 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:11:14 WARN TaskSetManager: Stage 974 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:11:16 WARN DAGScheduler: Broadcasting large task binary with size 1495.5 KiB\n",
      "Accuracy = 0.9818416124499808\n",
      "Test Error = 0.018158387550019195\n"
     ]
    }
   ],
   "source": [
    "evaluator = MulticlassClassificationEvaluator(labelCol=\"Class\", predictionCol=\"prediction\")\n",
    "accuracy = evaluator.evaluate(predictions)\n",
    "print(\"Accuracy = %s\" % (accuracy))\n",
    "print(\"Test Error = %s\" % (1.0 - accuracy))\n",
    "ACC.append(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "490880bf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:11:20 WARN TaskSetManager: Stage 990 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:11:21 WARN TaskSetManager: Stage 991 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:11:23 WARN DAGScheduler: Broadcasting large task binary with size 1479.7 KiB\n",
      "23/01/01 12:11:24 WARN DAGScheduler: Broadcasting large task binary with size 1479.7 KiB\n",
      "23/01/01 12:11:24 WARN DAGScheduler: Broadcasting large task binary with size 1482.7 KiB\n",
      "23/01/01 12:11:24 WARN DAGScheduler: Broadcasting large task binary with size 1492.2 KiB\n",
      "[[151.   0.   0.   0.   0.   0.   0.   0.   0.]\n",
      " [  2. 228.   0.   0.   0.   0.   0.   0.   0.]\n",
      " [  0.   0. 291.   1.   0.   0.   0.   1.   0.]\n",
      " [  0.   0.   0.  45.   0.   0.   0.   0.   0.]\n",
      " [  1.   0.   0.   0.   1.   0.   0.   1.   0.]\n",
      " [  1.   0.   0.   0.   0.  70.   0.   1.   0.]\n",
      " [  0.   0.   0.   1.   0.   0.  38.   0.   0.]\n",
      " [  7.   0.   0.   1.   0.   0.   0. 111.   0.]\n",
      " [  2.   0.   0.   0.   0.   0.   0.   0. 111.]]\n"
     ]
    }
   ],
   "source": [
    "from pyspark.mllib.evaluation import MulticlassMetrics\n",
    "from pyspark.sql.types import FloatType\n",
    "import pyspark.sql.functions as F\n",
    "\n",
    "preds_and_labels = predictions.select(['prediction','Class']).withColumn('Class', F.col('Class').cast(FloatType())).orderBy('prediction')\n",
    "preds_and_labels = preds_and_labels.select(['prediction','Class'])\n",
    "metrics = MulticlassMetrics(preds_and_labels.rdd.map(tuple))\n",
    "print(metrics.confusionMatrix().toArray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "d7857f7c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Summary Stats\n",
      "\n",
      "Precision = [0.9207317073170732, 1.0, 1.0, 0.9375, 1.0, 1.0, 1.0, 0.9736842105263158, 1.0]\n",
      "\n",
      "Recall = [1.0, 0.991304347826087, 0.9931740614334471, 1.0, 0.3333333333333333, 0.9722222222222222, 0.9743589743589743, 0.9327731092436975, 0.9823008849557522]\n",
      "\n",
      "F1 Score = [0.9587301587301587, 0.9956331877729258, 0.9965753424657535, 0.967741935483871, 0.5, 0.9859154929577464, 0.9870129870129869, 0.9527896995708156, 0.9910714285714286]\n",
      "\n",
      "True Positive Rate = [1.0, 0.991304347826087, 0.9931740614334471, 1.0, 0.3333333333333333, 0.9722222222222222, 0.9743589743589743, 0.9327731092436975, 0.9823008849557522]\n",
      "\n",
      "False Positive Rate = [0.014223194748358862, 0.0, 0.0, 0.0029411764705882353, 0.0, 0.0, 0.0, 0.003171247357293869, 0.0]\n"
     ]
    }
   ],
   "source": [
    "# Overall statistics\n",
    "\n",
    "precision = []\n",
    "recall = []\n",
    "f1Score = []\n",
    "tpr = []\n",
    "fpr = []\n",
    "# metrics.truePositiveRate\n",
    "# metrics.falsePositiveRate\n",
    "for i in range(9):\n",
    "    precision.append(metrics.precision(i+1.) )\n",
    "    recall.append(metrics.recall(i+1.) )\n",
    "    f1Score.append(metrics.fMeasure(i+1.) )\n",
    "    tpr.append(metrics.truePositiveRate(i+1.) )\n",
    "    fpr.append(metrics.falsePositiveRate(i+1.) )\n",
    "print(\"Summary Stats\")\n",
    "print(\"\\nPrecision = %s\" % precision)\n",
    "print(\"\\nRecall = %s\" % recall)\n",
    "print(\"\\nF1 Score = %s\" % f1Score)\n",
    "print(\"\\nTrue Positive Rate = %s\" % tpr)\n",
    "print(\"\\nFalse Positive Rate = %s\" % fpr)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b461c96",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "bae332b8",
   "metadata": {},
   "source": [
    "# Random Forest | numTrees=30, maxDepth=10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "ae058b61",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from pyspark.ml.classification import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "c78bfa18",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:12:11 WARN TaskSetManager: Stage 1021 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:12:12 WARN TaskSetManager: Stage 1022 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:12:38 WARN TaskSetManager: Stage 1033 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "[Stage 1033:>                                                       (0 + 1) / 1]\r",
      "\r",
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:12:39 WARN TaskSetManager: Stage 1034 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:12:41 WARN TaskSetManager: Stage 1046 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:12:42 WARN TaskSetManager: Stage 1047 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:12:51 WARN DAGScheduler: Broadcasting large task binary with size 1234.5 KiB\n",
      "23/01/01 12:12:52 WARN DAGScheduler: Broadcasting large task binary with size 1637.3 KiB\n",
      "23/01/01 12:12:53 WARN DAGScheduler: Broadcasting large task binary with size 2.0 MiB\n",
      "23/01/01 12:12:54 WARN DAGScheduler: Broadcasting large task binary with size 2.5 MiB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:12:56 WARN TaskSetManager: Stage 1121 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:12:57 WARN TaskSetManager: Stage 1122 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "rf = RandomForestClassifier(numTrees=30, maxDepth=10,featuresCol = \"features\", labelCol = \"Class\")\n",
    "s1 = time.time()\n",
    "rfModel = rf.fit(TRAIN)\n",
    "predictions = rfModel.transform(TEST)\n",
    "s2 = time.time()\n",
    "TIME.append(s2-s1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "7144c70f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:14:01 WARN TaskSetManager: Stage 1133 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:14:02 WARN TaskSetManager: Stage 1134 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:14:04 WARN DAGScheduler: Broadcasting large task binary with size 1966.1 KiB\n",
      "+-----+----------+\n",
      "|Class|prediction|\n",
      "+-----+----------+\n",
      "|    6|       6.0|\n",
      "|    1|       1.0|\n",
      "|    4|       4.0|\n",
      "|    2|       2.0|\n",
      "|    9|       9.0|\n",
      "|    3|       3.0|\n",
      "|    3|       3.0|\n",
      "|    1|       1.0|\n",
      "|    9|       9.0|\n",
      "|    2|       2.0|\n",
      "+-----+----------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions.select(\"Class\", \"prediction\").show(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "9b977867",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "bc750c48",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:14:15 WARN TaskSetManager: Stage 1149 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:14:16 WARN TaskSetManager: Stage 1150 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:14:18 WARN DAGScheduler: Broadcasting large task binary with size 1983.3 KiB\n",
      "Accuracy = 0.9836834345632279\n",
      "Test Error = 0.016316565436772068\n"
     ]
    }
   ],
   "source": [
    "evaluator = MulticlassClassificationEvaluator(labelCol=\"Class\", predictionCol=\"prediction\")\n",
    "accuracy = evaluator.evaluate(predictions)\n",
    "print(\"Accuracy = %s\" % (accuracy))\n",
    "print(\"Test Error = %s\" % (1.0 - accuracy))\n",
    "ACC.append(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "32761e9b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:14:32 WARN TaskSetManager: Stage 1166 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:14:33 WARN TaskSetManager: Stage 1167 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:14:35 WARN DAGScheduler: Broadcasting large task binary with size 1967.4 KiB\n",
      "23/01/01 12:14:35 WARN DAGScheduler: Broadcasting large task binary with size 1967.4 KiB\n",
      "23/01/01 12:14:36 WARN DAGScheduler: Broadcasting large task binary with size 1970.4 KiB\n",
      "23/01/01 12:14:36 WARN DAGScheduler: Broadcasting large task binary with size 1979.9 KiB\n",
      "[[151.   0.   0.   0.   0.   0.   0.   0.   0.]\n",
      " [  1. 229.   0.   0.   0.   0.   0.   0.   0.]\n",
      " [  0.   0. 292.   1.   0.   0.   0.   0.   0.]\n",
      " [  0.   0.   0.  44.   0.   1.   0.   0.   0.]\n",
      " [  1.   0.   0.   0.   1.   0.   0.   1.   0.]\n",
      " [  2.   0.   0.   0.   0.  69.   0.   1.   0.]\n",
      " [  0.   0.   0.   1.   0.   0.  38.   0.   0.]\n",
      " [  5.   0.   0.   1.   0.   0.   0. 113.   0.]\n",
      " [  2.   0.   0.   0.   0.   0.   0.   0. 111.]]\n"
     ]
    }
   ],
   "source": [
    "from pyspark.mllib.evaluation import MulticlassMetrics\n",
    "from pyspark.sql.types import FloatType\n",
    "import pyspark.sql.functions as F\n",
    "\n",
    "preds_and_labels = predictions.select(['prediction','Class']).withColumn('Class', F.col('Class').cast(FloatType())).orderBy('prediction')\n",
    "preds_and_labels = preds_and_labels.select(['prediction','Class'])\n",
    "metrics = MulticlassMetrics(preds_and_labels.rdd.map(tuple))\n",
    "print(metrics.confusionMatrix().toArray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "f775977e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Summary Stats\n",
      "\n",
      "Precision = [0.9320987654320988, 1.0, 1.0, 0.9361702127659575, 1.0, 0.9857142857142858, 1.0, 0.9826086956521739, 1.0]\n",
      "\n",
      "Recall = [1.0, 0.9956521739130435, 0.9965870307167235, 0.9777777777777777, 0.3333333333333333, 0.9583333333333334, 0.9743589743589743, 0.9495798319327731, 0.9823008849557522]\n",
      "\n",
      "F1 Score = [0.9648562300319489, 0.9978213507625272, 0.9982905982905983, 0.9565217391304347, 0.5, 0.971830985915493, 0.9870129870129869, 0.9658119658119659, 0.9910714285714286]\n",
      "\n",
      "True Positive Rate = [1.0, 0.9956521739130435, 0.9965870307167235, 0.9777777777777777, 0.3333333333333333, 0.9583333333333334, 0.9743589743589743, 0.9495798319327731, 0.9823008849557522]\n",
      "\n",
      "False Positive Rate = [0.012035010940919038, 0.0, 0.0, 0.0029411764705882353, 0.0, 0.0010070493454179255, 0.0, 0.0021141649048625794, 0.0]\n"
     ]
    }
   ],
   "source": [
    "# Overall statistics\n",
    "\n",
    "precision = []\n",
    "recall = []\n",
    "f1Score = []\n",
    "tpr = []\n",
    "fpr = []\n",
    "# metrics.truePositiveRate\n",
    "# metrics.falsePositiveRate\n",
    "for i in range(9):\n",
    "    precision.append(metrics.precision(i+1.) )\n",
    "    recall.append(metrics.recall(i+1.) )\n",
    "    f1Score.append(metrics.fMeasure(i+1.) )\n",
    "    tpr.append(metrics.truePositiveRate(i+1.) )\n",
    "    fpr.append(metrics.falsePositiveRate(i+1.) )\n",
    "print(\"Summary Stats\")\n",
    "print(\"\\nPrecision = %s\" % precision)\n",
    "print(\"\\nRecall = %s\" % recall)\n",
    "print(\"\\nF1 Score = %s\" % f1Score)\n",
    "print(\"\\nTrue Positive Rate = %s\" % tpr)\n",
    "print(\"\\nFalse Positive Rate = %s\" % fpr)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "id": "35721d61",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Summary mean Stats\n",
      "\n",
      "Precision = 0.9818435510627239\n",
      "\n",
      "Recall = 0.9075470378135234\n",
      "\n",
      "F1 Score = 0.9259130317252647\n",
      "\n",
      "True Positive Rate = 0.9075470378135234\n",
      "\n",
      "False Positive Rate = 0.0020108224068653086\n"
     ]
    }
   ],
   "source": [
    "# calculaing the means for all the above metrics \n",
    "\n",
    "precision_m = np.mean(precision)\n",
    "recall_m = np.mean(recall)\n",
    "f1Score_m = np.mean(f1Score)\n",
    "tpr_m = np.mean(tpr)\n",
    "fpr_m = np.mean(fpr)\n",
    "\n",
    "print(\"Summary mean Stats\")\n",
    "print(\"\\nPrecision = %s\" % precision_m)\n",
    "print(\"\\nRecall = %s\" % recall_m)\n",
    "print(\"\\nF1 Score = %s\" % f1Score_m)\n",
    "print(\"\\nTrue Positive Rate = %s\" % tpr_m)\n",
    "print(\"\\nFalse Positive Rate = %s\" % fpr_m)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1e22c89e",
   "metadata": {},
   "source": [
    "# Random Forest | numTrees=40, maxDepth=10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "d2e68cbf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from pyspark.ml.classification import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "c3f07b3a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:15:15 WARN TaskSetManager: Stage 1197 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:15:16 WARN TaskSetManager: Stage 1198 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:15:41 WARN TaskSetManager: Stage 1209 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "[Stage 1209:>                                                       (0 + 1) / 1]\r",
      "\r",
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:15:42 WARN TaskSetManager: Stage 1210 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:15:43 WARN TaskSetManager: Stage 1222 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:15:44 WARN TaskSetManager: Stage 1223 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:15:53 WARN DAGScheduler: Broadcasting large task binary with size 1084.0 KiB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:15:54 WARN DAGScheduler: Broadcasting large task binary with size 1500.5 KiB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:15:55 WARN DAGScheduler: Broadcasting large task binary with size 2.0 MiB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:15:57 WARN DAGScheduler: Broadcasting large task binary with size 2.6 MiB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:15:58 WARN DAGScheduler: Broadcasting large task binary with size 3.2 MiB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:16:00 WARN TaskSetManager: Stage 1297 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:16:01 WARN TaskSetManager: Stage 1298 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "rf = RandomForestClassifier(numTrees=40, maxDepth=10,featuresCol = \"features\", labelCol = \"Class\")\n",
    "s1 = time.time()\n",
    "rfModel = rf.fit(TRAIN)\n",
    "predictions = rfModel.transform(TEST)\n",
    "s2 = time.time()\n",
    "TIME.append(s2-s1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "id": "6b1803f3",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22/12/31 15:37:49 WARN TaskSetManager: Stage 180 contains a task of very large size (15647 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22/12/31 15:37:50 WARN TaskSetManager: Stage 181 contains a task of very large size (15647 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>db</th>\n",
       "      <th>mov</th>\n",
       "      <th>push</th>\n",
       "      <th>call</th>\n",
       "      <th>jmp</th>\n",
       "      <th>test</th>\n",
       "      <th>cmp</th>\n",
       "      <th>sub</th>\n",
       "      <th>add</th>\n",
       "      <th>jz</th>\n",
       "      <th>...</th>\n",
       "      <th>vpcmpeqd</th>\n",
       "      <th>daa</th>\n",
       "      <th>vpsrlw</th>\n",
       "      <th>stosd</th>\n",
       "      <th>Id</th>\n",
       "      <th>Class</th>\n",
       "      <th>features</th>\n",
       "      <th>rawPrediction</th>\n",
       "      <th>probability</th>\n",
       "      <th>prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1417101</td>\n",
       "      <td>6224</td>\n",
       "      <td>2370</td>\n",
       "      <td>922</td>\n",
       "      <td>918</td>\n",
       "      <td>827</td>\n",
       "      <td>1006</td>\n",
       "      <td>881</td>\n",
       "      <td>826</td>\n",
       "      <td>697</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>ecd3A46BQyk2SFJ7IMqb</td>\n",
       "      <td>8</td>\n",
       "      <td>(1417101.0, 6224.0, 2370.0, 922.0, 918.0, 827....</td>\n",
       "      <td>[0.0, 2.93246389296825, 4.7026962518154285, 5....</td>\n",
       "      <td>[0.0, 0.14662319464841253, 0.23513481259077146...</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>506008</td>\n",
       "      <td>2092</td>\n",
       "      <td>2741</td>\n",
       "      <td>1041</td>\n",
       "      <td>245</td>\n",
       "      <td>378</td>\n",
       "      <td>546</td>\n",
       "      <td>80</td>\n",
       "      <td>543</td>\n",
       "      <td>462</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>iVAMDtFgjQHusEGmWJCT</td>\n",
       "      <td>2</td>\n",
       "      <td>(506008.0, 2092.0, 2741.0, 1041.0, 245.0, 378....</td>\n",
       "      <td>[0.0, 2.8003635965720712, 4.688393468602078, 5...</td>\n",
       "      <td>[0.0, 0.14001817982860362, 0.23441967343010398...</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>934683</td>\n",
       "      <td>1236</td>\n",
       "      <td>776</td>\n",
       "      <td>386</td>\n",
       "      <td>270</td>\n",
       "      <td>391</td>\n",
       "      <td>285</td>\n",
       "      <td>180</td>\n",
       "      <td>127</td>\n",
       "      <td>315</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>gtiXlab7VqBfK2PYzdx6</td>\n",
       "      <td>3</td>\n",
       "      <td>(934683.0, 1236.0, 776.0, 386.0, 270.0, 391.0,...</td>\n",
       "      <td>[0.0, 2.8349880509799914, 4.624234393584697, 5...</td>\n",
       "      <td>[0.0, 0.14174940254899956, 0.23121171967923485...</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>14518</td>\n",
       "      <td>3087</td>\n",
       "      <td>1919</td>\n",
       "      <td>788</td>\n",
       "      <td>468</td>\n",
       "      <td>480</td>\n",
       "      <td>601</td>\n",
       "      <td>139</td>\n",
       "      <td>694</td>\n",
       "      <td>529</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>E5LXhtYsZMWfA4ebSVUk</td>\n",
       "      <td>3</td>\n",
       "      <td>(14518.0, 3087.0, 1919.0, 788.0, 468.0, 480.0,...</td>\n",
       "      <td>[0.0, 2.8443643749834537, 4.482604228622607, 5...</td>\n",
       "      <td>[0.0, 0.14221821874917268, 0.22413021143113038...</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>76924</td>\n",
       "      <td>489</td>\n",
       "      <td>860</td>\n",
       "      <td>33</td>\n",
       "      <td>1337</td>\n",
       "      <td>69</td>\n",
       "      <td>110</td>\n",
       "      <td>191</td>\n",
       "      <td>266</td>\n",
       "      <td>19</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>I2Rtqw5TSvPfYjFkXGKr</td>\n",
       "      <td>8</td>\n",
       "      <td>(76924.0, 489.0, 860.0, 33.0, 1337.0, 69.0, 11...</td>\n",
       "      <td>[0.0, 2.2545317939040976, 5.491689163622502, 4...</td>\n",
       "      <td>[0.0, 0.1127265896952049, 0.27458445818112515,...</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1063</th>\n",
       "      <td>649</td>\n",
       "      <td>3924</td>\n",
       "      <td>211</td>\n",
       "      <td>305</td>\n",
       "      <td>298</td>\n",
       "      <td>345</td>\n",
       "      <td>367</td>\n",
       "      <td>274</td>\n",
       "      <td>465</td>\n",
       "      <td>253</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6ZO9dMHY5JkCofwTSyvm</td>\n",
       "      <td>1</td>\n",
       "      <td>(649.0, 3924.0, 211.0, 305.0, 298.0, 345.0, 36...</td>\n",
       "      <td>[0.0, 2.7903251843666257, 4.615526141603453, 5...</td>\n",
       "      <td>[0.0, 0.13951625921833127, 0.23077630708017266...</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1064</th>\n",
       "      <td>173150</td>\n",
       "      <td>5899</td>\n",
       "      <td>3472</td>\n",
       "      <td>1341</td>\n",
       "      <td>746</td>\n",
       "      <td>726</td>\n",
       "      <td>728</td>\n",
       "      <td>816</td>\n",
       "      <td>680</td>\n",
       "      <td>566</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Ei92sQqeJWtwHkcx8ouK</td>\n",
       "      <td>8</td>\n",
       "      <td>(173150.0, 5899.0, 3472.0, 1341.0, 746.0, 726....</td>\n",
       "      <td>[0.0, 2.936369598391365, 4.73270969923399, 5.3...</td>\n",
       "      <td>[0.0, 0.14681847991956826, 0.23663548496169948...</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1065</th>\n",
       "      <td>7310</td>\n",
       "      <td>2555</td>\n",
       "      <td>407</td>\n",
       "      <td>204</td>\n",
       "      <td>75</td>\n",
       "      <td>11</td>\n",
       "      <td>190</td>\n",
       "      <td>376</td>\n",
       "      <td>426</td>\n",
       "      <td>78</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5FXu1yi9MvKsJIGPC72c</td>\n",
       "      <td>2</td>\n",
       "      <td>(7310.0, 2555.0, 407.0, 204.0, 75.0, 11.0, 190...</td>\n",
       "      <td>[0.0, 2.8606905950745176, 4.40337991911419, 5....</td>\n",
       "      <td>[0.0, 0.14303452975372585, 0.22016899595570946...</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1066</th>\n",
       "      <td>7905</td>\n",
       "      <td>2295</td>\n",
       "      <td>393</td>\n",
       "      <td>132</td>\n",
       "      <td>71</td>\n",
       "      <td>6</td>\n",
       "      <td>174</td>\n",
       "      <td>327</td>\n",
       "      <td>390</td>\n",
       "      <td>70</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>75IA0ztPUf4HQJiacnoW</td>\n",
       "      <td>3</td>\n",
       "      <td>(7905.0, 2295.0, 393.0, 132.0, 71.0, 6.0, 174....</td>\n",
       "      <td>[0.0, 2.89521599003667, 4.485606767897073, 5.4...</td>\n",
       "      <td>[0.0, 0.1447607995018335, 0.22428033839485365,...</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1067</th>\n",
       "      <td>1564</td>\n",
       "      <td>7571</td>\n",
       "      <td>5004</td>\n",
       "      <td>2377</td>\n",
       "      <td>416</td>\n",
       "      <td>1539</td>\n",
       "      <td>1263</td>\n",
       "      <td>446</td>\n",
       "      <td>2417</td>\n",
       "      <td>1370</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>19BtdpKYGOSyX3oZsirc</td>\n",
       "      <td>3</td>\n",
       "      <td>(1564.0, 7571.0, 5004.0, 2377.0, 416.0, 1539.0...</td>\n",
       "      <td>[0.0, 2.7837651503632332, 4.503392494625092, 5...</td>\n",
       "      <td>[0.0, 0.1391882575181617, 0.22516962473125463,...</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1068 rows × 725 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           db   mov  push  call   jmp  test   cmp  sub   add    jz  ...  \\\n",
       "0     1417101  6224  2370   922   918   827  1006  881   826   697  ...   \n",
       "1      506008  2092  2741  1041   245   378   546   80   543   462  ...   \n",
       "2      934683  1236   776   386   270   391   285  180   127   315  ...   \n",
       "3       14518  3087  1919   788   468   480   601  139   694   529  ...   \n",
       "4       76924   489   860    33  1337    69   110  191   266    19  ...   \n",
       "...       ...   ...   ...   ...   ...   ...   ...  ...   ...   ...  ...   \n",
       "1063      649  3924   211   305   298   345   367  274   465   253  ...   \n",
       "1064   173150  5899  3472  1341   746   726   728  816   680   566  ...   \n",
       "1065     7310  2555   407   204    75    11   190  376   426    78  ...   \n",
       "1066     7905  2295   393   132    71     6   174  327   390    70  ...   \n",
       "1067     1564  7571  5004  2377   416  1539  1263  446  2417  1370  ...   \n",
       "\n",
       "      vpcmpeqd  daa  vpsrlw  stosd                    Id  Class  \\\n",
       "0            0    0       0      0  ecd3A46BQyk2SFJ7IMqb      8   \n",
       "1            0    0       0      0  iVAMDtFgjQHusEGmWJCT      2   \n",
       "2            0    0       0      0  gtiXlab7VqBfK2PYzdx6      3   \n",
       "3            0    0       0      0  E5LXhtYsZMWfA4ebSVUk      3   \n",
       "4            0    0       0      0  I2Rtqw5TSvPfYjFkXGKr      8   \n",
       "...        ...  ...     ...    ...                   ...    ...   \n",
       "1063         0    0       0      0  6ZO9dMHY5JkCofwTSyvm      1   \n",
       "1064         0    0       0      0  Ei92sQqeJWtwHkcx8ouK      8   \n",
       "1065         0    0       0      0  5FXu1yi9MvKsJIGPC72c      2   \n",
       "1066         0    0       0      0  75IA0ztPUf4HQJiacnoW      3   \n",
       "1067         0    0       0      0  19BtdpKYGOSyX3oZsirc      3   \n",
       "\n",
       "                                               features  \\\n",
       "0     (1417101.0, 6224.0, 2370.0, 922.0, 918.0, 827....   \n",
       "1     (506008.0, 2092.0, 2741.0, 1041.0, 245.0, 378....   \n",
       "2     (934683.0, 1236.0, 776.0, 386.0, 270.0, 391.0,...   \n",
       "3     (14518.0, 3087.0, 1919.0, 788.0, 468.0, 480.0,...   \n",
       "4     (76924.0, 489.0, 860.0, 33.0, 1337.0, 69.0, 11...   \n",
       "...                                                 ...   \n",
       "1063  (649.0, 3924.0, 211.0, 305.0, 298.0, 345.0, 36...   \n",
       "1064  (173150.0, 5899.0, 3472.0, 1341.0, 746.0, 726....   \n",
       "1065  (7310.0, 2555.0, 407.0, 204.0, 75.0, 11.0, 190...   \n",
       "1066  (7905.0, 2295.0, 393.0, 132.0, 71.0, 6.0, 174....   \n",
       "1067  (1564.0, 7571.0, 5004.0, 2377.0, 416.0, 1539.0...   \n",
       "\n",
       "                                          rawPrediction  \\\n",
       "0     [0.0, 2.93246389296825, 4.7026962518154285, 5....   \n",
       "1     [0.0, 2.8003635965720712, 4.688393468602078, 5...   \n",
       "2     [0.0, 2.8349880509799914, 4.624234393584697, 5...   \n",
       "3     [0.0, 2.8443643749834537, 4.482604228622607, 5...   \n",
       "4     [0.0, 2.2545317939040976, 5.491689163622502, 4...   \n",
       "...                                                 ...   \n",
       "1063  [0.0, 2.7903251843666257, 4.615526141603453, 5...   \n",
       "1064  [0.0, 2.936369598391365, 4.73270969923399, 5.3...   \n",
       "1065  [0.0, 2.8606905950745176, 4.40337991911419, 5....   \n",
       "1066  [0.0, 2.89521599003667, 4.485606767897073, 5.4...   \n",
       "1067  [0.0, 2.7837651503632332, 4.503392494625092, 5...   \n",
       "\n",
       "                                            probability  prediction  \n",
       "0     [0.0, 0.14662319464841253, 0.23513481259077146...         3.0  \n",
       "1     [0.0, 0.14001817982860362, 0.23441967343010398...         3.0  \n",
       "2     [0.0, 0.14174940254899956, 0.23121171967923485...         3.0  \n",
       "3     [0.0, 0.14221821874917268, 0.22413021143113038...         3.0  \n",
       "4     [0.0, 0.1127265896952049, 0.27458445818112515,...         2.0  \n",
       "...                                                 ...         ...  \n",
       "1063  [0.0, 0.13951625921833127, 0.23077630708017266...         3.0  \n",
       "1064  [0.0, 0.14681847991956826, 0.23663548496169948...         3.0  \n",
       "1065  [0.0, 0.14303452975372585, 0.22016899595570946...         3.0  \n",
       "1066  [0.0, 0.1447607995018335, 0.22428033839485365,...         3.0  \n",
       "1067  [0.0, 0.1391882575181617, 0.22516962473125463,...         3.0  \n",
       "\n",
       "[1068 rows x 725 columns]"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# predictions.select(\"*\").toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "269b85b5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:17:51 WARN TaskSetManager: Stage 1309 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:17:52 WARN TaskSetManager: Stage 1310 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:17:54 WARN DAGScheduler: Broadcasting large task binary with size 2.4 MiB\n",
      "+-----+----------+\n",
      "|Class|prediction|\n",
      "+-----+----------+\n",
      "|    6|       6.0|\n",
      "|    1|       1.0|\n",
      "|    4|       4.0|\n",
      "|    2|       2.0|\n",
      "|    9|       9.0|\n",
      "|    3|       3.0|\n",
      "|    3|       3.0|\n",
      "|    1|       1.0|\n",
      "|    9|       9.0|\n",
      "|    2|       2.0|\n",
      "+-----+----------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions.select(\"Class\", \"prediction\").show(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "d4020e0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "c9330da1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:18:03 WARN TaskSetManager: Stage 1325 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:18:04 WARN TaskSetManager: Stage 1326 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:18:05 WARN DAGScheduler: Broadcasting large task binary with size 2.4 MiB\n",
      "Accuracy = 0.9827820210291215\n",
      "Test Error = 0.01721797897087851\n"
     ]
    }
   ],
   "source": [
    "evaluator = MulticlassClassificationEvaluator(labelCol=\"Class\", predictionCol=\"prediction\")\n",
    "accuracy = evaluator.evaluate(predictions)\n",
    "print(\"Accuracy = %s\" % (accuracy))\n",
    "print(\"Test Error = %s\" % (1.0 - accuracy))\n",
    "ACC.append(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "73050b95",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:18:25 WARN TaskSetManager: Stage 1342 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:18:26 WARN TaskSetManager: Stage 1343 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:18:30 WARN DAGScheduler: Broadcasting large task binary with size 2.4 MiB\n",
      "23/01/01 12:18:30 WARN DAGScheduler: Broadcasting large task binary with size 2.4 MiB\n",
      "23/01/01 12:18:31 WARN DAGScheduler: Broadcasting large task binary with size 2.4 MiB\n",
      "23/01/01 12:18:31 WARN DAGScheduler: Broadcasting large task binary with size 2.4 MiB\n",
      "[[151.   0.   0.   0.   0.   0.   0.   0.   0.]\n",
      " [  1. 229.   0.   0.   0.   0.   0.   0.   0.]\n",
      " [  0.   0. 291.   1.   0.   1.   0.   0.   0.]\n",
      " [  0.   0.   0.  45.   0.   0.   0.   0.   0.]\n",
      " [  1.   0.   0.   0.   1.   0.   0.   1.   0.]\n",
      " [  2.   0.   0.   1.   0.  69.   0.   0.   0.]\n",
      " [  0.   0.   0.   1.   0.   0.  38.   0.   0.]\n",
      " [  6.   0.   0.   1.   0.   0.   0. 112.   0.]\n",
      " [  2.   0.   0.   0.   0.   0.   0.   0. 111.]]\n"
     ]
    }
   ],
   "source": [
    "from pyspark.mllib.evaluation import MulticlassMetrics\n",
    "from pyspark.sql.types import FloatType\n",
    "import pyspark.sql.functions as F\n",
    "\n",
    "preds_and_labels = predictions.select(['prediction','Class']).withColumn('Class', F.col('Class').cast(FloatType())).orderBy('prediction')\n",
    "preds_and_labels = preds_and_labels.select(['prediction','Class'])\n",
    "metrics = MulticlassMetrics(preds_and_labels.rdd.map(tuple))\n",
    "print(metrics.confusionMatrix().toArray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "0f197c3a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Summary Stats\n",
      "\n",
      "Precision = [0.9263803680981595, 1.0, 1.0, 0.9183673469387755, 1.0, 0.9857142857142858, 1.0, 0.9911504424778761, 1.0]\n",
      "\n",
      "Recall = [1.0, 0.9956521739130435, 0.9931740614334471, 1.0, 0.3333333333333333, 0.9583333333333334, 0.9743589743589743, 0.9411764705882353, 0.9823008849557522]\n",
      "\n",
      "F1 Score = [0.9617834394904459, 0.9978213507625272, 0.9965753424657535, 0.9574468085106383, 0.5, 0.971830985915493, 0.9870129870129869, 0.9655172413793104, 0.9910714285714286]\n",
      "\n",
      "True Positive Rate = [1.0, 0.9956521739130435, 0.9931740614334471, 1.0, 0.3333333333333333, 0.9583333333333334, 0.9743589743589743, 0.9411764705882353, 0.9823008849557522]\n",
      "\n",
      "False Positive Rate = [0.01312910284463895, 0.0, 0.0, 0.00392156862745098, 0.0, 0.0010070493454179255, 0.0, 0.0010570824524312897, 0.0]\n"
     ]
    }
   ],
   "source": [
    "# Overall statistics\n",
    "\n",
    "precision = []\n",
    "recall = []\n",
    "f1Score = []\n",
    "tpr = []\n",
    "fpr = []\n",
    "# metrics.truePositiveRate\n",
    "# metrics.falsePositiveRate\n",
    "for i in range(9):\n",
    "    precision.append(metrics.precision(i+1.) )\n",
    "    recall.append(metrics.recall(i+1.) )\n",
    "    f1Score.append(metrics.fMeasure(i+1.) )\n",
    "    tpr.append(metrics.truePositiveRate(i+1.) )\n",
    "    fpr.append(metrics.falsePositiveRate(i+1.) )\n",
    "print(\"Summary Stats\")\n",
    "print(\"\\nPrecision = %s\" % precision)\n",
    "print(\"\\nRecall = %s\" % recall)\n",
    "print(\"\\nF1 Score = %s\" % f1Score)\n",
    "print(\"\\nTrue Positive Rate = %s\" % tpr)\n",
    "print(\"\\nFalse Positive Rate = %s\" % fpr)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "71952836",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "18456607",
   "metadata": {},
   "source": [
    "# Random Forest | numTrees=50, maxDepth=10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "33e924a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from pyspark.ml.classification import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "a6e89310",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:19:55 WARN TaskSetManager: Stage 1373 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:19:56 WARN TaskSetManager: Stage 1374 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:20:24 WARN TaskSetManager: Stage 1385 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:20:25 WARN TaskSetManager: Stage 1386 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:20:26 WARN TaskSetManager: Stage 1398 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:20:27 WARN TaskSetManager: Stage 1399 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:20:36 WARN DAGScheduler: Broadcasting large task binary with size 1235.8 KiB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:20:37 WARN DAGScheduler: Broadcasting large task binary with size 1745.2 KiB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:20:39 WARN DAGScheduler: Broadcasting large task binary with size 2.4 MiB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:20:40 WARN DAGScheduler: Broadcasting large task binary with size 3.1 MiB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:20:42 WARN DAGScheduler: Broadcasting large task binary with size 3.9 MiB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:20:45 WARN TaskSetManager: Stage 1473 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:20:46 WARN TaskSetManager: Stage 1474 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "rf = RandomForestClassifier(numTrees=50, maxDepth=10,featuresCol = \"features\", labelCol = \"Class\")\n",
    "s1 = time.time()\n",
    "rfModel = rf.fit(TRAIN)\n",
    "predictions = rfModel.transform(TEST)\n",
    "s2 = time.time()\n",
    "TIME.append(s2-s1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "id": "2f900b35",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22/12/31 15:37:49 WARN TaskSetManager: Stage 180 contains a task of very large size (15647 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22/12/31 15:37:50 WARN TaskSetManager: Stage 181 contains a task of very large size (15647 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>db</th>\n",
       "      <th>mov</th>\n",
       "      <th>push</th>\n",
       "      <th>call</th>\n",
       "      <th>jmp</th>\n",
       "      <th>test</th>\n",
       "      <th>cmp</th>\n",
       "      <th>sub</th>\n",
       "      <th>add</th>\n",
       "      <th>jz</th>\n",
       "      <th>...</th>\n",
       "      <th>vpcmpeqd</th>\n",
       "      <th>daa</th>\n",
       "      <th>vpsrlw</th>\n",
       "      <th>stosd</th>\n",
       "      <th>Id</th>\n",
       "      <th>Class</th>\n",
       "      <th>features</th>\n",
       "      <th>rawPrediction</th>\n",
       "      <th>probability</th>\n",
       "      <th>prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1417101</td>\n",
       "      <td>6224</td>\n",
       "      <td>2370</td>\n",
       "      <td>922</td>\n",
       "      <td>918</td>\n",
       "      <td>827</td>\n",
       "      <td>1006</td>\n",
       "      <td>881</td>\n",
       "      <td>826</td>\n",
       "      <td>697</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>ecd3A46BQyk2SFJ7IMqb</td>\n",
       "      <td>8</td>\n",
       "      <td>(1417101.0, 6224.0, 2370.0, 922.0, 918.0, 827....</td>\n",
       "      <td>[0.0, 2.93246389296825, 4.7026962518154285, 5....</td>\n",
       "      <td>[0.0, 0.14662319464841253, 0.23513481259077146...</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>506008</td>\n",
       "      <td>2092</td>\n",
       "      <td>2741</td>\n",
       "      <td>1041</td>\n",
       "      <td>245</td>\n",
       "      <td>378</td>\n",
       "      <td>546</td>\n",
       "      <td>80</td>\n",
       "      <td>543</td>\n",
       "      <td>462</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>iVAMDtFgjQHusEGmWJCT</td>\n",
       "      <td>2</td>\n",
       "      <td>(506008.0, 2092.0, 2741.0, 1041.0, 245.0, 378....</td>\n",
       "      <td>[0.0, 2.8003635965720712, 4.688393468602078, 5...</td>\n",
       "      <td>[0.0, 0.14001817982860362, 0.23441967343010398...</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>934683</td>\n",
       "      <td>1236</td>\n",
       "      <td>776</td>\n",
       "      <td>386</td>\n",
       "      <td>270</td>\n",
       "      <td>391</td>\n",
       "      <td>285</td>\n",
       "      <td>180</td>\n",
       "      <td>127</td>\n",
       "      <td>315</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>gtiXlab7VqBfK2PYzdx6</td>\n",
       "      <td>3</td>\n",
       "      <td>(934683.0, 1236.0, 776.0, 386.0, 270.0, 391.0,...</td>\n",
       "      <td>[0.0, 2.8349880509799914, 4.624234393584697, 5...</td>\n",
       "      <td>[0.0, 0.14174940254899956, 0.23121171967923485...</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>14518</td>\n",
       "      <td>3087</td>\n",
       "      <td>1919</td>\n",
       "      <td>788</td>\n",
       "      <td>468</td>\n",
       "      <td>480</td>\n",
       "      <td>601</td>\n",
       "      <td>139</td>\n",
       "      <td>694</td>\n",
       "      <td>529</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>E5LXhtYsZMWfA4ebSVUk</td>\n",
       "      <td>3</td>\n",
       "      <td>(14518.0, 3087.0, 1919.0, 788.0, 468.0, 480.0,...</td>\n",
       "      <td>[0.0, 2.8443643749834537, 4.482604228622607, 5...</td>\n",
       "      <td>[0.0, 0.14221821874917268, 0.22413021143113038...</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>76924</td>\n",
       "      <td>489</td>\n",
       "      <td>860</td>\n",
       "      <td>33</td>\n",
       "      <td>1337</td>\n",
       "      <td>69</td>\n",
       "      <td>110</td>\n",
       "      <td>191</td>\n",
       "      <td>266</td>\n",
       "      <td>19</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>I2Rtqw5TSvPfYjFkXGKr</td>\n",
       "      <td>8</td>\n",
       "      <td>(76924.0, 489.0, 860.0, 33.0, 1337.0, 69.0, 11...</td>\n",
       "      <td>[0.0, 2.2545317939040976, 5.491689163622502, 4...</td>\n",
       "      <td>[0.0, 0.1127265896952049, 0.27458445818112515,...</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1063</th>\n",
       "      <td>649</td>\n",
       "      <td>3924</td>\n",
       "      <td>211</td>\n",
       "      <td>305</td>\n",
       "      <td>298</td>\n",
       "      <td>345</td>\n",
       "      <td>367</td>\n",
       "      <td>274</td>\n",
       "      <td>465</td>\n",
       "      <td>253</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6ZO9dMHY5JkCofwTSyvm</td>\n",
       "      <td>1</td>\n",
       "      <td>(649.0, 3924.0, 211.0, 305.0, 298.0, 345.0, 36...</td>\n",
       "      <td>[0.0, 2.7903251843666257, 4.615526141603453, 5...</td>\n",
       "      <td>[0.0, 0.13951625921833127, 0.23077630708017266...</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1064</th>\n",
       "      <td>173150</td>\n",
       "      <td>5899</td>\n",
       "      <td>3472</td>\n",
       "      <td>1341</td>\n",
       "      <td>746</td>\n",
       "      <td>726</td>\n",
       "      <td>728</td>\n",
       "      <td>816</td>\n",
       "      <td>680</td>\n",
       "      <td>566</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Ei92sQqeJWtwHkcx8ouK</td>\n",
       "      <td>8</td>\n",
       "      <td>(173150.0, 5899.0, 3472.0, 1341.0, 746.0, 726....</td>\n",
       "      <td>[0.0, 2.936369598391365, 4.73270969923399, 5.3...</td>\n",
       "      <td>[0.0, 0.14681847991956826, 0.23663548496169948...</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1065</th>\n",
       "      <td>7310</td>\n",
       "      <td>2555</td>\n",
       "      <td>407</td>\n",
       "      <td>204</td>\n",
       "      <td>75</td>\n",
       "      <td>11</td>\n",
       "      <td>190</td>\n",
       "      <td>376</td>\n",
       "      <td>426</td>\n",
       "      <td>78</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5FXu1yi9MvKsJIGPC72c</td>\n",
       "      <td>2</td>\n",
       "      <td>(7310.0, 2555.0, 407.0, 204.0, 75.0, 11.0, 190...</td>\n",
       "      <td>[0.0, 2.8606905950745176, 4.40337991911419, 5....</td>\n",
       "      <td>[0.0, 0.14303452975372585, 0.22016899595570946...</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1066</th>\n",
       "      <td>7905</td>\n",
       "      <td>2295</td>\n",
       "      <td>393</td>\n",
       "      <td>132</td>\n",
       "      <td>71</td>\n",
       "      <td>6</td>\n",
       "      <td>174</td>\n",
       "      <td>327</td>\n",
       "      <td>390</td>\n",
       "      <td>70</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>75IA0ztPUf4HQJiacnoW</td>\n",
       "      <td>3</td>\n",
       "      <td>(7905.0, 2295.0, 393.0, 132.0, 71.0, 6.0, 174....</td>\n",
       "      <td>[0.0, 2.89521599003667, 4.485606767897073, 5.4...</td>\n",
       "      <td>[0.0, 0.1447607995018335, 0.22428033839485365,...</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1067</th>\n",
       "      <td>1564</td>\n",
       "      <td>7571</td>\n",
       "      <td>5004</td>\n",
       "      <td>2377</td>\n",
       "      <td>416</td>\n",
       "      <td>1539</td>\n",
       "      <td>1263</td>\n",
       "      <td>446</td>\n",
       "      <td>2417</td>\n",
       "      <td>1370</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>19BtdpKYGOSyX3oZsirc</td>\n",
       "      <td>3</td>\n",
       "      <td>(1564.0, 7571.0, 5004.0, 2377.0, 416.0, 1539.0...</td>\n",
       "      <td>[0.0, 2.7837651503632332, 4.503392494625092, 5...</td>\n",
       "      <td>[0.0, 0.1391882575181617, 0.22516962473125463,...</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1068 rows × 725 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           db   mov  push  call   jmp  test   cmp  sub   add    jz  ...  \\\n",
       "0     1417101  6224  2370   922   918   827  1006  881   826   697  ...   \n",
       "1      506008  2092  2741  1041   245   378   546   80   543   462  ...   \n",
       "2      934683  1236   776   386   270   391   285  180   127   315  ...   \n",
       "3       14518  3087  1919   788   468   480   601  139   694   529  ...   \n",
       "4       76924   489   860    33  1337    69   110  191   266    19  ...   \n",
       "...       ...   ...   ...   ...   ...   ...   ...  ...   ...   ...  ...   \n",
       "1063      649  3924   211   305   298   345   367  274   465   253  ...   \n",
       "1064   173150  5899  3472  1341   746   726   728  816   680   566  ...   \n",
       "1065     7310  2555   407   204    75    11   190  376   426    78  ...   \n",
       "1066     7905  2295   393   132    71     6   174  327   390    70  ...   \n",
       "1067     1564  7571  5004  2377   416  1539  1263  446  2417  1370  ...   \n",
       "\n",
       "      vpcmpeqd  daa  vpsrlw  stosd                    Id  Class  \\\n",
       "0            0    0       0      0  ecd3A46BQyk2SFJ7IMqb      8   \n",
       "1            0    0       0      0  iVAMDtFgjQHusEGmWJCT      2   \n",
       "2            0    0       0      0  gtiXlab7VqBfK2PYzdx6      3   \n",
       "3            0    0       0      0  E5LXhtYsZMWfA4ebSVUk      3   \n",
       "4            0    0       0      0  I2Rtqw5TSvPfYjFkXGKr      8   \n",
       "...        ...  ...     ...    ...                   ...    ...   \n",
       "1063         0    0       0      0  6ZO9dMHY5JkCofwTSyvm      1   \n",
       "1064         0    0       0      0  Ei92sQqeJWtwHkcx8ouK      8   \n",
       "1065         0    0       0      0  5FXu1yi9MvKsJIGPC72c      2   \n",
       "1066         0    0       0      0  75IA0ztPUf4HQJiacnoW      3   \n",
       "1067         0    0       0      0  19BtdpKYGOSyX3oZsirc      3   \n",
       "\n",
       "                                               features  \\\n",
       "0     (1417101.0, 6224.0, 2370.0, 922.0, 918.0, 827....   \n",
       "1     (506008.0, 2092.0, 2741.0, 1041.0, 245.0, 378....   \n",
       "2     (934683.0, 1236.0, 776.0, 386.0, 270.0, 391.0,...   \n",
       "3     (14518.0, 3087.0, 1919.0, 788.0, 468.0, 480.0,...   \n",
       "4     (76924.0, 489.0, 860.0, 33.0, 1337.0, 69.0, 11...   \n",
       "...                                                 ...   \n",
       "1063  (649.0, 3924.0, 211.0, 305.0, 298.0, 345.0, 36...   \n",
       "1064  (173150.0, 5899.0, 3472.0, 1341.0, 746.0, 726....   \n",
       "1065  (7310.0, 2555.0, 407.0, 204.0, 75.0, 11.0, 190...   \n",
       "1066  (7905.0, 2295.0, 393.0, 132.0, 71.0, 6.0, 174....   \n",
       "1067  (1564.0, 7571.0, 5004.0, 2377.0, 416.0, 1539.0...   \n",
       "\n",
       "                                          rawPrediction  \\\n",
       "0     [0.0, 2.93246389296825, 4.7026962518154285, 5....   \n",
       "1     [0.0, 2.8003635965720712, 4.688393468602078, 5...   \n",
       "2     [0.0, 2.8349880509799914, 4.624234393584697, 5...   \n",
       "3     [0.0, 2.8443643749834537, 4.482604228622607, 5...   \n",
       "4     [0.0, 2.2545317939040976, 5.491689163622502, 4...   \n",
       "...                                                 ...   \n",
       "1063  [0.0, 2.7903251843666257, 4.615526141603453, 5...   \n",
       "1064  [0.0, 2.936369598391365, 4.73270969923399, 5.3...   \n",
       "1065  [0.0, 2.8606905950745176, 4.40337991911419, 5....   \n",
       "1066  [0.0, 2.89521599003667, 4.485606767897073, 5.4...   \n",
       "1067  [0.0, 2.7837651503632332, 4.503392494625092, 5...   \n",
       "\n",
       "                                            probability  prediction  \n",
       "0     [0.0, 0.14662319464841253, 0.23513481259077146...         3.0  \n",
       "1     [0.0, 0.14001817982860362, 0.23441967343010398...         3.0  \n",
       "2     [0.0, 0.14174940254899956, 0.23121171967923485...         3.0  \n",
       "3     [0.0, 0.14221821874917268, 0.22413021143113038...         3.0  \n",
       "4     [0.0, 0.1127265896952049, 0.27458445818112515,...         2.0  \n",
       "...                                                 ...         ...  \n",
       "1063  [0.0, 0.13951625921833127, 0.23077630708017266...         3.0  \n",
       "1064  [0.0, 0.14681847991956826, 0.23663548496169948...         3.0  \n",
       "1065  [0.0, 0.14303452975372585, 0.22016899595570946...         3.0  \n",
       "1066  [0.0, 0.1447607995018335, 0.22428033839485365,...         3.0  \n",
       "1067  [0.0, 0.1391882575181617, 0.22516962473125463,...         3.0  \n",
       "\n",
       "[1068 rows x 725 columns]"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# predictions.select(\"*\").toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "8871d889",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:21:24 WARN TaskSetManager: Stage 1485 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:21:25 WARN TaskSetManager: Stage 1486 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:21:29 WARN DAGScheduler: Broadcasting large task binary with size 2.9 MiB\n",
      "+-----+----------+\n",
      "|Class|prediction|\n",
      "+-----+----------+\n",
      "|    6|       6.0|\n",
      "|    1|       1.0|\n",
      "|    4|       4.0|\n",
      "|    2|       2.0|\n",
      "|    9|       9.0|\n",
      "|    3|       3.0|\n",
      "|    3|       3.0|\n",
      "|    1|       1.0|\n",
      "|    9|       9.0|\n",
      "|    2|       2.0|\n",
      "+-----+----------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions.select(\"Class\", \"prediction\").show(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "19c6b994",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "1d33d940",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:21:37 WARN TaskSetManager: Stage 1501 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "[Stage 1501:>                                                       (0 + 1) / 1]\r",
      "\r",
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:21:37 WARN TaskSetManager: Stage 1502 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:21:39 WARN DAGScheduler: Broadcasting large task binary with size 2.9 MiB\n",
      "Accuracy = 0.9818302143699189\n",
      "Test Error = 0.018169785630081092\n"
     ]
    }
   ],
   "source": [
    "evaluator = MulticlassClassificationEvaluator(labelCol=\"Class\", predictionCol=\"prediction\")\n",
    "accuracy = evaluator.evaluate(predictions)\n",
    "print(\"Accuracy = %s\" % (accuracy))\n",
    "print(\"Test Error = %s\" % (1.0 - accuracy))\n",
    "ACC.append(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "290714fb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:21:44 WARN TaskSetManager: Stage 1518 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "[Stage 1518:>                                                       (0 + 1) / 1]\r",
      "\r",
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:21:44 WARN TaskSetManager: Stage 1519 contains a task of very large size (15403 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/01/01 12:21:46 WARN DAGScheduler: Broadcasting large task binary with size 2.9 MiB\n",
      "23/01/01 12:21:47 WARN DAGScheduler: Broadcasting large task binary with size 2.9 MiB\n",
      "23/01/01 12:21:47 WARN DAGScheduler: Broadcasting large task binary with size 2.9 MiB\n",
      "23/01/01 12:21:48 WARN DAGScheduler: Broadcasting large task binary with size 2.9 MiB\n",
      "[[151.   0.   0.   0.   0.   0.   0.   0.   0.]\n",
      " [  2. 228.   0.   0.   0.   0.   0.   0.   0.]\n",
      " [  0.   0. 291.   0.   0.   0.   1.   1.   0.]\n",
      " [  0.   0.   0.  45.   0.   0.   0.   0.   0.]\n",
      " [  0.   0.   0.   0.   1.   0.   0.   2.   0.]\n",
      " [  3.   0.   0.   0.   0.  68.   0.   1.   0.]\n",
      " [  0.   0.   0.   0.   0.   0.  39.   0.   0.]\n",
      " [  6.   0.   0.   0.   0.   0.   1. 112.   0.]\n",
      " [  2.   0.   0.   0.   0.   0.   0.   0. 111.]]\n"
     ]
    }
   ],
   "source": [
    "from pyspark.mllib.evaluation import MulticlassMetrics\n",
    "from pyspark.sql.types import FloatType\n",
    "import pyspark.sql.functions as F\n",
    "\n",
    "preds_and_labels = predictions.select(['prediction','Class']).withColumn('Class', F.col('Class').cast(FloatType())).orderBy('prediction')\n",
    "preds_and_labels = preds_and_labels.select(['prediction','Class'])\n",
    "metrics = MulticlassMetrics(preds_and_labels.rdd.map(tuple))\n",
    "print(metrics.confusionMatrix().toArray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "acf6b859",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Summary Stats\n",
      "\n",
      "Precision = [0.9207317073170732, 1.0, 1.0, 1.0, 1.0, 1.0, 0.9512195121951219, 0.9655172413793104, 1.0]\n",
      "\n",
      "Recall = [1.0, 0.991304347826087, 0.9931740614334471, 1.0, 0.3333333333333333, 0.9444444444444444, 1.0, 0.9411764705882353, 0.9823008849557522]\n",
      "\n",
      "F1 Score = [0.9587301587301587, 0.9956331877729258, 0.9965753424657535, 1.0, 0.5, 0.9714285714285714, 0.975, 0.9531914893617022, 0.9910714285714286]\n",
      "\n",
      "True Positive Rate = [1.0, 0.991304347826087, 0.9931740614334471, 1.0, 0.3333333333333333, 0.9444444444444444, 1.0, 0.9411764705882353, 0.9823008849557522]\n",
      "\n",
      "False Positive Rate = [0.014223194748358862, 0.0, 0.0, 0.0, 0.0, 0.0, 0.001949317738791423, 0.004228329809725159, 0.0]\n"
     ]
    }
   ],
   "source": [
    "# Overall statistics\n",
    "\n",
    "precision = []\n",
    "recall = []\n",
    "f1Score = []\n",
    "tpr = []\n",
    "fpr = []\n",
    "# metrics.truePositiveRate\n",
    "# metrics.falsePositiveRate\n",
    "for i in range(9):\n",
    "    precision.append(metrics.precision(i+1.) )\n",
    "    recall.append(metrics.recall(i+1.) )\n",
    "    f1Score.append(metrics.fMeasure(i+1.) )\n",
    "    tpr.append(metrics.truePositiveRate(i+1.) )\n",
    "    fpr.append(metrics.falsePositiveRate(i+1.) )\n",
    "print(\"Summary Stats\")\n",
    "print(\"\\nPrecision = %s\" % precision)\n",
    "print(\"\\nRecall = %s\" % recall)\n",
    "print(\"\\nF1 Score = %s\" % f1Score)\n",
    "print(\"\\nTrue Positive Rate = %s\" % tpr)\n",
    "print(\"\\nFalse Positive Rate = %s\" % fpr)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "280024da",
   "metadata": {},
   "source": [
    "# ACC and time plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "7d68d7a1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[60.8727753162384,\n",
       " 58.80563688278198,\n",
       " 61.73721790313721,\n",
       " 63.06793761253357,\n",
       " 67.4094226360321]"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TIME"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "733894a1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.979054122324214,\n",
       " 0.9818416124499808,\n",
       " 0.9836834345632279,\n",
       " 0.9827820210291215,\n",
       " 0.9818302143699189]"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ACC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "8aa2aa46",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[10, 20, 30, 40, 50]"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TREES = [10, 20, 30, 40, 50]\n",
    "TREES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "b3e2f5c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEWCAYAAABhffzLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAArMUlEQVR4nO3dd3hUZd7G8e+PEEroXUEIYqH3sIooiCBioSgBEZAmL8FFV1111cVVcRfX9V1UXAETpSlRqkoXEBDpmtAhICoKSFfpLSTP+8cM+0YkmGAmZzJzf64rFzNnyrnzkNw5ec7JOeacQ0REwkc+rwOIiEjuUvGLiIQZFb+ISJhR8YuIhBkVv4hImFHxi4iEGRW/eMrMNpnZzV7nEAkn+b0OIKHNzI5luBsFnAbS/PfjnHO1cz9V3mJm3YF4/90IoCBw4tzjzrmiXuSSvMv0B1ySW8zsO6Cfc+5Tr7PkVf7fjsY7567I5PEI51zahR4TOUdTPeIpM/vOzFr7b79gZpPNbLyZHTWzDWZ2rZk9Y2b7zWynmbXJ8NoSZjbKzPaY2Q9m9g8zi7jAOiqa2UkzK51hWUMzO2hmkWZ2tZktNrPD/mUTM8la1cycmfUysx3+5w7K8PhYM/tHhvs3m9mu8z7XJ81svZkd92evYGZz/J/vp2ZWKpvjN9bMRprZbDM7DrT0f75TzeyAmW03sz9leH4+M3vazL4xsx/NbNK5cTGzQv6x/9HMDpnZl2ZWITt5JG9Q8UuwaQe8B5QC1gBz8X2dVgJe5P+nPADGAWeBq4GGQBug3/lv6JzbDawAOmVY3A2Y4pxLBf4OzPOv8wrgP7+R8UagOtAKeM7Mambj8+sE3Apci+9znQP8FSjr/zz/lPlLM9UNGAIUA5YDM4B1+MasFfComd3mf+6fgI5AC6Ai8DMw3P9YL6AEUBkoAwwATl5CHglyKn4JNkucc3Odc2eByUA54GV/QU8AqppZSf+W6O3Ao8654865/cBrQNdM3vd94D4AMzP/8973P5YKRAMVnXOnnHNLfyPjYOfcSefcOnwFWz8bn99/nHP7nHM/AEuAVc65Nc6508BH+H6AZdc059wy51w6UBco55x70Tl3xjn3LfA2/z8uccAg59wu/zpfAGLNLD++cSgDXO2cS3POJTvnjlxCHgly2rkrwWZfhtsngYMZ5qzPbX0Wxbe1Ggns8fU44NuQ2ZnJ+04B/mNmFYFrAIeveAH+gm+r/wsz+xkY6pwbfZGMezPcPuHPk1Xnf37n37+UHbUZP+dooKKZHcqwLIL//1yjgY/MLD3D42lABXy/aVUGJphZSWA8vh8SqZeQSYKYil/yqp34jhAq6//t4KKcc4fMbB7QBagJfOD8RzY45/YC/wNgZjcCn5rZ5865r7OZ6Ti+I5fOuSybr79UGY/Q2Alsd85dk8lzdwJ9nXPLMnl8MDDYzKoCs4GtwKicCirBQVM9kic55/bgm5cfambF/TstrzKzFhd52ftAT3zz7OemeTCzzmZ27iiZn/EV6aUcGbMWuMPMSpvZZcCjl/Aev9cXwBEze8rMCptZhJnVMbMm/sffAoaYWTSAmZUzsw7+2y3NrK5/B/kRfFM/OkIoBKn4JS/rCRQANuMr7CnA5Rd5/nR80zz7/PPz5zQBVvn/5mA68Ihzbvsl5HkP35z/d/h+KF3w6KBA8k+LtQMaANuBg8A7+HbaAgzD9znOM7OjwErgOv9jl+EbwyNACrAY33SPhBgdxy8iEma0xS8iEmZU/CIiYUbFLyISZlT8IiJhJk8cx1+2bFlXtWpVr2OIiOQpycnJB51z5c5fnieKv2rVqiQlJXkdQ0QkTzGz7y+0XFM9IiJhRsUvIhJmVPwiImFGxS8iEmZU/CIiYUbFLyIShBI3JFL19arkG5yPqq9XJXFDYo69d544nFNEJJwkbkik/4z+nEg9AcD3h7+n/4z+AHSv2/13v7+2+EVEgsygBYP+W/rnnEg9waAFg3Lk/VX8IiJBZsfhHdlanl0qfhGRIFO6cOkLLq9SokqOvL+KX0QkiGzYt4Ejp46Qz35Zz1GRUQxpNSRH1qHiFxEJEkdOH6HTpE6ULVKW/9z+H6JLRGMY0SWiSWiXkCM7dkFH9YiIBAXnHP2m9+Pbn79lYa+FNI9uzh+b/DEg61Lxi4gEgTe/eJPJmyfzcquXaR7dPKDr0lSPiIjHVu1axePzHqfdte14stmTAV+fil9ExEM/nviRLlO6UKl4JcZ1HPernbqBoKkeERGPpLt07v/ofvYe28uyvssoVbhUrqxXxS8i4pGXl77MnK/nMPyO4cRUjMm19WqqR0TEA4u2L+Jvi/7GfXXu48GYB3N13Sp+EZFctufoHu6beh/XlrmWhHYJmFmurl9TPSIiuehs+lm6Tu3K0TNHWdBzAUULFM31DCp+EZFc9LeFf+Pz7z/n3Y7vUrt8bU8yaKpHRCSXzPxqJi8ve5n+jfpzf/37Pcuh4hcRyQXfHfqOnh/1pOFlDRl2+zBPs6j4RUQC7PTZ03Se3Jl0l87kzpMplL+Qp3k0xy8iEmCPz3ucpN1JfHTvR1xV+iqv42iLX0QkkCZsnMDwL4fzeNPH6Vijo9dxABW/iEjAbDm4hX7T+9GscjP+2eqfXsf5LxW/iEgAHD9znNhJsURFRjExdiKREZFeR/ovzfGLiOQw5xwPznqQzQc2M+/+eVQqXsnrSL+gLX4RkRz2zup3eG/9ezzf4nlaV2vtdZxfUfGLiOSgNXvW8PCch7m12q082/xZr+NckIpfRCSHHDp1iM6TO1M2qiyJ9yQSkS/C60gXpDl+EZEc4Jyjz7Q+fH/4exb3Xky5IuW8jpQpFb+ISA54beVrfLzlY4a2GcoNlW/wOs5FaapHROR3WrZjGU99+hR317ibx65/zOs4v0nFLyLyOxw4foB7p9xLdIloRncYnesXVbkUAS1+MytpZlPMbIuZpZhZU//yh81sq5ltMrNXAplBRCRQ0tLT6P5hdw6eOMiULlMoWaik15GyJNBz/MOAT5xzsWZWAIgys5ZAB6Cec+60mZUPcAYRkYD4x+f/YP6380m4K4EGlzXwOk6WBaz4zaw40BzoDeCcOwOcMbMHgZedc6f9y/cHKoOISKDM/2Y+gxcPpmf9nvRr1M/rONkSyKmeasABYIyZrTGzd8ysCHAtcJOZrTKzxWbWJIAZRERy3K4ju+j2YTdqlavFiDtG5Il5/YwCWfz5gUbASOdcQ+A48LR/eSngeuBJYJJdYNTMrL+ZJZlZ0oEDBwIYU0Qk61LTUrl3yr2cOnuKKV2mUKRAEa8jZVsgi38XsMs5t8p/fwq+HwS7gA+dzxdAOlD2/Bc75xKcczHOuZhy5YL3DyFEJLw8s+AZlu9cztvt3qZG2Rpex7kkASt+59xeYKeZVfcvagVsBj4GbgEws2uBAsDBQOUQEckpH6V8xNAVQxnYZCBd63T1Os4lC/RRPQ8Dif4jer4F+uCb8hltZhuBM0Av55wLcA4Rkd/lm5++ofe03jSp2IShbYZ6Hed3CWjxO+fWAjEXeKhHINcrIpKTTp09RefJnYmwCCZ1nkTB/AW9jvS76Fw9IiK/4ZE5j7Bm7xpm3DeDqiWreh3nd9MpG0RELuK9de+RsDqBp5s9zV3X3uV1nByh4hcRycSm/ZsYMGsALaJb8Pdb/u51nByj4hcRuYCjp4/SaVInihUoxgedPiB/vtCZGQ+dz0REJIc45+g/sz/bftrGp/d/yuXFLvc6Uo5S8YuInGdk0kgmbJzAkFuG0PLKll7HyXGa6hERySBpdxKPzX2MO665g6dvfNrrOAGh4hcR8fvp5E/ETorlsqKX8W7Hd8lnoVmRmuoREQHSXTq9Pu7F7qO7WdJnCWWiyngdKWBU/CIiwP8u+19mfjWTN9q+wXVXXOd1nIAKzd9jRESyYfF3ixm0cBCda3XmoT885HWcgFPxi0hY23tsL12nduWq0lfxTvt38txFVS6FpnpEJGylpafRbWo3Dp86zLwe8yhesLjXkXKFil9Ewtbznz3Pou8WMabDGOpWqOt1nFyjqR4RCUtzts1hyJIh9G3Ql94NensdJ1ep+EUk7Ow4vIMeH/WgXoV6vHnHm17HyXUqfhEJK2fSztBlchdS01KZ0nkKhSMLex0p12mOX0TCypPznmTVD6uY0nkK15S5xus4ntAWv4iEjcmbJvPGF2/w6HWP0qlWJ6/jeEbFLyJh4asfv+KB6Q9w/RXX869b/+V1HE+p+EUk5J1IPUHspFgKRBRgUuwkCkQU8DqSpzTHLyIh76HZD7Fx/0Zmd59N5RKVvY7jOW3xi0hIG71mNGPWjuHZ5s/S9uq2XscJCip+EQlZ6/auY+DsgbS6shXPt3je6zhBQ8UvIiHp8KnDxE6OpVShUiTek0hEvgivIwUNzfGLSMhxzvHA9AfY/vN2FvVaRIWiFbyOFFRU/CISct5Y9QZTU6bySutXuCn6Jq/jBB1N9YhISFm5ayVPzH+C9tXb88QNT3gdJyip+EUkZBw8cZAuk7twRfErGNthbFhcVOVSaKpHREJCukunx4c92Hd8H8v7LqdU4VJeRwpaKn4RCQkvLXmJud/MZeSdI2lcsbHXcYKapnpEJM9b8O0Cnv/sebrV7UZc4ziv4wQ9Fb+I5Gm7j+6m24fdqF6mOvF3xWtePws01SMiedbZ9LN0ndKVY2eOsajXIooWKOp1pDxBxS8iedagBYNYsmMJ4+8eT61ytbyOk2doqkdE8qTpW6fzyvJXiGscR/d63b2Ok6eo+EUkz9n+83Z6fdyLRpc34vW2r3sdJ88JaPGbWUkzm2JmW8wsxcyaZnjsCTNzZlY2kBlEJLScOnuKzpM745xjcufJFMpfyOtIeU6g5/iHAZ8452LNrAAQBWBmlYFbgR0BXr+IhJg/z/0zyXuS+fjej6lWqprXcfKkgG3xm1lxoDkwCsA5d8Y5d8j/8GvAXwAXqPWLSOh5f8P7jEwayRNNn6BDjQ5ex8mzAjnVUw04AIwxszVm9o6ZFTGz9sAPzrl1AVy3iISYlAMp9J/Rnxur3MhLrV7yOk6eFsjizw80AkY65xoCx4EXgEHAc7/1YjPrb2ZJZpZ04MCBAMYUkWB37MwxOk3qRFRkFBM6TSAyItLrSHlaIIt/F7DLObfKf38Kvh8EVwLrzOw74ApgtZlddv6LnXMJzrkY51xMuXLlAhhTRIKZc44BMwew5eAW3u/0PpWKV/I6Up4XsOJ3zu0FdppZdf+iVsBq51x551xV51xVfD8cGvmfKyLyKwnJCSRuSGTwzYNpXa2113FCQqCP6nkYSPQf0fMt0CfA6xORELJ6z2r+9MmfuO2q2xjUfJDXcUJGQIvfObcWiLnI41UDuX4Rybt+PvkzsZNiKV+kPOPvGU8+09+b5hSdq0dEgo5zjj7T+rDzyE4+7/05ZaP0d545ScUvIkFn6IqhTNs6jddue42mlZv+9gskW/S7k4gElaU7lvL0p0/TqWYnHrnuEa/jhCQVv4gEjf3H93PvlHu5stSVjGo/ShdVCRBN9YhIUEhLT6Pb1G78eOJHZvWbRYlCJbyOFLJ+c4vfzCqY2Sgzm+O/X8vMHgh8NBEJJy8ufpEF2xcw/I7hNLisgddxQlpWpnrGAnOBiv77XwGPBiiPiIShuV/P5e+f/51e9XvRt2Ffr+OEvKwUf1nn3CQgHcA5dxZIC2gqEQkbOw/vpPuH3aldvjYj7hyhef1ckJXiP25mZfCfQtnMrgcOBzSViISF1LRU7p1yL6fTTjOl8xSiIqO8jhQWsrJz98/AdOAqM1sGlANiA5pKRMLCU58+xYpdK5gYO5HqZav/9gskR/xm8TvnVptZC6A6YMBW51xqwJOJSEibunkqr618jYeaPESX2l28jhNWfrP4zSwCuAOo6n9+GzPDOfdqgLOJSIj6+qev6Tu9L00qNuHfbf7tdZywk5WpnhnAKWAD/h28IiKX6mTqSWInxRJhEUzuPJmC+Qt6HSnsZKX4r3DO1Qt4EhEJWYkbEhm0YBA7Du+gSGQRjqUeY+Z9M4kuGe11tLCUlaN65phZm4AnEZGQlLghkf4z+vP94e9xOI6lHiN/vvwcOn3I62hhKyvFvxL4yMxOmtkRMztqZkcCHUxEQsOgBYM4kXriF8vOpp9l0AJdWMUrWZnqGQo0BTY451yA84hIiHDOsXTHUr4//P0FH99xeEcuJ5JzslL824CNKn0RyYqfTv7Ee+veIz45npSDKRiG49f1UaVEFQ/SCWSt+PcAn/lP0nb63EIdziki5zjnWL5zOfHJ8UzePJlTZ09xXaXrGNV+FAAPz3n4F9M9UZFRDGk1xKu4YS8rxb/d/1HA/yEiAviuizt+/Xjik+PZdGATxQoUo0+DPvRv3P8XZ9gsmL/gf4/qqVKiCkNaDaF73e7eBQ9zlhdmcGJiYlxSUpLXMUQE39b9yl0riU+OZ9KmSZw8e5KYijHENY6ja52uFC1Q1OuI4mdmyc65mPOXZ7rFb2ZvOuceMrMZ8OsJOudc+xzOKCJB7PCpw//dut+wfwNFCxTl/nr3ExcTR6PLG3kdT7LhYlM9PYGHAP09tUiYcs7x5e4viU+KZ8KmCZxIPUGjyxsRf1c899W5j2IFi3kdUS7BxYr/GwDn3OJcyiIiQeLI6SMkrk8kYXUCa/eupUhkEbrV6UZcTBwxFX81cyB5zMWKv5yZ/TmzB3VUj0joSdqdRHxSPB9s/IDjqcdpcFkDRt45km51u1G8YHGv40kOuVjxRwBF8Z2KWURC1NHTR/lg4wfEJ8ezes9qoiKj6Fq7K3ExcTSp2ERXxApBFyv+Pc65F3MtiYjkqtV7VhOfFM/7G9/n2Jlj1C1flzdvf5Me9XpQolAJr+NJAF2s+PVjXiTEHDtzjAkbJxCfHE/S7iQK5y/MvXXupX+j/lx/xfXaug8TFyv+VrmWQkQCau3etSQkJzB+/XiOnjlK7XK1eaPtG/So14NShUt5HU9yWabF75z7KTeDiEjOOn7mOBM3TSQ+OZ4vfviCghEF6VK7C3GN47ih8g3aug9jWTllg4jkIRv2bSA+OZ731r/HkdNHqFm2Jq/f9jr317+f0oVLex1PgoCKXyQEnEg9weRNk4lPjmfFrhUUjChIbK1Y4hrHcWOVG7V1L7+g4hfJwzbt3/TfrftDpw5RvUx1hrYZSq/6vSgTVcbreBKkVPwieczJ1JNM2TyF+OR4lu1cRoGIAnSq2Ym4xnE0j26urXv5TSp+kTwi5UAK8cnxvLvuXX4+9TPXlL6G/731f+lVvxflipTzOp7kISp+kSB26uwppm6eSnxyPEt2LCEyXyR317ybuMZxtKzaUlv3cklU/CJBaOvBrSQkJzBu3Th+PPkjV5W6in+1/he9G/SmfJHyXseTPC6gxW9mJYF3gDr4zunfF7gHaAecwXcG0D7OuUOBzCGSF5w+e5oPUz4kYXUCn333Gfnz5adjjY7ENY7jlitvIZ/l8zqihIhAb/EPAz5xzsWaWQEgCpgPPOOcO2tm/wKeAZ4KcA6RoLXtx20kJCcwdt1YDp44yJUlr+SlW16iT8M+XFb0Mq/jSQgKWPGbWXGgOdAbwDl3Bt9W/rwMT1sJxAYqg0iwOpN2ho+3fEx8cjwLty8kwiLoUKMDcY3jaF2ttbbuJaACucVfDTgAjDGz+kAy8Ihz7niG5/QFJgYwg0hQ+eanb3h79duMWTuG/cf3E10imn+0/Ad9G/bl8mKXex1PwkQgiz8/0Ah42Dm3ysyGAU8DfwMws0HAWSDxQi82s/5Af4AqVaoEMKZIYKWmpTJt6zTik+P59NtPibAI7rr2LuIax9HmqjZE5IvwOqKEmUAW/y5gl3Nulf/+FHzFj5n1Au4CWjnnfnUhdwDnXAKQABATE3PB54gEs+0/b+ft1W8zes1o9h3fR+XilXnx5hfp27AvlYpX8jqehLGAFb9zbq+Z7TSz6s65rfhO87zZzNri25nbwjl3IlDrF/FCaloqM76aQXxyPPO/mY+Zcec1dxLXOI62V7fV1r0EhUAf1fMwkOg/oudboA/wJVAQmO//45OVzrkBAc4hkqMSNyQyaMEgdhzeQZUSVXjs+sc4cOIAo9eMZs+xPVxR/Aqeb/E8DzR6gCuKX+F1XJFfsExmWoJKTEyMS0pK8jqGCOAr/f4z+nMi9de/sJ7bur/9mtvJn09/HyneMrNk51zM+cv1lSmSTYMWDLpg6VcqVomZ3WZ6kEgke3SwsEg27Ti844LLdx/dnctJRC6Nil8kG779+dtMd9BWKaHDjiVvUPGLZFHy7mSajmpKgXwFKBhR8BePRUVGMaTVEI+SiWSPil8kC+Zsm0OLsS0onL8wSf2TGNVhFNElojGM6BLRJLRLoHvd7l7HFMkS7dwV+Q2jVo8ibmYc9SrUY1a3WVxe7HJqlqupopc8S1v8IplwzvHCZy/Qb0Y/WlVrxeLei3U+HQkJ2uIXuYDUtFQenPUgo9aMolf9Xrzd7m0iIyK9jiWSI1T8Iuc5duYYXSZ3Yc7Xc3j2pmd5seWLusShhBQVv0gG+47t487372TN3jW8dedbxMXEeR1JJMep+EX8th7cyu2Jt7Pv+D6mdZ3GXdfe5XUkkYBQ8YsAK3auoN0H7chn+VjUaxF/qPQHryOJBIyO6pGw9/GWj7nl3VsoVbgUyx9YrtKXkKfil7A24ssRdJrUiXoV6rG873KuLn2115FEAk7FL2Ep3aXz9KdPM3D2QO645g4W9lxIuSLlvI4lkis0xy9h50zaGfpO60vihkTiGsfx5h1v6tz5Elb01S5h5fCpw9wz6R4Wbl/IkFuG8MyNz+gYfQk7ITvVk7ghkaqvVyXf4HxUfb0qiRsSvY4kHvvhyA80H9ucz7//nHEdx/HXm/6q0pewFJJb/OdfGu/7w9/Tf0Z/AJ1YK0xt2r+JtoltOXTqELO6zaLNVW28jiTimZDc4r/QpfFOpJ5g0IJBHiUSL3323Wc0G92MtPQ0lvRZotKXsBeSxZ/ZpfEyWy6ha8LGCdw2/jYqFqvIigdW0OCyBl5HEvFcSBZ/ZpfAK124NM65XE4jXnDOMXT5UO6beh/XVbqOpX2XEl0y2utYIkEhJIt/SKshREVG/WJZPsvHjyd/5OZxN5NyIMWbYJIr0tLTeGzuYzwx/wlia8Uy7/55lC5c2utYIkEjJIu/e93uJLRL+MWl8cZ1HMc77d5hw74N1H+rPs8teo5TZ095HVVy2Kmzp7h3yr0MWzWMR657hImxEymUv5DXsUSCiuWFqY+YmBiXlJSUI++1//h+Hp/3OOPXj+fq0lcz8s6RtK7WOkfeW7z108mf6DChA0t3LOXVNq/yWNPHvI4k4ikzS3bOxZy/PCS3+C+mfJHyvHf3e8y/fz4At753K/d/dD/7j+/3OJn8Ht8d+o5mo5vxxQ9fMDF2okpf5CLCrvjPaV2tNRse3MDfmv+NiRsnUuPNGoxaPYp0l+51NMmmNXvW0HRUU/Ye28u8HvPoUruL15FEglrYFj9AofyFeLHli6wbsI66FerSb0Y/WoxtweYDm72OJlk075t5NB/bnMh8kSzts5QWVVt4HUkk6IV18Z9Ts1xNPuv1GaPbj2bzgc00eKsBzy58lpOpJ72OJhcxbu047nz/TqqVqsaKB1ZQu3xtryOJ5Akqfj8zo0/DPmwZuIX76t7HkCVDqDuyLvO/me91NDmPc44hnw+h97TetIhuwee9P6dS8UpexxLJM1T85ylXpBzjOo5jQc8F5LN8tBnfhu4fdmffsX1eRxPgbPpZBswcwLOLnqVHvR7M7j6bEoVKeB1LJE9R8WfilitvYf2D63m+xfNM2TyFGsNr8Hby29r566HjZ45z98S7SVidwNPNnubdju9SIKKA17FE8hwV/0UUyl+IF25+gXUD1lG/Qn36z+xP8zHN2bR/k9fRws7+4/u55d1bmL1tNsPvGM4/W/9Tp1QWuUQq/iyoUbYGi3otYkyHMWw5uIUG8Q3464K/audvLvn6p6+5YdQNrN+3ng+7fMgfm/zR60gieZqKP4vMjN4NerPloS30qNeDfy79J3VG1mHu13O9jhbSVu1aRdNRTTl06hALey6kQ40OXkcSyfNU/NlUNqosYzqMYVGvRUTmi6RtYlu6Te3G3mN7vY4WcmZsnUHLcS0pVqAYyx9YTtPKTb2OJBISVPyX6OaqN7NuwDoG3zyYqSlTqfFmDeKT4rXzN4fEJ8XTcWJHapevzYoHVnBtmWu9jiQSMgJa/GZW0symmNkWM0sxs6ZmVtrM5pvZNv+/pQKZIZAK5i/Icy2eY/2A9TS6vBEDZg3gxtE3smHfBq+j5VnOOZ5d+CwDZg2g7dVtWdRrERWKVvA6lkhICfQW/zDgE+dcDaA+kAI8DSxwzl0DLPDfz9Oql63Ogp4LGNdxHF/9+BWNEhrxzKfP/Oryj3JxqWmp9JnWhyFLhtCvYT+mdZ1G0QJFvY4lEnICVvxmVhxoDowCcM6dcc4dAjoA4/xPGwd0DFSG3GRm9Kzfk60PbaVnvZ68vOxl6oyowydff+J1tDzhyOkj3Pn+nYxbN47BNw8moV0C+fPl9zqWSEgK5BZ/NeAAMMbM1pjZO2ZWBKjgnNsD4P+3/IVebGb9zSzJzJIOHDgQwJg5q0xUGUZ1GMVnvT6jYP6C3J54O12ndGXP0T1eRwtau4/upsXYFizcvpDR7UfzXIvndIy+SAAFsvjzA42Akc65hsBxsjGt45xLcM7FOOdiypUrF6iMAdOiagvWxq3l7y3/zsdbPqbm8Jq8lfSWdv6eJ+VACk1HNWXbj9uY2W0mfRr28TqSSMgLZPHvAnY551b570/B94Ngn5ldDuD/N2SvgFIwf0Gebf4sGx7cQEzFGB6c9SDNRjdj/b71XkcLCkt3LKXZ6GacPnuaxb0X0/bqtl5HEgkLASt+59xeYKeZVfcvagVsBqYDvfzLegHTApUhWFxT5hrm3z+f9+5+j69/+ppG8Y14av5THD9z3Otonpm6eSqt321N+SLlWfHAChpXbOx1JJGwEeijeh4GEs1sPdAAeAl4GbjVzLYBt/rvhzwzo0e9Hmx9aCt9GvThleWvUGdkHWZvm+11tFw3bOUwOk/uTOOKjVnWdxlXlrrS60giYSXsLrYeLJZ8v4S4mXGkHEyhc63OvN72dSoWq+h1rIBKd+k8Oe9JXl35KnfXuJvEexIpHFnY61giIUsXWw8yN0XfxNoBa/lHy38wfet0ag6vyYgvR5CWnuZ1tIA4ffY03aZ249WVr/JQk4eY3HmySl/EIyp+DxWIKMCg5oPY+MeNXFfpOgbOHkiz0c1Yt3ed19Fy1M8nf+a28bcxcdNE/tX6X7xx+xtE5IvwOpZI2FLxB4GrS1/N3B5zSbwnke2HttM4oTFPznsyJHb+7jy8k5vG3MTynctJvCeRvzT7i47RF/GYij9ImBnd6nYjZWAKfRv25d8r/k3tEbWZ+dVMr6NdsvX71nP9qOvZeWQnc3vMpVvdbl5HEhFU/EGndOHSJLRLYEmfJRQtUJR2H7QjdlIsPxz5weto2bLg2wXcOPpGDGNpn6W0vLKl15FExE/FH6RurHIjq+NW89ItLzFr2yxqDq/Jm1+8mSd2/iauT+T2xNupUqIKKx5YQd0Kdb2OJCIZqPiDWIGIAjxz0zNsfHAjTSs35eE5D9N0VFPW7l3rdbQLcs7x8tKX6fFRD5pVacbSvkupXKKy17FE5Dwq/jzgqtJX8Un3T/ig0wfsOLyDmIQYnpj3BMfOHPM62n+lpafx0OyHeGbBM3St05VPun9CyUIlvY4lIheg4s8jzIyudbqSMjCFfo36MXTFUGoNr8WMrTO8jsaJ1BN0mtSJEUkjePKGJ0m8J5GC+Qt6HUtEMqHiz2NKFS7FW3e9xbK+yyhRqATtJ7Sn06ROnu38PXjiIK3ebcX0rdN5o+0bvHLrK+QzfVmJBDN9h+ZRN1S+gdX9V/Nyq5eZs20ONYfX5I1Vb+Tqzt9vf/6WG0bdwJo9a5jceTIPX/dwrq1bRC6dij8Pi4yI5Kkbn2LjHzfSrEozHvnkEa4fdT2r96wO+LqTdifRdFRTfjz5Iwt6LqBTrU4BX6eI5AwVfwioVqoas7vNZmLsRHYd2UWTt5vw57l/DtjO3znb5nDz2JuJioxiWd9lNKvSLCDrEZHAUPGHCDOjS+0upAxMIa5xHK+vfJ1aw2sxbUvOXu5g1OpRtPugHdeWuZYVD6ygRtkaOfr+IhJ4Kv4QU7JQSUbcOYJlfZdRslBJOk7syN0T72bn4Z2/632dc7zw2Qv0m9GP1tVas7j3Yi4relkOpRaR3KTiD1FNKzcluX8yr7R+hblfz6XWiFoMWznsknb+pqal0m96PwYvHkzvBr2Zcd8MihUsFoDUIpIbVPwhLDIikiebPcnmgZtpHt2cR+c+ynXvXEfy7uQsv8exM8doP6E9o9eO5rnmzzG6/WgiIyIDmFpEAk3FHwaqlqzKzPtmMrnzZHYf3c0f3vkDj37yKEdPH73o6/Yd28fNY29m/jfzSbgrgcEtB+uUyiIhQMUfJsyM2FqxpAxM4cGYB3lj1RvUGlGLj7d8fMHnbz24laajmpJyMIVpXafxP43/J3cDi0jA6Jq7YWrVrlX0n9mf9fvW0756e1pd2YpXV7zKjsM7KF+kPMfOHCMqMopZ3WbRpFITr+OKyCXI7Jq7Kv4wlpqWyrBVw/jrgr+Smp76i8cMY2iboTzW9DGP0onI76WLrcuvREZE8sQNT1C+SPlfPeZwDFs1zINUIhJoKn5h99HdF1y+4/COXE4iIrlBxS9UKVElW8tFJG9T8QtDWg0hKjLqF8uiIqMY0mqIR4lEJJBU/EL3ut1JaJdAdIloDCO6RDQJ7RLoXre719FEJAB0VI+ISIjSUT0iIgKo+EVEwo6KX0QkzKj4RUTCjIpfRCTM5ImjeszsAPD9Jb68LHAwB+PkFOXKHuXKHuXKnmDNBb8vW7Rzrtz5C/NE8f8eZpZ0ocOZvKZc2aNc2aNc2ROsuSAw2TTVIyISZlT8IiJhJhyKP8HrAJlQruxRruxRruwJ1lwQgGwhP8cvIiK/FA5b/CIikoGKX0QkzIRU8ZvZaDPbb2YbMywrbWbzzWyb/99SQZLrBTP7wczW+j/u8CBXZTNbZGYpZrbJzB7xL/d0zC6Sy9MxM7NCZvaFma3z5xrsX+71eGWWy/OvMX+OCDNbY2Yz/fc9/57MJJfn42Vm35nZBv/6k/zLcny8Qqr4gbFA2/OWPQ0scM5dAyzw389tY/l1LoDXnHMN/B+zczkTwFngcedcTeB6YKCZ1cL7McssF3g7ZqeBW5xz9YEGQFszux7vxyuzXOD91xjAI0BKhvtej9c55+eC4Bivlv71nzt2P8fHK6SK3zn3OfDTeYs7AOP8t8cBHXMzE2Say3POuT3OudX+20fxfRNUwuMxu0guTzmfY/67kf4Ph/fjlVkuz5nZFcCdwDsZFnv+PZlJrmCV4+MVUsWfiQrOuT3gKxSgvMd5MnrIzNb7p4I8+XX3HDOrCjQEVhFEY3ZeLvB4zPzTA2uB/cB851xQjFcmucD7r7HXgb8A6RmWeT5emeQC78fLAfPMLNnM+vuX5fh4hUPxB6uRwFX4fjXfAwz1KoiZFQWmAo865454leN8F8jl+Zg559Kccw2AK4A/mFmd3M5wIZnk8nS8zOwuYL9zLjk31/tbLpLL868voJlzrhFwO74pzuaBWEk4FP8+M7scwP/vfo/zAOCc2+f/Zk0H3gb+4EUOM4vEV66JzrkP/Ys9H7ML5QqWMfNnOQR8hm/fjefjdaFcQTBezYD2ZvYdMAG4xczG4/14XTBXEIwXzrnd/n/3Ax/5M+T4eIVD8U8Hevlv9wKmeZjlv879R/rdDWzM7LkBzGDAKCDFOfdqhoc8HbPMcnk9ZmZWzsxK+m8XBloDW/B+vC6Yy+vxcs4945y7wjlXFegKLHTO9cDj8cosl9fjZWZFzKzYudtAG3+GnB8v51zIfAAf4PsVLRXYBTwAlMG3J3yb/9/SQZLrPWADsN7/H3u5B7luxDenuB5Y6/+4w+sxu0guT8cMqAes8a9/I/Ccf7nX45VZLs+/xjJkvBmYGQzjdZFcXn99VQPW+T82AYMCNV46ZYOISJgJh6keERHJQMUvIhJmVPwiImFGxS8iEmZU/CIiYUbFL/I7mNmgDGdzTMtw+09eZxPJjA7nFMkhZnbMOVf0vGWG7/vs/HPCiHhGW/wSssysqvnO6f+2/zz18/x/2YqZfWZmMf7bZf1/vo+Z9Tazj81shpltN7OHzOzP/vO2rzSz0tlY7whgNVDZzJ40sy/9JwAbnOG5Pcx3Lv21ZhbvP9lahJmNNbON/nOzPxaQAZKwpeKXUHcNMNw5Vxs4BHTKwmvqAN3wnSdlCHDCOdcQWAH0zOJ6qwPv+l9X3Z/jD/hOANbYzJqbWU3gXnwn5moApAHd/c+p5Jyr45yrC4zJ4jpFsiS/1wFEAmy7c26t/3YyUDULr1nkfNcBOGpmh4EZ/uUb8J0eISu+d86t9N9u4/9Y479fFN8PgnpAY+BL34wQhfGdgGsGUM3M/gPMAuZlcZ0iWaLil1B3OsPtNHzlCr6rfJ37jbfQRV6TnuF+Oln/njme4bYB/3TOxWd8gpk9DIxzzj1z/ovNrD5wGzAQ6AL0zeJ6RX6TpnokXH2Hb2sbIDbA65oL9PVfXwAzq2Rm5fGdcCvWf/vctVWjzawskM85NxX4G9AowPkkzGiLX8LVv4FJZnY/sDCQK3LOzfPP56/wT+kcA3o45zab2bP4rriUD9/ZWwcCJ4Ex/mUAv/qNQOT30OGcIiJhRlM9IiJhRsUvIhJmVPwiImFGxS8iEmZU/CIiYUbFLyISZlT8IiJh5v8ArEU0Qn7rbJgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plotting the duration of training for each num_trees in experiments\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "plt.plot(TREES, TIME,\"go-\")\n",
    "plt.title(\"Time vs num Trees\")\n",
    "plt.xlabel(\"num Trees\")\n",
    "plt.ylabel(\"Time\")\n",
    "plt.savefig(\"Time vs num_Trees.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "7ccbce64",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA0cUlEQVR4nO3deXhUd9n/8fedPWFLgBAgCdkTSCCkEAIuj/66aFu1rbZarV3tgmg3a+1ia2ur8lhtba22taW1tVXsYhdt1doFrfXREgiQAAESQhJCCJCwhgAJSeb+/TGHOk0TmIFMziS5X9c1V+Zscz5zkpk753vO+R5RVYwxxhh/hbkdwBhjzOBihcMYY0xArHAYY4wJiBUOY4wxAbHCYYwxJiBWOIwxxgTECocxxpiAWOEwrhCRd0Rkj4hEu53FfJCITBGRNp+HisgBn+H/cTujcZfYBYBmoIlIOrAJ2Ad8XVX/MIDrjlDVroFa31AgIgrkqGpNL9Nsew5Dtsdh3HAJsBT4DXCp7wQRSRWRl0WkRUR2ichDPtOuEpH1IrJfRNaJyCxnvIpIts98vxGRHznP/5+INIrILSKyHXhKRBJE5M/OOvY4z1N8lh8rIk+JSJMz/Y/O+LUicpbPfJEislNEinq+QSfn53yGI5x5Z4lIjIj8znl/e0VkuYgk9bahRKReRL4jIqtFZJ+IPC8iMc60y0Tk/3rM//62cLbDIyLyurOn8G8RmSgiP3fe1wYROekYv6ueeS5zXucBEdkN3CUi0SJyn4g0iMgOEXlURGJ9lvmciJQ77/U/IlLoM+0WEdnq/E6rROTUQPIYd1jhMG64BFjsPE4/8qUpIuHAn4HNQDqQDDznTPsScJez7GjgbGCXn+ubCIwF0oD5eP/un3KGpwCHgId85v8tEAcUABOAB5zxzwAX+cz3GWCbqpb3ss5ngQt8hk8HdqrqSrzFcgyQCowDFjgZ+nI+cAaQARQClx1l3t6W/R4wHugA3gNWOsMvAvcH8FpHzAVq8W6bhcBPgFygCMjG+3u7E8Ap7k8CX8f7Xh8DXnWKTR5wDTBHVUfh3Ub1x5HHDDRVtYc9BuwBfBzoBMY7wxuAG5znHwFagIhelnsDuL6P11Qg22f4N8CPnOf/DzgMxBwlUxGwx3k+CfAACb3MNxnYD4x2hl8Ebu7jNbOdeeOc4cXAnc7zy4H/AIV+bK964CKf4Z8CjzrPLwP+r69t4WyHx32mXQus9xmeAez1I4Pva14GNPhME+AAkOUz7iNAnfP8V8APe7xeFfBJZxs1A6cBkW7/bdrD/4ftcZiBdinwpqrudIZ/z3+bq1KBzdp7m3kq3uMix6NFVduPDIhInIg8JiKbRaQVeBeId/Z4UoHdqrqn54uoahPwb+A8EYkHzsRbED5EvccD1gNniUgc3j2k3zuTf4u3ED7nNIf9VEQij5J/u8/zg8BIv9611w6f54d6GQ7ktY7Y4vM8Ee/e2QqnKWov8DdnPHj36m48Ms2ZngpMdrbRt/DuSTaLyHMiMvk48pgBFuF2ADN8OO3e5wPhzvEGgGi8X9oz8X4hTenjgOsWIKuPlz6I98vriIlAo89wzzNAbgTygLmqut05RrEK73/PW4CxIhKvqnt7WdfTwJV4PzvvqerWvt4v/22uCgPWOV+UqGoncDdwt3OiwF/x/hf+66O8Vm8O4PO+RWRigMsfL9/tuRNvASroY1tsARaq6sJeX0j198DvRWQ03masnwAX93Ne089sj8MMpM8D3UA+3uahImAa8C+8xy6WAduAe0RkhHMQ+WPOsk8A3xGR2eKVLSJpzrRy4KsiEi4iZ+BtBjmaUXi/7PaKyFjg+0cmqOo24HXgEecgeqSIfMJn2T8Cs4Dr8R7zOJrngE8D3+C/exuIyMkiMsPZw2nF23TXfYzX6k0FUCAiRc4B87uO4zVOiKp6gMeBB0RkAoCIJIvI6c4sjwMLRGSu83sbISKfFZFRIpInIqeI95Tsdry/k+PZDmaAWeEwA+lS4ClVbVDV7UceeA9MX4j3P/6z8LZ9N+Dda/gygHpP2V2I9wt4P94v8LHO617vLLfXeZ0/HiPHz4FYvP8tL8XbtOLrYrxf5hvwtsF/68gEVT0EvIT3QPXLR1uJU4TeAz4KPO8zaSLe4yOteJuz/gn87hiZe3v9auAHwNvARuD/jr5E0NwC1ABLnaa/t/Hu0aGqZcBVeH/He5z5LnOWiwbuwft72I73YPttAxncHB+7jsOYAInInUCuql50zJmNGYLsGIcxAXCatq7A2uHNMGZNVcb4SUSuwnuw93VVfdftPMa4xZqqjDHGBMT2OIwxxgRkWBzjGD9+vKanp7sdwxhjBpUVK1bsVNXEnuOHReFIT0+nrKzM7RjGGDOoiMjm3sZbU5UxxpiAWOEwxhgTECscxhhjAmKFwxhjTECscBhjjAmIFQ5jXLBj8Q7eS3+Pd8Le4b3099ixeMexFzImRAyL03GNCSU7Fu+gan4VnoMeADo2d1A1vwqApAt7vfW4MSHF9jiMGWC1t9e+XzSO8Bz0UHt7rUuJjAmMFQ5jBlhHQ0dA440JNVY4jBlg0cnRfU6ru6uOzr2dA5jGmMBZ4TBmAHXu7kTlwz1Sh8WEMXL2SDbfvZnSjFLqf1hPV2vP264bExqscBgzQLr2dbH6jNV07ugk9ZZUotOiQSA6LZq8J/IoXl7M7JWzGfOJMdTfWc/SjKVs/t/NdO23AmJCy7C4H0dxcbFaJ4fGTV1tXaw+fTX7l+2n4JUCxn9u/FHnby1rpf6uenb/ZTcR4yKYctMUJl89mYiRdiKkGTgiskJVi3uOtz0OY4Ks+2A3a89aS2tpK/nP5R+zaACMLh5N4Z8LmVU6i9FzRlN7ay2lmaU03NdA98HuAUhtTN+scBgTRN3t3az9wlr2/nMv056ZRuJ5H7q1wVGNLhlN4euFnPSfkxhZNJLam2pZmrmULQ9sofuQFRDjDiscxgSJ57CHdeevY8+be8j7dR5JXz3+i/vGfGQMM9+cSdG/ihhRMIJN395EaVYpjb9opLvdCogZWFY4jAkCT5eHdV9dx67XdpHzSA6TvjapX143/uPxFC0pouidImJzYqm5vobS7FK2PrIVT4fn2C9gTD+wwmFMP9NuZcOlG9j50k6y7s8i+RvJ/b6O+E/GU/ROETOXzCQ2I5aNV2+kNKeUpsea8By2AmKCywqHMf1IPUrV/Cqaf99Mxo8zSL0hNWjrEhESTkmg6N0iCt8qJDolmuoF1ZTmltL0eBOeTisgJjiscBjTT1SVjddsZPuT20m7M420W9MGZL0iwtjTxnLSv0+i8G+FRCVFUT2/mmV5y9j25DYrIKbfWeEwph+oKptu3ETTr5pIvTmV9LvSBzyDiDD29LHMWjqLGX+ZQcTYCKquqGLZtGVsf3o7ni4rIKZ/WOEw5gSpKnW319H4QCPJ1yWTeU8mIuJaHhFh3GfGMXv5bKa/Op2I0RFsuGwDy/OXs/1329HuoX/RrwkuKxzGnKDNP9pMw48bmDR/Etk/z3a1aPgSEcafNZ7ZK2ZT8EoBYbFhbLh4A8unL2fHszusgJjjZoXDmBPQcG8D9XfWk3RpErm/yg2ZouFLREj8fCLFq4opeLEAwmH9V9ezvHA5zS80ox4rICYwVjiMOU6Nv2yk9uZaJnxlAlN/PRUJC72i4UvChMTzEpmzeg75z+eDwrovr6NsZhktL7VYATF+s8JhzHFoWtREzXU1jP/CeKY+MxUJD+2i4UvChAnnT2DOmjlM+/00PJ0eKr9YSdmsMlr+2MJw6PjUnBgrHMYEaPsz26leUM3Yz4wl/7l8wiIH58dIwoWkC5IoqSxh6m+n4jngofILlawoXsHO13ZaATF9Gpx/8ca4pPn5ZjZ8bQMJpyZQ8FIBYVGD/yMk4cLEiyYyZ/0cpv5mKl17u1h79lpWlqxk1193WQExHzL4/+qNGSAtr7Sw7sJ1jPn4GKb/cTrhMeFuR+pXYRFhTLx0IiUbSsj7dR6dOztZ89k1rPzISna/sdsKiHmfFQ5j/LDrr7tY9+V1jJ4zmhl/nkH4iKFVNHyFRYYx6fJJlFSVkLsol8PbDrP6jNWs+vgqdr9tBcRY4TDmmHa/vZu1565lxIwRzHh9BhGjhsdd+MKiwph81WTmbpxLzq9y6GjoYPWnVlP+yXL2vLPH7XjGRVY4jDmKve/uZe3Za4nLjWPmmzOJjI90O9KAC4sKI3lBMnNr5pLzUA6HNh2i4uQKyk8uZ++7e92OZ1xghcOYPuxbuo81n11DTFoMM9+eSeS44Vc0fIVFh5F8dTJzN80l+8FsDm44SPknyyk/rZx9/97ndjwzgKxwGNOL/Sv3s/qM1UQmRTJzyUyiJkS5HSlkhMeEk3JdCnNr55J1fxYH1hxg1cdXUXF6BfuWWgEZDoJaOETkDBGpEpEaEbm1l+kJIvKKiKwWkWUiMt1n2g0iUikia0XkWRGJccb/0Jm/XETeFJHJwXwPZvhpW9NGxacqiIiPoOjvRURPjnY7UkgKjw0n9YZU5tXOI/PeTNpWtrHqI6tY/ZnVtC5vdTueCaKgFQ4RCQceBs4E8oELRCS/x2y3AeWqWghcAjzoLJsMXAcUq+p0IBz4irPMvapaqKpFwJ+BO4P1Hszwc2D9ASpOrSAsNoyivxcRMyXG7UghL3xEOFO+M4W5dXPJvCeT1mWtrCxZyZqz1rB/5X6345kgCOYeRwlQo6q1qnoYeA44p8c8+cASAFXdAKSLSJIzLQKIFZEIIA5ocubz/VdmBGDnBpp+cbDmIBWnVkAYFC0pIjYz1u1Ig0rEyAim3DKFeXXzyFiYwb5/72PF7BWs+fwa9pdbARlKglk4koEtPsONzjhfFcC5ACJSAqQBKaq6FbgPaAC2AftU9c0jC4nIQhHZAlxIH3scIjJfRMpEpKylpaWf3pIZqto3t1NxagXaqRQtKSIuL87tSINWxKgI0m5LY179PNJ/kM6+f+5jxUkrWHveWtrWtLkdz/SDYBaO3np967l3cA+QICLlwLXAKqBLRBLw7p1kAJOBESJy0fsvonq7qqYCi4Frelu5qi5S1WJVLU5MTDzhN2OGrvbGdspPKae7tZvCtwoZUTDC7UhDQsToCNLvSGdu3VzSvp/Gnrf3UFZYRuX5lRyoPOB2PHMCglk4GoFUn+EUnOamI1S1VVW/5hyvuARIBOqA04A6VW1R1U7gZeCjvazj98B5QchuhomO7R1UnFpBZ0snhW8UMqpolNuRhpzI+Egy7spgXv080r6Xxu6/7Wb5jOWsu2AdB9ZbARmMglk4lgM5IpIhIlF4D26/6juDiMQ70wCuBN51jmE0APNEJE68d8Y5FVjvLJPj8xJnAxuC+B7MEHZ452EqTqugY2sHha8XMrpktNuRhrTIhEgyfpjBvLp5TLl1Cjtf28nyguWsu2gdB6sPuh3PBCBohUNVu/A2I72B90v/BVWtFJEFIrLAmW0aUCkiG/CefXW9s2wp8CKwEljj5FzkLHOPc4ruauDTR5YxJhCdezpZ/anVtG9qZ8ZrMxjzsTFuRxo2IsdFkvm/mcyrm0fqTansfGUny6YtY/2l6zlYYwVkMJDh0GFZcXGxlpWVuR3DhIiu1i4qTqugraKNGa/OYOzpY92ONKwdbj5Mw08baHqkCc9hDxMvmUja99LsrLYQICIrVLW453i7ctwMK11tXaz+zGraVrVR8IcCKxohIGpCFNn3ZTO3di4p16bQ/Gwzy/KWUXVVFYfqD7kdz/TC9jjMsNF9qJs1n13D3n/uJf/5fCZ8cYLbkUwvOpo6aLingaZFTeCBiZdPJC4/jsb7G+lo6CB6SjSZCzNJujDp2C9mTkhfexzDo39oM+x5Ojys/cJa9r6zl2m/nWZFI4RFT44m5xc5TLllCpt/vJmmR5ug+7/TOzZ3UDW/CsCKh0usqcoMeZ5OD5XnV7LnjT3kPZ5nXzaDRHRyNLkP5RI18cMdTHoOeqi9vdaFVAascJghztPlYf2F69n16i5yHs5h0hWT3I5kAnS46XCv4zs2d9B9oLvXaSa4rHCYIUu7lQ2XbaDlDy1k/SyL5G/27PHGDAbRU/runXhp1lK2PrIVT6dnABMZKxxmSFKPUvX1KpoXN5OxMIPUb6ceeyETkjIXZhIW98GvqrC4MNK+n0ZcXhwbr97IsmnL2PHsDtQz9E/2CQVWOMyQo6psvG4j23+9nbQ70ki7Lc3tSOYEJF2YRN6iPKLTokEgOi2avEV5ZNyVQdE7Rcz46wzCR4az/qvrWVG8gt1v7GY4nC3qJjsd1wwpqsqm72yi8f5GUr+TSuZPM/H2WmOGMvUozc82U3dHHe117cSfHE/mjzMZPde6kTkRdgGgGRbq7qij8f5Gkq9JtqIxjEiYkHRhEiUbSsj+ZTYHKg+wct5K1p63lgMbrCPF/maFwwwZ9T+qp2FhA5OumkT2g9lWNIahsKgwUq5JYW7NXNLvTmfPW3tYXrCcqquqaG9sdzvekGGFwwwJDfc1UH9HPUkXJ5H7aC4SZkVjOIsYFUH6nenM3TSXlOtS2P7MdpblLGPTzZvo3N3pdrxBzwqHGfQaH2qk9qZaEs9PJO/JPCsa5n1RiVFkP5BNSVUJiecnsuW+LZRmlbL5ns10H7RrQI6XFQ4zqDU93kTNtTWMO2cc0343jbAI+5M2HxabHsu0p6dRXFHMmP8ZQ9136yjNLqXpsSa7BuQ42KfMDFrbf7ud6q9XM/bMsRQ8X0BYpP05m6MbOWMkM16dQdG/iojJjKF6QTXLC5bT/IdmO4U3APZJM4NS8wvNbLhsA/Enx1PwUgFh0fanbPwX//F4TvrXSUx/bTph0WGsO38dK+asYPfbu92ONijYp80MOjv/tJN1X13HmI+OYcarMwiPDXc7khmERITxnxtPcXkxU5+eSudO710hKz5VQWtZq9vxQpoVDjOo7Hp9F5VfqmRU8Shm/GUG4SOsaJgTI+HCxEsmMrdqLtk/z6atvI2Vc1ZSeX6l3Qu9D1Y4zKCxZ8keKs+tZMT0ERT+rZCI0XY7GdN/wqLDSLk+hbmb5pJ2Zxq7/rqLZfnLqFpQRUdTh9vxQooVDjMo7P3XXtacvYbY7FgK3ywkMj7S7UhmiIoYHUHG3RnM2zSP5G8ms/3J7ZRml1L73Vo699o1IGCFwwwCraWtrPnMGqJTo5n59kyixn/4xj7G9LeopChyfpFDyYYSxp87noafNFCaWUrDvQ10Hxre14BY4TAhbf/K/VScXkHkhEiKlhQRlWRFwwys2MxY8n+XT/GqYkZ/ZDS1N9dSmlNK0xNNeLqG5zUgVjhMyGpb00bFpyuIGBNB0d+LiE7u+4Y+xgTbyJkjKfxLIUX/LCImNYbqq6opm1FGy8stw+4aECscJiQd2HCAitMqCIsOo+jvRcSkxbgdyRgA4j8Rz0n/OYmCVwpAoPK8SlbOW8mef+xxO9qAscJhQs6hTYeoOLUCgJlLZhKbFetyImM+SERI/Hwic9bMIe/JPA5vO0zFKRVUnFHB/lX73Y4XdFY4TEhp39xO+SnleDo8zFwykxFTR7gdyZg+Sbgw6WuTKKkuIetnWexfvp8Vs1aw7oJ1HKwZuteAWOEwIaNjawflp5TT3drNzDdnMnL6SLcjGeOX8JhwUr+dyrzaeUy5fQo7X93J8mnLqb66mo7tQ+8aECscJiQc3nGY8lPL6WzppPBvhYyaNcrtSMYELGJMBJk/ymRuzVwmXTWJbYu2UZpVSu33auna1+V2vH5jhcO47vDOw1ScVkHHlg5m/GWG3SfaDHrRk6LJfSSXOevnMP7s8TQsbGBp1lK23L+F7vbBfw2IFQ7jqs49naz+9GoO1RxixmsziP+feLcjGdNv4rLjyH82n9krZzOqeBSbbtzEstxlbHtqG9o9eE/htcJhXNPV2sXqM1dzoPIABa8UkHBKgtuRjAmKUSeNYubfZjLz7zOJmhRF1eVVLC9czs4/7RyU14BY4TCu6D7QzZrPrqFtRRsFLxQw7oxxbkcyJugSTk5g1tJZFLxUgHYraz+/llUfW8Xed/e6HS0gVjjMgOs+1M2as9ew7z/7mLZ4GuPPGe92JGMGjIiQeG4ic9bOIffxXNob2in/ZDmrP7uattVtbsfzixUOM6A8HR4qz61k7z/2MvXpqUw4f4LbkYxxRVhEGJOvnMzcjXPJ/Gkmre+1UlZUxrqL1nGo7pDb8Y4qqIVDRM4QkSoRqRGRW3uZniAir4jIahFZJiLTfabdICKVIrJWRJ4VkRhn/L0issFZ5hURiQ/mezD9x9PpofLLlez+225yF+Uy8aKJbkcyxnXhseFMuWkKczfNZcotU9j58k6W5S1j43UbOdx82O14vQpa4RCRcOBh4EwgH7hARPJ7zHYbUK6qhcAlwIPOssnAdUCxqk4HwoGvOMu8BUx3lqkGvhus92D6j6fLw/oL17PrT7vI/mU2k6+c7HYkY0JKZEIkmT/2XgMy8fKJbH1kK0szl1L3/Tq6WkPrGpBg7nGUADWqWquqh4HngHN6zJMPLAFQ1Q1AuogkOdMigFgRiQDigCZnvjdV9chWXAqkBPE9mH6gHqXq8ipa/tBC1n1ZpFxjvzJj+hI9OZq8R/MoWVfCuM+MY/MPNlOaVUrjg414OkKjG/dgFo5kYIvPcKMzzlcFcC6AiJQAaUCKqm4F7gMagG3APlV9s5d1XA683tvKRWS+iJSJSFlLS8sJvRFz/NSjVH+9mh2/3UH6D9NJvTHV7UjGDApxuXEUvFDArOWzGDFzBDXfqqE0r5Ttz2x3/RqQYBYO6WVcz3d7D5AgIuXAtcAqoEtEEvDunWQAk4ERInLRB15c5HagC1jc28pVdZGqFqtqcWJi4gm9EXN8VJWa62vY9sQ2ptw+hfTvpbsdyZhBZ3TxaIreLqLwrUIix0ey4dINlBWVsfPP7l0DEszC0Qj4/nuZgtPcdISqtqrq11S1CO8xjkSgDjgNqFPVFlXtBF4GPnpkORG5FPgccKEOxqtnhgFVpfbmWrY+tJWUG1PI+GGG25GMGdTGnjaW2ctmk/9CPp4OD2vPWkv5J8rZ9+99A54lmIVjOZAjIhkiEoX34ParvjOISLwzDeBK4F1VbcXbRDVPROJERIBTgfXOMmcAtwBnq+rQ7bd4kKv/fj1b7tvC5Ksnk3VvFt5fozHmREiYMOFLE5hTOYfcR3M5tOkQqz6+ijVnr6Ft7cBdAxIRrBdW1S4RuQZ4A+9ZUU+qaqWILHCmPwpMA54RkW5gHXCFM61URF4EVuJtjloFLHJe+iEgGnjL+TJaqqoLgvU+jH92LN5B7e21dDR0ED46nO593Uy8YiI5v8ixomFMPwuLDGPy1yeTdHESjQ820vCTBsoKy0i6OImMH2QQkxbzgc9k9JRoMhdmknRh0rFf3A8yHFp6iouLtayszO0YQ9aOxTuoml+F56DPGR/hMPWpqUy82K7VMCbYOnd30nBPA1t/uRX1KPGnxLPvn/vwHPrvZzIsLoy8RXkBFQ8RWaGqxT3H25Xj5oTV3l77waIB0A11d9S5E8iYYSZybCRZP82iZGMJEy+ZyJ6/7flA0QDwHPRQe3ttv6zvmIVDRD4nIlZgTJ86Gnq/w1lf440xwRGTEkPe43m9n9NK/30m/SkIXwE2ishPRWRav6zVDCnRU6IDGm+MCa5gfyaPWThU9SLgJGAT8JSIvOdcXGf39jQAJJ734etkwuLCyFyY6UIaY0zmwkzC4j749d6fn0m/mqCcU2RfwtttyCTgC8BKEbm2X1KYQaurtYvm55uJSony/jcjEJ0WHfBBOGNM/0m6MIm8RXlEpwXnM3nM03FF5Cy8XXtkAb8FSlS1WUTi8F5b8ct+SWIGpdrv1nK46TCzls5idIndK9yYUJF0YVLQ/nnz5zqOLwEPqOq7viNV9aCIXB6UVGZQ2PfvfTT9qonk65KtaBgzjPhTOL6Pt6NBAEQkFkhS1XpVXRK0ZCakeTo8VF1VRXRqNBk/su5EjBlO/DnG8QfA94TgbmecGcYa7mng4PqD5D6aS8TIoHVAYIwJQf4UjgjnfhoAOM+jjjK/GeIOrDvA5oWbmfDVCYw7c5zbcYwxA8yfwtEiImcfGRCRc4CdwYtkQpl6lKqrqggfHU72z7PdjmOMcYE/bQwLgMUi8hDe6xG34O0C3QxDTY820fqfVqY+M5WoRNvxNGY4OmbhUNVNeLs4H4m3U8T9wY9lQlH7lnZqb60l4VMJJF1k12gYM1z5dVRTRD4LFAAxR7rIVtUfBDGXCTGqysarN6LdSu5judZVujHDmD8XAD4KxAEnA08AXwSWBTmXCTEtL7aw67VdZP0si9iMWLfjGGNc5M/B8Y+q6iXAHlW9G/gIH7wlrBniOnd3svGajYycPZLk65LdjmOMcZk/TVXtzs+DIjIZ2AXYFV/DyKabNtG5q5PCNwoJi7Ae9o0Z7vwpHK+JSDxwL95buSrweDBDmdCx5+972P7kdlJvSWVUkXWIbIw5RuFwbuC0RFX3Ai+JyJ+BGFXdNxDhjLu6D3VTNb+KmKwY0r+f7nYcY0yIOGq7g6p6gJ/5DHdY0Rg+6u+up31TO3mP5xEeG+52HGNMiPCnwfpNETlP7PzLYWV/+X623LeFiZdPJOHkBLfjGGNCiD/HOL4NjAC6RKQd79XjqqrWj/YQ5enyUHVlFZHjI8m6N8vtOMaYEOPPleN2RHSY2frgVtpWtJH/Qj6RYyPdjmOMCTH+XAD4id7G97yxkxkaDtUeou6OOsadNY7EL374XuLGGONPU9VNPs9jgBJgBXBKUBIZ16gq1QuqkQgh55Ec61bEGNMrf5qqzvIdFpFU4KdBS2Rcs+O3O9jz1h5yHs4hJiXG7TjGmBB1PJcBNwLT+zuIcdfh5sPU3FDD6I+OZvKCyW7HMcaEMH+OcfwS79Xi4C00RUBFEDMZF9TcUEP3/m7yHs9DwqyJyhjTN3+OcZT5PO8CnlXVfwcpj3HBrtd30fz7ZtLvSmdE/gi34xhjQpw/heNFoF1VuwFEJFxE4lT1YHCjmYHQ1dZF9YJq4qbFMeXWKW7HMcYMAv4c41gC+N6AIRZ4OzhxzECr+14dHVs6yHsij7Bo6/nWGHNs/nxTxKhq25EB53lc8CKZgdJa2srWX2xl8jcnM+ajY9yOY4wZJPwpHAdEZNaRARGZDRwKXiQzEDyHPVRdVUXU5Cgy/zfT7TjGmEHEn2Mc3wL+ICJNzvAk4MtBS2QGxJZ7t3BgzQGm/2k6EaP9uvW8McYAfuxxqOpyYCrwDeCbwDRVXeHPi4vIGSJSJSI1InJrL9MTROQVEVktIstEZLrPtBtEpFJE1orIsyIS44z/kjPeIyLF/r5R818Hqw5S/4N6Es9PZPzZ492OY4wZZI5ZOETkamCEqq5V1TXASBH5ph/LhQMPA2cC+cAFIpLfY7bbgHJVLQQuAR50lk0GrgOKVXU6EA58xVlmLXAuYH1lHQf1KFXzqwiPCyf7wWy34xhjBiF/jnFc5dwBEABV3QNc5cdyJUCNqtaq6mHgOeCcHvPk4z1rC1XdAKSLSJIzLQKIFZEIvAfjm5z51qtqlR/rN73Y9sQ29r27j6yfZRE9MdrtOMaYQcifwhHmexMnZ08iyo/lkoEtPsONzjhfFXj3HhCREiANSFHVrcB9QAOwDdinqm/6sc73ich8ESkTkbKWlpZAFh2yOpo62HTzJuJPiWfi1ya6HccYM0j5UzjeAF4QkVNF5BTgWeB1P5brrd8K7TF8D5AgIuXAtcAqvDeMSsC7d5IBTAZGiMhFfqzzvytSXaSqxapanJho3YMDbLx2I9qh5D6Waz3fGmOOmz+n09wCzMd7cFzwfrlP8mO5RiDVZzgFp7npCFVtBb4G4OzV1DmP04E6VW1xpr0MfBT4nR/rNb1oeaWFnS/vJPOeTOKy7TIcY8zx8+esKg+wFKgFioFTgfV+vPZyIEdEMkQkCu/B7Vd9ZxCReGcawJXAu04xaQDmiUicU1D8XafpRefeTjZevZGRRSNJ+XaK23GMMYNcn3scIpKL98v+AmAX8DyAqp7szwurapeIXIO3qSsceFJVK0VkgTP9UWAa8IyIdAPrgCucaaUi8iKwEm/HiquARU6uLwC/BBKBv4hIuaqeHugbH05qb63l8I7DzHhtBmGR1q2IMebEiGrPww7OBBEP8C/gClWtccbVquqgu8y4uLhYy8rKjj3jELT33b2Uf7KclBtTyL7PTr81xvhPRFao6oeulzvav5/nAduBf4jI4yJyKr0f8DYhqru9m6r5VcRkxJBxd4bbcYwxQ0SfhUNVX1HVL+O9avwd4AYgSUR+JSKfHqB85gQ0LGzgUNUhch/LJXxEuNtxjDFDhD8Hxw+o6mJV/RzeM6PKgQ91H2JCS9uaNhruaSDpkiTGfmqs23GMMUNIQEdKVXW3qj6mqqcEK5A5cdqtVF1VRURCBNn323ENY0z/sm5Rh6CtD29lf+l+pi2eRuS4SLfjGGOGGDs3c4hp39xO7W21jD1zLBMumOB2HGPMEGSFYwhRVaq/WQ1A7q+sWxFjTHBYU9UQ0vxcM7v/upvsn2cTkxbjdhxjzBBlexxDROeuTmqur2FUySiSr+nZCbExxvQf2+MYImpurKFrTxd5S/KQcGuiMsYEj+1xDAG739rNjqd3kHpLKiNnjHQ7jjFmiLPCMch1H+im+uvVxObGkva9NLfjGGOGAWuqGuTq76qnva6don8WER5j3YoYY4LP9jgGsf0r9rPl/i1Mmj+J+E/Eux3HGDNMWOEYpDydHqqurCIqKYrMnwy6nu6NMYOYNVUNUo0PNNJW3kbBSwVExlu3IsaYgWN7HIPQwZqD1H+/nvFfGE/iuYluxzHGDDNWOAYZVaX669VIlJDzUI7bcYwxw5A1VQ0y23+znb1/30vuo7lET452O44xZhiyPY5BpGN7B5tu3MSY/xnDpKsmuR3HGDNMWeEYRGqur6H7QDd5j+chYdatiDHGHVY4Bomdr+2k5YUW0u5IIy4vzu04xphhzArHINDV2sXGb25kxPQRTLl5ittxjDHDnB0cHwTqbq+jY2sHBS8WEBZltd4Y4y77Fgpx+/6zj60PbyX52mRGzx3tdhxjjLHCEco8HR6qrqoiOjWajB9luB3HGGMAa6oKaQ0/aeDguoPM+OsMIkbZr8oYExpsjyNEHVh3gM0/2syECyYw7sxxbscxxpj3WeEIQepRqq6qInxUONk/z3Y7jjHGfIC1f4SgpseaaP1PK1OfnkrUhCi34xhjzAfYHkeIaW9sp/aWWhI+lUDSxUluxzHGmA+xwhFCVJWNV29Eu5TcR3MRsW5FjDGhx5qqQkjLSy3senUXWfdlEZsZ63YcY4zpVVD3OETkDBGpEpEaEbm1l+kJIvKKiKwWkWUiMt1n2g0iUikia0XkWRGJccaPFZG3RGSj8zMhmO9hoHTu6WTjNRsZOXskydcnux3HGGP6FLTCISLhwMPAmUA+cIGI5PeY7TagXFULgUuAB51lk4HrgGJVnQ6EA19xlrkVWKKqOcASZ3jQ23TTJjp3dpL3eB5hEdaCaIwJXcH8hioBalS1VlUPA88B5/SYJx/vlz+qugFIF5EjR4QjgFgRiQDigCZn/DnA087zp4HPB+0dDJA9/9jD9l9vJ/U7qYw6aZTbcYwx5qiCWTiSgS0+w43OOF8VwLkAIlICpAEpqroVuA9oALYB+1T1TWeZJFXdBuD8nBC0dzAAug91Uz2/mpisGNK/n+52HGOMOaZgFo7eTgnSHsP3AAkiUg5cC6wCupzjFucAGcBkYISIXBTQykXmi0iZiJS1tLQEHH6gbP7BZg7VHCJvUR7hseFuxzHGmGMKZuFoBFJ9hlP4b3MTAKraqqpfU9UivMc4EoE64DSgTlVbVLUTeBn4qLPYDhGZBOD8bO5t5aq6SFWLVbU4MTGxH99W/2mraKPh3gYmXj6RhFOGxDF+Y8wwEMzCsRzIEZEMEYnCe3D7Vd8ZRCTemQZwJfCuqrbibaKaJyJx4r2Y4VRgvTPfq8ClzvNLgT8F8T0EjafLQ9WVVUSOiyTr3iy34xhjjN+Cdh2HqnaJyDXAG3jPinpSVStFZIEz/VFgGvCMiHQD64ArnGmlIvIisBLowtuEtch56XuAF0TkCrwF5kvBeg/BtPUXW9lftp/85/OJHBvpdhxjjPGbqPY87DD0FBcXa1lZmdsx3neo7hDLpy8n4dQEpv9pul0hbowJSSKyQlWLe463CwYGmKpSvaAaCRNyHs6xomGMGXSsy5EBtuN3O9jz5h5yHsohJjXG7TjGGBMw2+MYQIdbDlNzQw2jPzKayd+Y7HYcY4w5LlY4BlDNDTV0t3aT93geEmZNVMaYwckKxwDZ9foumhc3M+W2KYwoGOF2HGOMOW5WOAZAV1sX1d+oJm5aHGnfTXM7jjHGnBA7OD4A6u+op2NzByf930mERVutNsYMbvYtFmSty1ppfLCRyd+czJiPjXE7jjHGnDArHEHk6fR2KxI1OYrMH2e6HccYY/qFNVUF0ZZ7t3BgzQGm/2k6EaNtUxtjhgbb4wiSg9UHqf9BPYlfSmT82ePdjmOMMf3GCkcQqEepml9FeGw42b/IdjuOMcb0K2s/CYJtv97Gvn/uI++JPKInRrsdxxhj+pXtcfSzjm0dbLppE/EnxzPx8oluxzHGmH5nhaOfbbx2I552D7mP5VrPt8aYIcmaqvpRyx9b2PnSTjJ+nEFcTpzbcYwxJihsj6OfdO3rYuPVGxkxcwSpN6YeewFjjBmkbI+jn9TeWsvh7YeZ/qfphEVaPTbGDF32DdcP9v5rL02PNpHyrRRGF492O44xxgSVFY4T1N3eTfX8amLSY8j4QYbbcYwxJuisqeoENfxvAwc3HKTwjULCR4S7HccYY4LO9jhOQNvaNhp+3EDSxUmM/fRYt+MYY8yAsMJxnLRbqbqyioj4CLLuz3I7jjHGDBhrqjpOWx/Zyv7S/Uz73TSixke5HccYYwaM7XEch/aGdmq/W8vYM8Yy4asT3I5jjDEDygpHgFSV6m9UA5D7qHUrYowZfqypKkDNzzez+6+7yXogi5i0GLfjGGPMgLM9jgB07uqk5roaRs0ZRcq1KW7HMcYYV9geRwA2fWcTXXu6yHs7Dwm3JipjzPBkexx+2v32brb/ZjupN6cysnCk23GMMcY1Vjj80H2wm+qvVxObG0vaHWluxzHGGFdZU5Uf6u+qp722naJ3igiPsW5FjDHDmxWOPuxYvIPa22vpaOgAhTEnjyH+k/FuxzLGGNdZU1UvdizeQdX8Kjo2e4sGwP6l+9mxeIe7wYwxJgQEtXCIyBkiUiUiNSJyay/TE0TkFRFZLSLLRGS6Mz5PRMp9Hq0i8i1n2kwReU9E1ojIayLS7zfAqL29Fs9BzwfGeQ55qL29tr9XZYwxg07QCoeIhAMPA2cC+cAFIpLfY7bbgHJVLQQuAR4EUNUqVS1S1SJgNnAQeMVZ5gngVlWd4Yy7qb+zdzR0BDTeGGOGk2DucZQANapaq6qHgeeAc3rMkw8sAVDVDUC6iCT1mOdUYJOqbnaG84B3nedvAef1d/DoKdEBjTfGmOEkmIUjGdjiM9zojPNVAZwLICIlQBrQ85LsrwDP+gyvBc52nn8JSO1t5SIyX0TKRKSspaUloOCZCzMJi/vgpgmLCyNzYWZAr2OMMUNRMAtHb5dWa4/he4AEESkHrgVWAV3vv4BIFN4i8QefZS4HrhaRFcAo4HBvK1fVRaparKrFiYmJAQVPujCJvEV5RKdFg0B0WjR5i/JIurDnzpAxxgw/wTwdt5EP7g2kAE2+M6hqK/A1APF2M1vnPI44E1ipqjt8ltkAfNpZJhf4bDDCJ12YZIXCGGN6Ecw9juVAjohkOHsOXwFe9Z1BROKdaQBXAu86xeSIC/hgMxUiMsH5GQZ8D3g0SPmNMcb0ImiFQ1W7gGuAN4D1wAuqWikiC0RkgTPbNKBSRDbg3bu4/sjyIhIHfAp4ucdLXyAi1cAGvHswTwXrPRhjjPkwUe152GHoKS4u1rKyMrdjGGPMoCIiK1S1uOd4u3LcGGNMQKxwGGOMCciwaKoSkRZg8zFn7N14YGc/xukvliswliswliswoZoLTixbmqp+6HqGYVE4ToSIlPXWxuc2yxUYyxUYyxWYUM0FwclmTVXGGGMCYoXDGGNMQKxwHNsitwP0wXIFxnIFxnIFJlRzQRCy2TEOY4wxAbE9DmOMMQGxwmGMMSYgVjh8iMiTItIsImt9xo0VkbdEZKPzMyFEct0lIlt9bq/7GRdypYrIP0RkvYhUisj1znhXt9lRcrm6zUQkxrlFcoWT625nvNvbq69crv+NOTnCRWSViPzZGXb9M9lHLte3l4jUO7fVLheRMmdcv28vKxwf9BvgjB7jbgWWqGoO3rsVfuje6QPgN3w4F8ADR26xq6p/HeBM4L13yo2qOg2Yh/c+Kfm4v836ygXubrMO4BRVnQkUAWeIyDzc31595QL3/8bA2/npep9ht7fXET1zQWhsr5Od9R+5dqPft5cVDh+q+i6wu8foc4CnnedPA58fyEzQZy7Xqeo2VV3pPN+P90OUjMvb7Ci5XKVebc5gpPNQ3N9efeVynYik4L3nzhM+o13/TPaRK1T1+/aywnFsSaq6DbxfSMAEl/P4ukZEVjtNWa7srh8hIunASUApIbTNeuQCl7eZ07xRDjQDb6lqSGyvPnKB+39jPwduBjw+41zfXn3kAve3lwJvisgKEZnvjOv37WWFY/D6FZCFt2lhG/Azt4KIyEjgJeBbPW7E5apecrm+zVS1W1WL8N4Rs0REpg90ht70kcvV7SUinwOaVXXFQK73WI6Sy/W/L+BjqjoL7/2NrhaRTwRjJVY4jm2HiEwCcH42u5wHAFXd4XzYPcDjQIkbOUQkEu+X82JVPXLTLde3WW+5QmWbOVn2Au/gPXbl+vbqLVcIbK+PAWeLSD3wHHCKiPwO97dXr7lCYHuhqk3Oz2bgFSdDv28vKxzH9ipwqfP8UuBPLmZ535E/BMcXgLV9zRvEDAL8Glivqvf7THJ1m/WVy+1tJiKJIhLvPI8FTsN7J0u3t1evudzeXqr6XVVNUdV0vLee/ruqXoTL26uvXG5vLxEZISKjjjwHPu1k6P/tpar2cB5472++DegEGoErgHF4z0TY6PwcGyK5fgusAVY7fxiTXMj1cbxtqquBcufxGbe32VFyubrNgEJglbP+tcCdzni3t1dfuVz/G/PJ+P+AP4fC9jpKLrf/vjKBCudRCdwerO1lXY4YY4wJiDVVGWOMCYgVDmOMMQGxwmGMMSYgVjiMMcYExAqHMcaYgFjhMMZFInK7T2+q3T7Pr3M7mzF9sdNxjQkRItKmqiN7jBO8n9OefSIZ4xrb4zCmDyKSLt57ejzu3KfiTefKakTkHREpdp6Pd7qfQEQuE5E/ishrIlInIteIyLed+zYsFZGxAaz3EWAlkCoiN4nIcqcDvbt95r1IvPfSKBeRx5zOCsNF5Dcista5N8MNQdlAZtiywmHM0eUAD6tqAbAXOM+PZaYDX8XbT9BC4KCqngS8B1zi53rzgGec5fKcHCV4O9CbLSKfEJFpwJfxdmxXBHQDFzrzJKvqdFWdATzl5zqN8UuE2wGMCXF1qlruPF8BpPuxzD/Uex+Q/SKyD3jNGb8Gb/ce/tisqkud5592Hquc4ZF4C0khMBtY7m3RIhZvB3avAZki8kvgL8Cbfq7TGL9Y4TDm6Dp8nnfj/XIG710Gj+yxxxxlGY/PsAf/P3MHfJ4L8GNVfcx3BhG5FnhaVb/bc2ERmQmcDlwNnA9c7ud6jTkma6oy5vjU4/1vH+CLQV7XG8Dlzv1FEJFkEZmAt8O6LzrPj9xbOk1ExgNhqvoScAcwK8j5zDBjexzGHJ/7gBdE5GLg78Fckaq+6RzPeM9pkmoDLlLVdSLyPbx3fAvD23vy1cAh4ClnHMCH9kiMORF2Oq4xxpiAWFOVMcaYgFjhMMYYExArHMYYYwJihcMYY0xArHAYY4wJiBUOY4wxAbHCYYwxJiD/H4swZcKKjEMiAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plotting the the accuracy of models for each num_trees in experiments\n",
    "\n",
    "plt.plot(TREES, ACC,\"mo-\")\n",
    "plt.title(\"Accuracy vs num Trees\")\n",
    "plt.xlabel(\"num Trees\")\n",
    "plt.ylabel(\"Accuracy\")\n",
    "plt.savefig(\"Accuracy vs num_Trees.png\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}