FFT data prep.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "import cupy as cp\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm.notebook import tqdm\n",
    "import soundfile as sf\n",
    "import gc\n",
    "from numpy import savez_compressed\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4727, 1992, 'data/train\\\\00204008d.flac')"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainfiles = glob.glob( 'data/train/*.flac' )\n",
    "testfiles = glob.glob( 'data/test/*.flac' )\n",
    "len(trainfiles), len(testfiles), trainfiles[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "traint = pd.read_csv( 'data/train_tp.csv' )\n",
    "trainf = pd.read_csv( 'data/train_fp.csv' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['003bec244', '006ab765f', '007f87ba2', ..., 'ffebe7313',\n",
       "       'fff163132', 'fffb79246'], dtype=object)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full =pd.concat([traint, trainf])\n",
    "full.recording_id.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([14, 23, 12, 17, 10,  8,  0, 18, 15,  1,  6,  7, 11, 13, 16, 19,  3,\n",
       "        2,  4, 22,  9, 21, 20,  5], dtype=int64)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full.species_id.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Simple reshape to 1000 instead of mapping to a 1d domain\n",
    "def extract_fft(fn):\n",
    "    data, samplerate = sf.read(fn)\n",
    "    data = cp.array(data)\n",
    "    varfft = cp.abs( cp.fft.fft(data)[:(len(data)//2)] )\n",
    "    return cp.asnumpy( varfft.reshape( (1000,1440) ).mean(axis=1) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1216, 1000), 1216)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TRAIN = []\n",
    "TARGET = []\n",
    "for i in tqdm(range(traint.shape[0])):\n",
    "    fn = traint.recording_id.values[i]\n",
    "    fft = extract_fft(\"data/train/\" + fn + \".flac\")\n",
    "    TRAIN.append(fft)\n",
    "    TARGET.append(traint.species_id.values[i])\n",
    "\n",
    "TP = np.stack(TRAIN)\n",
    "TARGET = np.array(TARGET)\n",
    "TP.shape, len(TARGET)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e0b303837de74b52b95de1292acd613f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=7781.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(7781, 1000)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TRAIN = []\n",
    "TARGET = []\n",
    "for i in tqdm(range(trainf.shape[0])):\n",
    "    fn = trainf.recording_id.values[i]\n",
    "    fft = extract_fft(\"data/train/\" + fn + \".flac\")\n",
    "    TRAIN.append(fft)\n",
    "\n",
    "FP = np.stack(TRAIN)\n",
    "gc.collect()\n",
    "FP.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8997, 1000)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TRAIN = np.vstack( (TP, FP) )\n",
    "TRAIN.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "savez_compressed(\"savedata/TF1/TRAIN.npz\", TRAIN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "18bcbf59513c470990743362155df86e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=1992.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1992, 1000)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test = []\n",
    "for fn in tqdm(testfiles):\n",
    "    X_test.append( extract_fft(fn) )\n",
    "X_test = np.stack(X_test)\n",
    "gc.collect()\n",
    "\n",
    "X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "savez_compressed(\"savedata/TF1/TEST.npz\", X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1216, 26)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# y_train = include labels for true positives and false positives\n",
    "tt = traint[['recording_id','species_id']].copy()\n",
    "# tf = trainf[['recording_id','species_id']].copy()\n",
    "# tf['species_id'] = -1\n",
    "\n",
    "# y_train_all_classes = pd.concat( (tt, tf) )\n",
    "y_train_all_classes = tt\n",
    "\n",
    "for i in range(24):\n",
    "    y_train_all_classes['s'+str(i)] = 0\n",
    "    # Notice that the False Positive labels should be 0\n",
    "    # Will correct them in the helper function\n",
    "    y_train_all_classes.loc[y_train_all_classes.species_id==i,'s'+str(i)] = 1\n",
    "\n",
    "y_train_all_classes.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1216, 26)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_all_classes.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = pd.DataFrame(X_train)\n",
    "recording_id = [path[11:20] for path in trainfiles]\n",
    "X_train = X_train.set_index(pd.Index(recording_id))\n",
    "\n",
    "X_test = pd.DataFrame(X_test)\n",
    "recording_id = [path[10:19] for path in testfiles]\n",
    "X_test = X_test.set_index(pd.Index(recording_id))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>990</th>\n",
       "      <th>991</th>\n",
       "      <th>992</th>\n",
       "      <th>993</th>\n",
       "      <th>994</th>\n",
       "      <th>995</th>\n",
       "      <th>996</th>\n",
       "      <th>997</th>\n",
       "      <th>998</th>\n",
       "      <th>999</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>00204008d</th>\n",
       "      <td>11.790391</td>\n",
       "      <td>35.339951</td>\n",
       "      <td>33.839856</td>\n",
       "      <td>24.667511</td>\n",
       "      <td>19.069494</td>\n",
       "      <td>15.649259</td>\n",
       "      <td>15.250774</td>\n",
       "      <td>15.013724</td>\n",
       "      <td>13.815972</td>\n",
       "      <td>12.297996</td>\n",
       "      <td>...</td>\n",
       "      <td>2.239256</td>\n",
       "      <td>2.334394</td>\n",
       "      <td>2.206936</td>\n",
       "      <td>2.271055</td>\n",
       "      <td>2.221637</td>\n",
       "      <td>2.324264</td>\n",
       "      <td>2.284480</td>\n",
       "      <td>2.319976</td>\n",
       "      <td>2.269804</td>\n",
       "      <td>2.315348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>003b04435</th>\n",
       "      <td>23.789466</td>\n",
       "      <td>41.844881</td>\n",
       "      <td>34.087390</td>\n",
       "      <td>24.990900</td>\n",
       "      <td>33.028959</td>\n",
       "      <td>29.955412</td>\n",
       "      <td>39.047059</td>\n",
       "      <td>32.389619</td>\n",
       "      <td>25.473174</td>\n",
       "      <td>23.336571</td>\n",
       "      <td>...</td>\n",
       "      <td>3.865874</td>\n",
       "      <td>3.812509</td>\n",
       "      <td>3.811468</td>\n",
       "      <td>3.771802</td>\n",
       "      <td>3.760778</td>\n",
       "      <td>3.813280</td>\n",
       "      <td>3.733179</td>\n",
       "      <td>3.802968</td>\n",
       "      <td>3.776638</td>\n",
       "      <td>3.893550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>003bec244</th>\n",
       "      <td>10.672936</td>\n",
       "      <td>21.634743</td>\n",
       "      <td>33.368987</td>\n",
       "      <td>27.741980</td>\n",
       "      <td>63.404136</td>\n",
       "      <td>19.311545</td>\n",
       "      <td>22.348948</td>\n",
       "      <td>15.599358</td>\n",
       "      <td>32.488332</td>\n",
       "      <td>19.724591</td>\n",
       "      <td>...</td>\n",
       "      <td>3.154830</td>\n",
       "      <td>3.184257</td>\n",
       "      <td>3.164174</td>\n",
       "      <td>3.174290</td>\n",
       "      <td>3.087597</td>\n",
       "      <td>2.892178</td>\n",
       "      <td>3.010763</td>\n",
       "      <td>3.027793</td>\n",
       "      <td>2.996881</td>\n",
       "      <td>3.074568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>005f1f9a5</th>\n",
       "      <td>227.698311</td>\n",
       "      <td>277.244232</td>\n",
       "      <td>394.248199</td>\n",
       "      <td>141.685471</td>\n",
       "      <td>178.871626</td>\n",
       "      <td>247.544134</td>\n",
       "      <td>157.892434</td>\n",
       "      <td>274.957322</td>\n",
       "      <td>230.505231</td>\n",
       "      <td>229.685355</td>\n",
       "      <td>...</td>\n",
       "      <td>5.692819</td>\n",
       "      <td>7.393680</td>\n",
       "      <td>6.652094</td>\n",
       "      <td>6.163386</td>\n",
       "      <td>6.314197</td>\n",
       "      <td>6.556996</td>\n",
       "      <td>6.150115</td>\n",
       "      <td>6.168900</td>\n",
       "      <td>5.961046</td>\n",
       "      <td>5.587845</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>006ab765f</th>\n",
       "      <td>97.733087</td>\n",
       "      <td>207.211464</td>\n",
       "      <td>217.810163</td>\n",
       "      <td>178.775455</td>\n",
       "      <td>296.608300</td>\n",
       "      <td>331.616375</td>\n",
       "      <td>206.194665</td>\n",
       "      <td>133.995695</td>\n",
       "      <td>99.661902</td>\n",
       "      <td>62.896095</td>\n",
       "      <td>...</td>\n",
       "      <td>2.363506</td>\n",
       "      <td>2.374879</td>\n",
       "      <td>2.395297</td>\n",
       "      <td>2.411069</td>\n",
       "      <td>2.405776</td>\n",
       "      <td>2.426955</td>\n",
       "      <td>2.392858</td>\n",
       "      <td>2.413523</td>\n",
       "      <td>2.325952</td>\n",
       "      <td>2.367381</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1000 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  0           1           2           3           4    \\\n",
       "00204008d   11.790391   35.339951   33.839856   24.667511   19.069494   \n",
       "003b04435   23.789466   41.844881   34.087390   24.990900   33.028959   \n",
       "003bec244   10.672936   21.634743   33.368987   27.741980   63.404136   \n",
       "005f1f9a5  227.698311  277.244232  394.248199  141.685471  178.871626   \n",
       "006ab765f   97.733087  207.211464  217.810163  178.775455  296.608300   \n",
       "\n",
       "                  5           6           7           8           9    ...  \\\n",
       "00204008d   15.649259   15.250774   15.013724   13.815972   12.297996  ...   \n",
       "003b04435   29.955412   39.047059   32.389619   25.473174   23.336571  ...   \n",
       "003bec244   19.311545   22.348948   15.599358   32.488332   19.724591  ...   \n",
       "005f1f9a5  247.544134  157.892434  274.957322  230.505231  229.685355  ...   \n",
       "006ab765f  331.616375  206.194665  133.995695   99.661902   62.896095  ...   \n",
       "\n",
       "                990       991       992       993       994       995  \\\n",
       "00204008d  2.239256  2.334394  2.206936  2.271055  2.221637  2.324264   \n",
       "003b04435  3.865874  3.812509  3.811468  3.771802  3.760778  3.813280   \n",
       "003bec244  3.154830  3.184257  3.164174  3.174290  3.087597  2.892178   \n",
       "005f1f9a5  5.692819  7.393680  6.652094  6.163386  6.314197  6.556996   \n",
       "006ab765f  2.363506  2.374879  2.395297  2.411069  2.405776  2.426955   \n",
       "\n",
       "                996       997       998       999  \n",
       "00204008d  2.284480  2.319976  2.269804  2.315348  \n",
       "003b04435  3.733179  3.802968  3.776638  3.893550  \n",
       "003bec244  3.010763  3.027793  2.996881  3.074568  \n",
       "005f1f9a5  6.150115  6.168900  5.961046  5.587845  \n",
       "006ab765f  2.392858  2.413523  2.325952  2.367381  \n",
       "\n",
       "[5 rows x 1000 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>990</th>\n",
       "      <th>991</th>\n",
       "      <th>992</th>\n",
       "      <th>993</th>\n",
       "      <th>994</th>\n",
       "      <th>995</th>\n",
       "      <th>996</th>\n",
       "      <th>997</th>\n",
       "      <th>998</th>\n",
       "      <th>999</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>000316da7</th>\n",
       "      <td>26.777072</td>\n",
       "      <td>48.789862</td>\n",
       "      <td>48.658318</td>\n",
       "      <td>38.345478</td>\n",
       "      <td>29.403780</td>\n",
       "      <td>22.604672</td>\n",
       "      <td>21.870240</td>\n",
       "      <td>22.177086</td>\n",
       "      <td>23.431294</td>\n",
       "      <td>19.994830</td>\n",
       "      <td>...</td>\n",
       "      <td>2.844235</td>\n",
       "      <td>2.904594</td>\n",
       "      <td>2.859302</td>\n",
       "      <td>2.780773</td>\n",
       "      <td>2.749330</td>\n",
       "      <td>2.705327</td>\n",
       "      <td>2.715763</td>\n",
       "      <td>2.707143</td>\n",
       "      <td>2.663817</td>\n",
       "      <td>2.696315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>003bc2cb2</th>\n",
       "      <td>15.370870</td>\n",
       "      <td>134.196483</td>\n",
       "      <td>137.053862</td>\n",
       "      <td>86.158029</td>\n",
       "      <td>119.924272</td>\n",
       "      <td>102.650543</td>\n",
       "      <td>116.111480</td>\n",
       "      <td>270.525276</td>\n",
       "      <td>275.029161</td>\n",
       "      <td>189.287502</td>\n",
       "      <td>...</td>\n",
       "      <td>2.894080</td>\n",
       "      <td>2.900234</td>\n",
       "      <td>2.911039</td>\n",
       "      <td>2.951832</td>\n",
       "      <td>2.927140</td>\n",
       "      <td>2.856842</td>\n",
       "      <td>2.900470</td>\n",
       "      <td>2.921184</td>\n",
       "      <td>3.007650</td>\n",
       "      <td>2.850644</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0061c037e</th>\n",
       "      <td>102.291469</td>\n",
       "      <td>146.259771</td>\n",
       "      <td>184.226037</td>\n",
       "      <td>87.693767</td>\n",
       "      <td>52.898082</td>\n",
       "      <td>51.871333</td>\n",
       "      <td>46.215581</td>\n",
       "      <td>58.855169</td>\n",
       "      <td>63.327388</td>\n",
       "      <td>84.239141</td>\n",
       "      <td>...</td>\n",
       "      <td>3.613505</td>\n",
       "      <td>3.624661</td>\n",
       "      <td>3.666196</td>\n",
       "      <td>3.659059</td>\n",
       "      <td>3.619957</td>\n",
       "      <td>3.654109</td>\n",
       "      <td>3.714032</td>\n",
       "      <td>3.708906</td>\n",
       "      <td>3.647022</td>\n",
       "      <td>3.700800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>010eb14d3</th>\n",
       "      <td>41.571906</td>\n",
       "      <td>50.391221</td>\n",
       "      <td>52.416428</td>\n",
       "      <td>47.872607</td>\n",
       "      <td>43.501722</td>\n",
       "      <td>41.260662</td>\n",
       "      <td>43.912675</td>\n",
       "      <td>48.554317</td>\n",
       "      <td>37.681341</td>\n",
       "      <td>33.533002</td>\n",
       "      <td>...</td>\n",
       "      <td>11.037309</td>\n",
       "      <td>11.684867</td>\n",
       "      <td>11.904491</td>\n",
       "      <td>12.397820</td>\n",
       "      <td>13.009769</td>\n",
       "      <td>13.343707</td>\n",
       "      <td>13.792227</td>\n",
       "      <td>14.027738</td>\n",
       "      <td>14.297619</td>\n",
       "      <td>14.354401</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>011318064</th>\n",
       "      <td>9.232961</td>\n",
       "      <td>26.556042</td>\n",
       "      <td>20.103448</td>\n",
       "      <td>17.712726</td>\n",
       "      <td>17.324920</td>\n",
       "      <td>12.793497</td>\n",
       "      <td>12.460659</td>\n",
       "      <td>12.131253</td>\n",
       "      <td>11.745227</td>\n",
       "      <td>11.762307</td>\n",
       "      <td>...</td>\n",
       "      <td>3.470543</td>\n",
       "      <td>3.545655</td>\n",
       "      <td>3.652730</td>\n",
       "      <td>3.485103</td>\n",
       "      <td>3.645361</td>\n",
       "      <td>3.431659</td>\n",
       "      <td>3.618027</td>\n",
       "      <td>3.526598</td>\n",
       "      <td>3.630001</td>\n",
       "      <td>3.662966</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1000 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  0           1           2          3           4    \\\n",
       "000316da7   26.777072   48.789862   48.658318  38.345478   29.403780   \n",
       "003bc2cb2   15.370870  134.196483  137.053862  86.158029  119.924272   \n",
       "0061c037e  102.291469  146.259771  184.226037  87.693767   52.898082   \n",
       "010eb14d3   41.571906   50.391221   52.416428  47.872607   43.501722   \n",
       "011318064    9.232961   26.556042   20.103448  17.712726   17.324920   \n",
       "\n",
       "                  5           6           7           8           9    ...  \\\n",
       "000316da7   22.604672   21.870240   22.177086   23.431294   19.994830  ...   \n",
       "003bc2cb2  102.650543  116.111480  270.525276  275.029161  189.287502  ...   \n",
       "0061c037e   51.871333   46.215581   58.855169   63.327388   84.239141  ...   \n",
       "010eb14d3   41.260662   43.912675   48.554317   37.681341   33.533002  ...   \n",
       "011318064   12.793497   12.460659   12.131253   11.745227   11.762307  ...   \n",
       "\n",
       "                 990        991        992        993        994        995  \\\n",
       "000316da7   2.844235   2.904594   2.859302   2.780773   2.749330   2.705327   \n",
       "003bc2cb2   2.894080   2.900234   2.911039   2.951832   2.927140   2.856842   \n",
       "0061c037e   3.613505   3.624661   3.666196   3.659059   3.619957   3.654109   \n",
       "010eb14d3  11.037309  11.684867  11.904491  12.397820  13.009769  13.343707   \n",
       "011318064   3.470543   3.545655   3.652730   3.485103   3.645361   3.431659   \n",
       "\n",
       "                 996        997        998        999  \n",
       "000316da7   2.715763   2.707143   2.663817   2.696315  \n",
       "003bc2cb2   2.900470   2.921184   3.007650   2.850644  \n",
       "0061c037e   3.714032   3.708906   3.647022   3.700800  \n",
       "010eb14d3  13.792227  14.027738  14.297619  14.354401  \n",
       "011318064   3.618027   3.526598   3.630001   3.662966  \n",
       "\n",
       "[5 rows x 1000 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_train.to_hdf(\"savedata/TF1/X_train.hd5\", key=\"stage\", mode='w')\n",
    "# X_test.to_hdf(\"savedata/TF1/X_test.hd5\", key=\"stage\", mode='w')\n",
    "y_train_all_classes.to_hdf(\"savedata/TF1/TP_y_train.hd5\", key=\"stage\", mode='w')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}