train_models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from training_methods import train_model\n",
    "from data_gen import *\n",
    "from argparse import Namespace\n",
    "from DLGN_enums import *\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np\n",
    "import torch\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "use_dataset = True\n",
    "if use_dataset == False:\n",
    "    torch.manual_seed(42)\n",
    "\n",
    "    dim_in = 10\n",
    "    tree_depth = 3\n",
    "    num_points = 10000\n",
    "\n",
    "    X,Y = gen_spherical_data(depth=tree_depth, dim_in=dim_in, type_data='spherical', num_points=num_points, feat_index_start=0,radius=1)\n",
    "\n",
    "\n",
    "    x_train, x_test, y_train, y_test = train_test_split(\n",
    "        X, Y, test_size=0.1, random_state=42, stratify=Y\n",
    "    )\n",
    "\n",
    "    data_config = {\n",
    "        'dim_in': dim_in,\n",
    "        'tree_depth': tree_depth,\n",
    "        'num_points': num_points,\n",
    "        'type_data': 'spherical',\n",
    "        'feat_index_start': 0,\n",
    "        'radius': 1\n",
    "    }\n",
    "    data_config = Namespace(**data_config)\n",
    "    data = {}\n",
    "    data['train_data'] = x_train\n",
    "    data['train_labels'] = y_train \n",
    "    data['test_data'] = x_test\n",
    "    data['test_labels'] = y_test\n",
    "\n",
    "else:\n",
    "    DATA_DIR = 'data/dataset2'\n",
    "    data = {}\n",
    "    data['train_data'] = torch.tensor(np.load(DATA_DIR + '/x_train.npy'))\n",
    "    data['train_labels'] = torch.tensor(np.load(DATA_DIR + '/y_train.npy'))\n",
    "    data['test_data'] = torch.tensor(np.load(DATA_DIR + '/x_test.npy'))\n",
    "    data['test_labels'] = torch.tensor(np.load(DATA_DIR + '/y_test.npy'))\n",
    "    data_config = np.load(DATA_DIR + '/config.npy', allow_pickle=True).item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device('cuda:1' if torch.cuda.is_available() else 'cpu')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = {\n",
    "    'device' : device,\n",
    "    'model_type' : ModelTypes.VN,\n",
    "    'num_data' : len(data['train_data']),\n",
    "    'dim_in' : data_config.dim_in,\n",
    "    'num_hidden_nodes' : [500]*4, \n",
    "    'beta' : 10,\n",
    "    'loss_fn_type' : LossTypes.CE,\n",
    "    'optimizer_type' : Optim.ADAM,\n",
    "    'mode' : 'pwc',\n",
    "    'lr_ratio' : 10000,\n",
    "    'log_features' : False,\n",
    "    'lr' : 0.001,\n",
    "    'epochs' : 1000,\n",
    "    'use_wandb' : False\n",
    "}\n",
    "config = Namespace(**config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = {\n",
    "    'device' : device,\n",
    "    'model_type' : ModelTypes.VT,\n",
    "    'num_data' : len(data['train_data']),\n",
    "    'dim_in' : data_config.dim_in,\n",
    "    'num_hidden_nodes' : [8]*4,\n",
    "    'beta' : 30,\n",
    "    'mode' : 'pwc',\n",
    "    'value_scale' : 100,\n",
    "    'BN' : False,\n",
    "    'prod' : 'op',\n",
    "    'feat' : 'sf', \n",
    "    'loss_fn_type' : LossTypes.CE,\n",
    "    'optimizer_type' : Optim.ADAM,\n",
    "    'epochs' : 200,\n",
    "    'save_freq' : 100,\n",
    "    'value_freq' : 100,\n",
    "    'lr': 0.001,\n",
    "    'vt_fit' : VtFit.LOGISTIC,\n",
    "    'reg' : 1,\n",
    "    'use_wandb' : False,\n",
    "}\n",
    "config = Namespace(**config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = {\n",
    "    'device' : device,\n",
    "    'model_type' : ModelTypes.KERNEL,\n",
    "    'num_data' : len(data['train_data']),\n",
    "    'dim_in' : data_config.dim_in,\n",
    "    'width' : 128,\n",
    "    'depth' : 4,\n",
    "    'beta' : 30,\n",
    "    'alpha_init' : None,\n",
    "    'BN' : True,\n",
    "    'feat' : 'cf',\n",
    "    'weight_decay' : 0.01,\n",
    "    'train_method' : KernelTrainMethod.PEGASOS,\n",
    "    'reg' : 0.001,\n",
    "    'loss_fn_type' : LossTypes.HINGE,\n",
    "    'optimizer_type' : Optim.ADAM,\n",
    "    'log_features' : False,\n",
    "    'gates_lr' : 0.01,\n",
    "    'alpha_lr' : 0.1,\n",
    "    'epochs' : 1000,\n",
    "    'value_freq': 100,\n",
    "    'num_iter' : 50000,\n",
    "    'threshold' : 0.3,\n",
    "    'use_wandb' : False,\n",
    "}\n",
    "config = Namespace(**config)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/201 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.48615\n",
      "Loss before updating alphas at epoch 0  is  0.6984536426544189\n",
      "Time taken to fit value net:  129.04826164245605\n",
      "Accuracy:  0.76735\n",
      "Test Accuracy:  0.63775\n",
      "Loss after updating value_net at epoch 0  is  0.4773755151748657\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss 0.450010:  50%|████▉     | 100/201 [02:54<00:42,  2.37it/s] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.7864\n",
      "Loss before updating alphas at epoch 100  is  0.4500099825382233\n",
      "Time taken to fit value net:  126.23777484893799\n",
      "Accuracy:  0.811\n",
      "Test Accuracy:  0.6739\n",
      "Loss after updating value_net at epoch 100  is  0.4118704149723053\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss 0.392677: 100%|█████████▉| 200/201 [05:44<00:00,  2.33it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.8194\n",
      "Loss before updating alphas at epoch 200  is  0.3926772129535675\n",
      "Time taken to fit value net:  106.48081302642822\n",
      "Accuracy:  0.83595\n",
      "Test Accuracy:  0.6907\n",
      "Loss after updating value_net at epoch 200  is  0.3706959485054016\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss 0.361747: 100%|██████████| 201/201 [07:32<00:00,  2.25s/it]\n"
     ]
    }
   ],
   "source": [
    "model = train_model(data,config)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "env1",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}