gen_synthetic.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[16:21:20] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "baseline scores 0.8943365764540936 0.9291957270239252 0.7989745284027456\n"
     ]
    }
   ],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import sys\n",
    "sys.path.append('/home/rishibalakrishnan/DECAF/tests')\n",
    "\n",
    "from typing import Tuple\n",
    "\n",
    "import networkx as nx\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pytest\n",
    "import pytorch_lightning as pl\n",
    "import torch\n",
    "from sklearn.metrics import precision_score, recall_score, accuracy_score, roc_auc_score\n",
    "from tests.utils import gen_data_nonlinear, load_adult\n",
    "from xgboost import XGBClassifier\n",
    "import random\n",
    "\n",
    "from decaf import DECAF, DataModule\n",
    "import pandas as pd\n",
    "from opacus import PrivacyEngine\n",
    "from tqdm.notebook import tqdm\n",
    "from opacus.validators import ModuleValidator\n",
    "from opacus.data_loader import shape_safe\n",
    "\n",
    "torch.manual_seed(0)\n",
    "random.seed(0)\n",
    "np.random.seed(0)\n",
    "\n",
    "\n",
    "df, dag_seed, feat_num = load_adult()\n",
    "X, y = df[:, :-1], df[:, -1]\n",
    "# v_prop = .2\n",
    "# v_len = int(len(total)*v_prop)\n",
    "# np.random.shuffle(total)\n",
    "# val, train = total[:v_len], total[v_len:]\n",
    "# X, y = train[:, :-1], train[:, -1]\n",
    "# X, y = total[:, :-1], total[:, -1]\n",
    "\n",
    "baseline_clf = XGBClassifier(use_label_encoder=False).fit(X, y)\n",
    "y_pred = baseline_clf.predict(X)\n",
    "\n",
    "print(\n",
    "    \"baseline scores\",\n",
    "    precision_score(y, y_pred),\n",
    "    recall_score(y, y_pred),\n",
    "    roc_auc_score(y, y_pred),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import decaf\n",
    "import importlib\n",
    "importlib.reload(decaf)\n",
    "from decaf import DECAF, DataModule"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/rbala/DECAF/venv/lib/python3.8/site-packages/pytorch_lightning/core/datamodule.py:175: LightningDeprecationWarning: DataModule property `dims` was deprecated in v1.5 and will be removed in v1.7.\n",
      "  rank_zero_deprecation(\"DataModule property `dims` was deprecated in v1.5 and will be removed in v1.7.\")\n",
      "GPU available: True, used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [2,7]\n",
      "\n",
      "  | Name          | Type             | Params\n",
      "---------------------------------------------------\n",
      "0 | generator     | Generator_causal | 651 K \n",
      "1 | discriminator | Discriminator    | 57.6 K\n",
      "---------------------------------------------------\n",
      "708 K     Trainable params\n",
      "1.1 K     Non-trainable params\n",
      "709 K     Total params\n",
      "2.837     Total estimated model params size (MB)\n",
      "/home/rbala/DECAF/venv/lib/python3.8/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:631: UserWarning: Checkpoint directory /home/rbala/DECAF/checkpoints exists and is not empty.\n",
      "  rank_zero_warn(f\"Checkpoint directory {dirpath} exists and is not empty.\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialised adjacency matrix as parsed:\n",
      " Parameter containing:\n",
      "tensor([[0., 0., 0., 0., 1., 1., 1., 1., 0., 0., 1.],\n",
      "        [0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1.],\n",
      "        [0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1.],\n",
      "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
      "        [0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1.],\n",
      "        [0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]])\n",
      "Epoch 49: 100%|██████████| 473/473 [00:35<00:00, 13.33it/s, loss=0.061, precision=0.943, recall=0.377, auroc=0.654]    \n"
     ]
    }
   ],
   "source": [
    "#Total list of changes that I've made\n",
    "#Using y in the model too\n",
    "#Giving seed for DAG\n",
    "\n",
    "total, dag_seed, feat_num = load_adult()\n",
    "v_prop = .2\n",
    "v_len = int(len(total)*v_prop)\n",
    "np.random.shuffle(total)\n",
    "val, train = total[:v_len], total[v_len:]\n",
    "X, y = train[:, :-1], train[:, -1]\n",
    "\n",
    "dm = DataModule(train, batch_size=64, val_data=val, num_workers=32)\n",
    "\n",
    "model = DECAF(\n",
    "    len(feat_num),\n",
    "    use_mask=True,\n",
    "    dag_seed=dag_seed,\n",
    "    grad_dag_loss=False,\n",
    "    lambda_privacy=0,\n",
    "    lambda_gp=10,\n",
    "    weight_decay=1e-2,\n",
    "    l1_g=0,\n",
    "    p_gen=-1,\n",
    "    d_updates=10,\n",
    "    batch_size=100,\n",
    "    feat_num=feat_num\n",
    ")\n",
    "\n",
    "trainer = pl.Trainer(max_epochs=50, logger=False, gpus=1)\n",
    "trainer.fit(model, dm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/rishibalakrishnan/.local/lib/python3.7/site-packages/pytorch_lightning/core/datamodule.py:170: LightningDeprecationWarning: DataModule property `dims` was deprecated in v1.5 and will be removed in v1.7.\n",
      "  rank_zero_deprecation(\"DataModule property `dims` was deprecated in v1.5 and will be removed in v1.7.\")\n",
      "GPU available: True, used: False\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "/home/rishibalakrishnan/.local/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py:1567: UserWarning: GPU available but not used. Set the gpus flag in your trainer `Trainer(gpus=1)` or script `--gpus=1`.\n",
      "  \"GPU available but not used. Set the gpus flag in your trainer `Trainer(gpus=1)` or script `--gpus=1`.\"\n",
      "/home/rishibalakrishnan/.local/lib/python3.7/site-packages/pytorch_lightning/trainer/configuration_validator.py:116: UserWarning: You passed in a `val_dataloader` but have no `validation_step`. Skipping val loop.\n",
      "  rank_zero_warn(\"You passed in a `val_dataloader` but have no `validation_step`. Skipping val loop.\")\n",
      "\n",
      "  | Name          | Type             | Params\n",
      "---------------------------------------------------\n",
      "0 | generator     | Generator_causal | 128 K \n",
      "1 | discriminator | Discriminator    | 43.4 K\n",
      "---------------------------------------------------\n",
      "171 K     Trainable params\n",
      "0         Non-trainable params\n",
      "171 K     Total params\n",
      "0.686     Total estimated model params size (MB)\n",
      "/home/rishibalakrishnan/.local/lib/python3.7/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:617: UserWarning: Checkpoint directory /srv/home/rishibalakrishnan/DECAF/checkpoints exists and is not empty.\n",
      "  rank_zero_warn(f\"Checkpoint directory {dirpath} exists and is not empty.\")\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "023ab1b5a85140fdb1fbc1520d964b94",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Total list of changes that I've made\n",
    "#Using y in the model too\n",
    "#Giving seed for DAG\n",
    "#d updates at 10 instead of 5\n",
    "\n",
    "X, y = load_adult()\n",
    "# v_prop = .2\n",
    "# v_len = int(len(total)*v_prop)\n",
    "# np.random.shuffle(total)\n",
    "# val, train = total[:v_len], total[v_len:]\n",
    "# X, y = train[:, :-1], train[:, -1]\n",
    "\n",
    "dm = DataModule(X, num_workers=56)\n",
    "\n",
    "model = DECAF(\n",
    "    dm.dims[0],\n",
    "    use_mask=True,\n",
    "    grad_dag_loss=False,\n",
    "    lambda_privacy=0,\n",
    "    lambda_gp=10,\n",
    "    weight_decay=1e-2,\n",
    "    l1_g=0,\n",
    "    p_gen=-1,\n",
    "    batch_size=100,\n",
    ")\n",
    "trainer = pl.Trainer(max_epochs=10, logger=False)\n",
    "trainer.fit(model, dm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/rishibalakrishnan/.local/lib/python3.7/site-packages/pytorch_lightning/core/datamodule.py:175: LightningDeprecationWarning: DataModule property `dims` was deprecated in v1.5 and will be removed in v1.7.\n",
      "  rank_zero_deprecation(\"DataModule property `dims` was deprecated in v1.5 and will be removed in v1.7.\")\n",
      "GPU available: True, used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]\n",
      "\n",
      "  | Name          | Type             | Params\n",
      "---------------------------------------------------\n",
      "0 | generator     | Generator_causal | 289 K \n",
      "1 | discriminator | GradSampleModule | 57.6 K\n",
      "---------------------------------------------------\n",
      "346 K     Trainable params\n",
      "1.1 K     Non-trainable params\n",
      "347 K     Total params\n",
      "1.390     Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialised adjacency matrix as parsed:\n",
      " Parameter containing:\n",
      "tensor([[0., 0., 0., 0., 1., 1., 1., 1., 0., 0., 1.],\n",
      "        [0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1.],\n",
      "        [0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1.],\n",
      "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
      "        [0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1.],\n",
      "        [0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]])\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/rishibalakrishnan/.local/lib/python3.7/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:617: UserWarning: Checkpoint directory /srv/home/rishibalakrishnan/DECAF/checkpoints exists and is not empty.\n",
      "  rank_zero_warn(f\"Checkpoint directory {dirpath} exists and is not empty.\")\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2df21587e2bf47e7955c6d5df106c438",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation sanity check: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/rishibalakrishnan/.local/lib/python3.7/site-packages/pytorch_lightning/trainer/data_loading.py:111: UserWarning: The dataloader, val_dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 56 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  f\"The dataloader, {name}, does not have many workers which may be a bottleneck.\"\n",
      "/home/rishibalakrishnan/.local/lib/python3.7/site-packages/pytorch_lightning/trainer/data_loading.py:111: UserWarning: The dataloader, train_dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 56 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  f\"The dataloader, {name}, does not have many workers which may be a bottleneck.\"\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f77213d57cd84d3599c0a58005c366a4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/rishibalakrishnan/.local/lib/python3.7/site-packages/torch/nn/modules/module.py:795: UserWarning: Using a non-full backward hook when the forward contains multiple autograd Nodes is deprecated and will be removed in future versions. This hook will be missing some grad_input. Please use register_full_backward_hook to get the documented behavior.\n",
      "  warnings.warn(\"Using a non-full backward hook when the forward contains multiple autograd Nodes \"\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "95f30fc6e2d94b15a739e0bfa0d045be",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a43898b119d147b780807d5e029c7e27",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8f55bb253e5f4191b9ec231e7d42f983",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dc773077ceb94c7f91496486c02ce006",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cd2933954acc48cbb06787b61462213e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1cb5e8b04cad4663a7200afe4ab829a0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1f4a810aef754ec7ae8f78751d24bb7b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6a38b9bfea604ee8aabd22308244d200",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e08f01963fe046b7bc5b07026a0c322e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5bfbcb72911e4d35a7b40aff6f8b7a5d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9d71bc092856481e865776dfba591730",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/rishibalakrishnan/.local/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py:685: UserWarning: Detected KeyboardInterrupt, attempting graceful shutdown...\n",
      "  rank_zero_warn(\"Detected KeyboardInterrupt, attempting graceful shutdown...\")\n"
     ]
    }
   ],
   "source": [
    "# total, dag_seed, feat_num = load_adult()\n",
    "# v_prop = .2\n",
    "# v_len = int(len(total)*v_prop)\n",
    "# np.random.shuffle(total)\n",
    "# val, train = total[:v_len], total[v_len:]\n",
    "# X, y = train[:, :-1], train[:, -1]\n",
    "\n",
    "batch_size = 64\n",
    "sample_rate = batch_size / len(train)\n",
    "noise_multiplier = .5\n",
    "max_grad_norm = 2\n",
    "\n",
    "dm = DataModule(train, val, num_workers=0, batch_size=batch_size, enable_dp=True)\n",
    "\n",
    "model = DECAF(\n",
    "    len(feat_num),\n",
    "    use_mask=True,\n",
    "    dag_seed=dag_seed,\n",
    "    grad_dag_loss=False,\n",
    "    lambda_privacy=0,\n",
    "    lambda_gp=10,\n",
    "    weight_decay=1e-2,\n",
    "    l1_g=1,\n",
    "    p_gen=-1,\n",
    "    d_updates=10,\n",
    "    noise_multiplier=noise_multiplier,\n",
    "    sample_rate=sample_rate,\n",
    "    max_grad_norm = max_grad_norm,\n",
    "    batch_size=batch_size,\n",
    "    enable_dp=True, \n",
    "    feat_num=feat_num\n",
    ")\n",
    "\n",
    "trainer = pl.Trainer(max_epochs=50, logger=False, gpus=1)\n",
    "trainer.fit(model, dm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generated Y - fraction 1's: 0.7483630335681724\n",
      "Trained on real data - predicting on synthetic 0.8191362813550556 0.8770627976520102 0.6505702657561764\n",
      "[16:52:38] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "Trained on synthetic data - predicting on real 0.9410081443979749 0.47232350016572755 0.6917025596382719\n",
      "[16:52:57] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "Real Data on holdout set: 0.8782238766285563 0.9076669414674361 0.7609101345921848\n",
      "Synth Data on holdout set: 0.9444752915047196 0.4674361088211047 0.6915950552530123\n"
     ]
    }
   ],
   "source": [
    "# print(\"Generated synthetic data\")\n",
    "device = torch.device('cuda')\n",
    "model.to(device)\n",
    "# total_synth = model.generator.sequential(\n",
    "#     dm.dataset.x.to(device),\n",
    "#     model.sample_z(dm.dataset.x.shape[0]).type_as(dm.dataset.x).to(device),\n",
    "#     gen_order=model.get_gen_order(),\n",
    "#     biased_edges=[],\n",
    "# ).detach().cpu().numpy()\n",
    "total_synth = (\n",
    "    model.gen_synthetic(\n",
    "        dm.train_dataset.x.to(device),\n",
    "        gen_order=model.get_gen_order(),\n",
    "    )\n",
    "    .detach()\n",
    "    .cpu()\n",
    "    .numpy()\n",
    ")\n",
    "\n",
    "X_synth, y_synth = total_synth[:, :-1], total_synth[:, -1]\n",
    "\n",
    "y_synth = np.round(y_synth, 0).astype(np.uint8)\n",
    "\n",
    "# y_synth = baseline_clf.predict(X_synth) #Using the model fitted on the real data to produce predictions\n",
    "# #I guess these predictions are all one class?\n",
    "print(f\"Generated Y - fraction 1's: {sum(y_synth)/len(y_synth)}\")\n",
    "\n",
    "y_pred = baseline_clf.predict(X_synth)\n",
    "# y_pred = synth_clf.predict(X_synth)\n",
    "\n",
    "print(\n",
    "    \"Trained on real data - predicting on synthetic\",\n",
    "    precision_score(y_synth, y_pred),\n",
    "    recall_score(y_synth, y_pred),\n",
    "    roc_auc_score(y_synth, y_pred),\n",
    ")\n",
    "\n",
    "#Train model only on synthetic data and then see how it performs to actual\n",
    "synth_clf = XGBClassifier(use_label_encoder=False).fit(X_synth, y_synth)\n",
    "y_pred = synth_clf.predict(X)\n",
    "print(\n",
    "    \"Trained on synthetic data - predicting on real\",\n",
    "    precision_score(y, y_pred),\n",
    "    recall_score(y, y_pred),\n",
    "    roc_auc_score(y, y_pred),\n",
    ")\n",
    "\n",
    "#Holdout set\n",
    "prop = .2\n",
    "length = int(prop*len(X))\n",
    "X_val, y_val = X[:length], y[:length]\n",
    "X_train, y_train = X[length:], y[length:]\n",
    "real_clf = XGBClassifier(use_label_encoder=False).fit(X_train, y_train)\n",
    "\n",
    "real_pred = real_clf.predict(X_val)\n",
    "print(\"Real Data on holdout set:\",\n",
    "    precision_score(y_val, real_pred),\n",
    "    recall_score(y_val, real_pred),\n",
    "    roc_auc_score(y_val, real_pred),\n",
    ")\n",
    "\n",
    "synth_pred = synth_clf.predict(X_val)\n",
    "print(\"Synth Data on holdout set:\",\n",
    "    precision_score(y_val, synth_pred),\n",
    "    recall_score(y_val, synth_pred),\n",
    "    roc_auc_score(y_val, synth_pred),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/rishibalakrishnan/.local/lib/python3.7/site-packages/pytorch_lightning/core/datamodule.py:175: LightningDeprecationWarning: DataModule property `dims` was deprecated in v1.5 and will be removed in v1.7.\n",
      "  rank_zero_deprecation(\"DataModule property `dims` was deprecated in v1.5 and will be removed in v1.7.\")\n",
      "GPU available: True, used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]\n",
      "\n",
      "  | Name          | Type             | Params\n",
      "---------------------------------------------------\n",
      "0 | generator     | Generator_causal | 289 K \n",
      "1 | discriminator | Discriminator    | 57.6 K\n",
      "---------------------------------------------------\n",
      "346 K     Trainable params\n",
      "1.1 K     Non-trainable params\n",
      "347 K     Total params\n",
      "1.390     Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialised adjacency matrix as parsed:\n",
      " Parameter containing:\n",
      "tensor([[0., 0., 0., 0., 1., 1., 1., 1., 0., 0., 1.],\n",
      "        [0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1.],\n",
      "        [0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1.],\n",
      "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
      "        [0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1.],\n",
      "        [0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]])\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/rishibalakrishnan/.local/lib/python3.7/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:617: UserWarning: Checkpoint directory /srv/home/rishibalakrishnan/DECAF/checkpoints exists and is not empty.\n",
      "  rank_zero_warn(f\"Checkpoint directory {dirpath} exists and is not empty.\")\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b18553ec8dc24c72a3fb983f1eadbbef",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation sanity check: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c3e38d00d33140f88a8d796249ef4f5d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2a9be00ea1c44110b669dd470815822c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b4735a619c1741e8be8342e553656e7b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Total list of changes that I've made\n",
    "#Using y in the model too\n",
    "#Giving seed for DAG\n",
    "\n",
    "total, dag_seed, feat_num = load_adult()\n",
    "v_prop = .2\n",
    "v_len = int(len(total)*v_prop)\n",
    "np.random.shuffle(total)\n",
    "val, train = total[:v_len], total[v_len:]\n",
    "X, y = train[:, :-1], train[:, -1]\n",
    "\n",
    "dm = DataModule(train, batch_size=64, valid_data=val, num_workers=56)\n",
    "\n",
    "model = DECAF(\n",
    "    len(feat_num),\n",
    "    use_mask=True,\n",
    "    dag_seed=dag_seed,\n",
    "    grad_dag_loss=False,\n",
    "    lambda_privacy=0,\n",
    "    lambda_gp=0,\n",
    "    weight_decay=1e-2,\n",
    "    l1_g=0,\n",
    "    p_gen=-1,\n",
    "    d_updates=10,\n",
    "    batch_size=100,\n",
    "    feat_num=feat_num\n",
    ")\n",
    "\n",
    "trainer = pl.Trainer(max_epochs=50, logger=False, gpus=1)\n",
    "trainer.fit(model, dm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch import nn\n",
    "h_dim = 200\n",
    "ml = nn.ModuleList([nn.Sequential(\n",
    "                nn.Linear(h_dim, h_dim),\n",
    "                nn.ReLU(inplace=True),\n",
    "                nn.Linear(h_dim, 2)\n",
    "            ) \n",
    "            for i in range(5)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([9.99695897e-01, 2.17938083e-16, 1.34072951e-28, 2.73538471e-21,\n",
       "       5.48444431e-15, 7.62095332e-01, 9.99686360e-01, 9.98649299e-01,\n",
       "       4.39007199e-05, 3.77879222e-03, 9.79912118e-04, 6.31651492e-04,\n",
       "       1.31357185e-04, 6.44884247e-04, 1.84138201e-03, 1.17452955e-05,\n",
       "       3.46159941e-05, 1.24797898e-05, 8.97080824e-03, 1.89770130e-03,\n",
       "       3.03674133e-05, 9.39239617e-05, 2.42579915e-03, 2.19374849e-03,\n",
       "       1.04936818e-03, 5.97884785e-03, 3.37162381e-03, 1.76287447e-08,\n",
       "       3.39009639e-05, 1.57756207e-04, 1.49013812e-03, 1.22399665e-02,\n",
       "       1.57482827e-05, 1.72681571e-03, 4.76889654e-05, 1.33217103e-03,\n",
       "       2.01368774e-03, 1.39350025e-03, 1.13616064e-04, 1.14452050e-04,\n",
       "       6.21793297e-05, 6.14365053e-05, 5.08206249e-05, 3.21408720e-07,\n",
       "       1.13838381e-04, 6.61843037e-03, 4.20357464e-05, 2.15125736e-04,\n",
       "       1.00000000e+00, 0.00000000e+00, 8.85730128e-26, 5.63101136e-33,\n",
       "       0.00000000e+00, 3.82171960e-18, 4.28199126e-15, 5.26398480e-01,\n",
       "       0.00000000e+00, 9.89841878e-01, 2.59725818e-15, 3.69826558e-10,\n",
       "       6.33460840e-14, 1.58549399e-29, 9.36332500e-39, 4.02830392e-06,\n",
       "       0.00000000e+00, 8.43598361e-17, 1.35630331e-13, 1.43161404e-23,\n",
       "       2.65474203e-14, 1.63363694e-18, 5.08420527e-01, 1.00000000e+00,\n",
       "       0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
       "       0.00000000e+00, 0.00000000e+00, 3.40490851e-06, 2.27174684e-30,\n",
       "       9.99768555e-01, 9.97215022e-09, 0.00000000e+00, 0.00000000e+00,\n",
       "       6.35395586e-01], dtype=float32)"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_synth[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialised adjacency matrix as parsed:\n",
      " Parameter containing:\n",
      "tensor([[0., 0., 0., 0., 1., 1., 1., 1., 0., 0., 1.],\n",
      "        [0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1.],\n",
      "        [0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1.],\n",
      "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
      "        [0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1.],\n",
      "        [0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]])\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/rishibalakrishnan/.local/lib/python3.7/site-packages/pytorch_lightning/core/datamodule.py:175: LightningDeprecationWarning: DataModule property `dims` was deprecated in v1.5 and will be removed in v1.7.\n",
      "  rank_zero_deprecation(\"DataModule property `dims` was deprecated in v1.5 and will be removed in v1.7.\")\n",
      "/home/rishibalakrishnan/.local/lib/python3.7/site-packages/pytorch_lightning/core/datamodule.py:170: LightningDeprecationWarning: DataModule property `dims` was deprecated in v1.5 and will be removed in v1.7.\n",
      "  rank_zero_deprecation(\"DataModule property `dims` was deprecated in v1.5 and will be removed in v1.7.\")\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2f719b7f618e4d4983a8448ff898d0b8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1dfc7479a0464029ac45b240b6b96655",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ac886ffd7f7d43b2bdc0b74e7f745d97",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ee6745ecd4884d268391fe5c5defdcda",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "91915d557f53463f869cd69b44a6ac3c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5214e8fd65704c0b8d84be9484aaf1dd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d53cd0265e59436dabac2a3fa8fac790",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b6e28ff23fd7499599342854ebe6af34",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "94dedd74b6924d6db22b35be34d5ee66",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "04ae170640494e2a9fa376e3973f12b0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6c38297ad5904b27af79321928f30e6a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3cf5813627704c4caf3a0a9eb52dfa8f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0fd97a8896de45018f00fefb0d7f9950",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9b9820fd56344d609cbb8ae9b2033db3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1a070a900d3d44379c46a9caf45b47d9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a3186d667cdf461db3c74dfab064b7fb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "007b5016573e438f85659ee58905d11a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ccc82cc7037c40aa975d1ceb796a9c6c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3f05c2a4ca994a75bc4784edaff02662",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8cf26e86b22b40a4afa9c03f8b8cf76e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "53fd6b004f6f4b98b1ffb31f1279f35d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "79ffa5433e1a4ff38c06908b2a833d99",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3a0f5d16c8144d5793bd5ff24e34c33d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "156b3cd0e7d04369a0d1189e6a5fb530",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0f05ef5646b24050acac9b7995861fe1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ce09d35e03914ea5a793ff6d345c5835",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "645f149f136543779dc6cdc3fa54bfbd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "540d654d55e943238c8edb3d42272330",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e116b9768b1f4316b89ce83fbd121067",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "99ebfe6ffea74feb87ef0bae4e107780",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bd466609fc4746678df532828d754c79",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "78148ca5907848f5a5b4f011b588ccc1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f709d9d14bca4e8182990097bac2070f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6d9e39aa1f5645689632ad2784bdd704",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1c0b43fb23a4474488633a486505f11d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "19fa8f3d7eb94079a40278f1afe2b445",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c70e28254c064e53b669a68f1cc4aa65",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3a5cc1eec18a494eb5b8c8b3c625aa3d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b615416c7b1e48e1bf456dc3c63d775c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8cfff5ee059b4404be2ee65384ef44c2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eba981a2bcda4e4caa183f5aa62c8185",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0dd28b8019c74ec49599b4f5ef8ee90a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2cbf88e212e043ae938e63c24c1b7ac9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "485813c573794cffa58dd564b719f9b3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e3a78ddb8b904bc8904e96fadff61a9a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fe23ae9df3c046eca7ee052a59ce4c1a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "71203fd198b849c68f6eef1bc12c881f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b0bb9f48cf834c4eb6ba118fd58f7aa4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "57d3358c4b6a4df6a4a1523ba41b9d43",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f6152baf60f94287a8e1fc7e37727fab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "batch_size = 64\n",
    "sampling_rate = batch_size / len(total)\n",
    "noise_multiplier=1.1\n",
    "max_grad_norm=2.\n",
    "delta=1e-5\n",
    "\n",
    "dm = DataModule(total, num_workers=0, batch_size=batch_size)\n",
    "\n",
    "device = torch.device('cuda')\n",
    "\n",
    "model = DECAF(\n",
    "    dm.dims[0],\n",
    "    use_mask=True,\n",
    "    dag_seed=dag_seed,\n",
    "    grad_dag_loss=False,\n",
    "    lambda_privacy=0,\n",
    "    lambda_gp=10,\n",
    "    weight_decay=1e-2,\n",
    "    l1_g=0,\n",
    "    p_gen=-1,\n",
    "    d_updates=10,\n",
    ")\n",
    "\n",
    "dict_opt = model.configure_optimizers()\n",
    "d_opt = dict_opt[0][\"optimizer\"]\n",
    "d_iter = dict_opt[0][\"frequency\"]\n",
    "\n",
    "g_opt = dict_opt[1][\"optimizer\"]\n",
    "g_iter = dict_opt[1][\"frequency\"]\n",
    "\n",
    "# train_dataloader = dm.train_dataloader()\n",
    "\n",
    "# privacy_engine = PrivacyEngine()\n",
    "\n",
    "# discriminator, d_opt, train_dataloader = privacy_engine.make_private(module=model.discriminator, \n",
    "#                                                             optimizer=d_opt, \n",
    "#                                                             data_loader=train_dataloader, \n",
    "#                                                             noise_multiplier=noise_multiplier, \n",
    "#                                                             max_grad_norm=max_grad_norm) \n",
    "# model.discriminator = discriminator\n",
    "\n",
    "model.cuda()\n",
    "model.train()\n",
    "torch.set_grad_enabled(True)\n",
    "\n",
    "epochs = 50\n",
    "for i in range(epochs):\n",
    "    data_bar = tqdm(train_dataloader)\n",
    "    for idx, batch in enumerate(data_bar):\n",
    "        opt_idx = int(idx % (g_iter + d_iter) == d_iter)\n",
    "\n",
    "        batch = batch.cuda()\n",
    "        d_opt.zero_grad()\n",
    "\n",
    "        z = model.sample_z(batch.shape[0]).type_as(batch).to(device)\n",
    "\n",
    "        generated_batch = model.generator.sequential(batch, z, model.get_gen_order())\n",
    "\n",
    "        if opt_idx == 0:\n",
    "            fake_loss = torch.mean(model.discriminator(generated_batch.detach()))\n",
    "            real_loss = torch.mean(model.discriminator(batch))\n",
    "            gp = model.hparams.lambda_gp * model.compute_gradient_penalty(batch, generated_batch)\n",
    "            loss = fake_loss - real_loss + gp\n",
    "\n",
    "            loss.backward()\n",
    "            d_opt.step()\n",
    "            d_opt.zero_grad()\n",
    "\n",
    "        # model.discriminator.remove_hooks()\n",
    "        # gp = model.hparams.lambda_gp * model.compute_gradient_penalty(batch, generated_batch)\n",
    "        # gp.backward(create_graph=True, retain_graph=True)\n",
    "        # model.discriminator.add_hooks()\n",
    "\n",
    "        #Train discriminator\n",
    "        # fake_output = model.discriminator(generated_batch.detach())\n",
    "        # fake_loss = torch.mean(fake_output)\n",
    "        # fake_loss.backward()\n",
    "        # d_opt.step()\n",
    "        # d_opt.zero_grad()\n",
    "\n",
    "        # real_output = model.discriminator(batch)\n",
    "        # real_loss = - torch.mean(real_output)\n",
    "        # real_loss.backward()\n",
    "\n",
    "        # d_opt.step()\n",
    "        # d_opt.zero_grad()\n",
    "\n",
    "       \n",
    "            disc_loss = fake_loss.detach() - real_loss.detach()\n",
    "            data_bar.set_description(f\"epoch: {i}, Loss: {disc_loss.item():.2f}, GP: {gp.detach().item():.2f}\")\n",
    "\n",
    "        else:\n",
    "            #Train generator\n",
    "            g_opt.zero_grad()\n",
    "            g_output = model.discriminator(generated_batch)\n",
    "            g_loss = - torch.mean(g_output)\n",
    "            g_loss.backward()\n",
    "            g_opt.step()\n",
    "            data_bar.set_description(f\"epoch: {i}, Loss: {g_loss.detach().item():.2f}\")\n",
    "    # eps = privacy_engine.get_epsilon(1e-5)\n",
    "    # print(f\"EPOCH {i} EPSILON {eps}\")\n",
    "\n",
    "# for i in range(epochs):\n",
    "#     for idx, batch in tqdm(enumerate(train_dataloader)):\n",
    "#         output = model.training_step(batch, idx, 0)\n",
    "#         loss = output[\"loss\"]\n",
    "#         loss.backward()\n",
    "#         d_opt.step()\n",
    "#         d_opt.zero_grad()\n",
    "        \n",
    "\n",
    "#         # if idx % d_iter == (d_iter - 1):\n",
    "#         # if idx % 5 == 0:\n",
    "#         output = model.training_step(batch, idx, 1)\n",
    "#         loss = output[\"loss\"]\n",
    "#         loss.backward()\n",
    "#         g_opt.step()\n",
    "#         g_opt.zero_grad()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "import decaf\n",
    "import importlib\n",
    "importlib.reload(decaf)\n",
    "from decaf import DECAF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'model' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_32033/1561000685.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mM\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'model' is not defined"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([85, 85])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.generator.M.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "synth_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "import decaf\n",
    "import importlib\n",
    "importlib.reload(decaf)\n",
    "from decaf import DECAF, DataModule"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'train' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_56764/1304302252.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mbatch_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m64\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0msample_rate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch_size\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mnoise_multiplier\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m.000000001\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mmax_grad_norm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m100000\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'train' is not defined"
     ]
    }
   ],
   "source": [
    "batch_size = 64\n",
    "sample_rate = batch_size / len(train)\n",
    "noise_multiplier = .5\n",
    "max_grad_norm = 1.1\n",
    "\n",
    "dm = DataModule(train, val, num_workers=56, batch_size=batch_size, enable_dp=True)\n",
    "\n",
    "model = DECAF(\n",
    "    dm.dims[0],\n",
    "    use_mask=True,\n",
    "    dag_seed=dag_seed,\n",
    "    grad_dag_loss=False,\n",
    "    lambda_privacy=0,\n",
    "    lambda_gp=10,\n",
    "    weight_decay=1e-2,\n",
    "    l1_g=0,\n",
    "    p_gen=-1,\n",
    "    d_updates=10,\n",
    "    noise_multiplier=noise_multiplier,\n",
    "    sample_rate=sample_rate,\n",
    "    max_grad_norm = max_grad_norm,\n",
    "    batch_size=batch_size,\n",
    "    enable_dp=True\n",
    ")\n",
    "\n",
    "trainer = pl.Trainer(max_epochs=50, logger=False, gpus=1)\n",
    "trainer.fit(model, dm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "The label must consist of integer labels of form 0, 1, 2, ..., [num_class - 1].",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_42522/2201398973.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0my_synth\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mround\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_synth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mclf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mXGBClassifier\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0muse_label_encoder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_synth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_synth\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/.local/lib/python3.7/site-packages/xgboost/core.py\u001b[0m in \u001b[0;36minner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    504\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marg\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    505\u001b[0m             \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marg\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 506\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    507\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    508\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0minner_f\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.7/site-packages/xgboost/sklearn.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight, base_margin, eval_set, eval_metric, early_stopping_rounds, verbose, xgb_model, sample_weight_eval_set, base_margin_eval_set, feature_weights, callbacks)\u001b[0m\n\u001b[1;32m   1197\u001b[0m                 \u001b[0;32mnot\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray_equal\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclasses_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_classes_\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1198\u001b[0m             ):\n\u001b[0;32m-> 1199\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabel_encoding_check_error\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1200\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1201\u001b[0m         \u001b[0mparams\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_xgb_params\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: The label must consist of integer labels of form 0, 1, 2, ..., [num_class - 1]."
     ]
    }
   ],
   "source": [
    "synth_data = model.gen_synthetic(torch.ones((30162, model.x_dim)).cuda()).detach().cpu().numpy()\n",
    "X_synth, y_synth = synth_data[:, :-1], synth_data[:, -1]\n",
    "y_synth = np.round(y_synth, 0)\n",
    "\n",
    "clf = XGBClassifier(use_label_encoder=False).fit(X_synth, y_synth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(y_synth)/len(y_synth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# y_synth = y_synth / np.max(y_synth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.        , 0.53424658, 1.        , 0.        , 2.        ,\n",
       "       0.86666667, 8.        , 0.40816327, 5.        , 3.        ,\n",
       "       1.        ])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "feat_num = {0:1, 1:5, 2:2, 3:4}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "previous_idx = 0\n",
    "feat_ranges = {}\n",
    "for idx in sorted(feat_num.keys()):\n",
    "    feat_ranges[idx] = np.arange(previous_idx, previous_idx + feat_num[idx])\n",
    "    previous_idx += feat_num[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dag_seed = [[0, 4], [1, 4], [2, 4], [3, 4], [0, 5], [4, 5], [1, 5], [2, 5], [3, 5], [0, 6], [1, 6], [2, 6], [4, 6], [5, 6], [0, 7], [1, 7], [4, 7], [3, 7], [2, 7], [5, 7], [1, 8], [4, 8], [2, 8], [5, 8], [3, 8], [4, 9], [5, 9], [1, 9], [2, 9], [3, 9], [0, 10], [1, 10], [2, 10], [3, 10], [4, 10], [5, 10], [6, 10], [7, 10], [8, 10], [9, 10]]\n",
    "\n",
    "M_init = torch.rand(11, 11) * 0.0\n",
    "for pair in dag_seed:\n",
    "    M_init[pair[0], pair[1]] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "feat_num = {0: 5, 1: 1, 2:1, 3: 41, 4: 7, 5: 1, 6:14, 7: 1, 8:7, 9:6, 10:1}\n",
    "oh_M = torch.empty((0, len(feat_num)))\n",
    "for idx in sorted(feat_num.keys()):\n",
    "    oh_idx = torch.unsqueeze(M_init[idx, :], 0)\n",
    "    oh_idx = torch.tile(oh_idx, (feat_num[idx], 1))\n",
    "    oh_M = torch.cat([oh_M, oh_idx], 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0., 0., 0., 0., 1., 1., 1., 1., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 1., 1., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 1., 1., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 1., 1., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 1., 1., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "oh_M"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 1., 1., 1., 1.],\n",
       "        [1., 1., 1., 1., 1.],\n",
       "        [1., 1., 1., 1., 1.],\n",
       "        [1., 1., 1., 1., 1.],\n",
       "        [1., 1., 1., 1., 1.],\n",
       "        [1., 1., 1., 1., 1.],\n",
       "        [1., 1., 1., 1., 1.],\n",
       "        [1., 1., 1., 1., 1.],\n",
       "        [1., 1., 1., 1., 1.],\n",
       "        [1., 1., 1., 1., 1.],\n",
       "        [0., 0., 0., 0., 0.]])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "M_fake = torch.empty((11, 0))\n",
    "a = torch.tile(torch.unsqueeze(M_init[:, -1], 1), (1, 5))\n",
    "torch.cat([M_fake, a], 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generated Y - fraction 1's: 0.7225979709568331\n",
      "Trained on real data - predicting on synthetic 0.721552386359841 0.9912365221381051 0.49741101832971946\n",
      "[08:34:24] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "Trained on synthetic data - predicting on real 0.7697882352941177 0.7220799858744592 0.5352541644875759\n",
      "[08:34:25] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "Real Data on holdout set: 0.8761497326203208 0.905393457117595 0.7607206012378426\n",
      "Synth Data on holdout set: 0.7665250236071766 0.7177276746242264 0.530946065428824\n"
     ]
    }
   ],
   "source": [
    "# print(\"Generated synthetic data\")\n",
    "device = torch.device('cuda')\n",
    "model.to(device)\n",
    "total_synth = (\n",
    "    model.gen_synthetic(\n",
    "        dm.dataset.x.to(device),\n",
    "        gen_order=model.get_gen_order(),\n",
    "    )\n",
    "    .detach()\n",
    "    .cpu()\n",
    "    .numpy()\n",
    ")\n",
    "\n",
    "X_synth, y_synth = total_synth[:, :-1], total_synth[:, -1]\n",
    "\n",
    "y_synth = np.round(y_synth, 0).astype(np.uint8)\n",
    "\n",
    "# y_synth = baseline_clf.predict(X_synth) #Using the model fitted on the real data to produce predictions\n",
    "# #I guess these predictions are all one class?\n",
    "print(f\"Generated Y - fraction 1's: {sum(y_synth)/len(y_synth)}\")\n",
    "\n",
    "y_pred = baseline_clf.predict(X_synth)\n",
    "# y_pred = synth_clf.predict(X_synth)\n",
    "\n",
    "print(\n",
    "    \"Trained on real data - predicting on synthetic\",\n",
    "    precision_score(y_synth, y_pred),\n",
    "    recall_score(y_synth, y_pred),\n",
    "    roc_auc_score(y_synth, y_pred),\n",
    ")\n",
    "\n",
    "#Train model only on synthetic data and then see how it performs to actual\n",
    "synth_clf = XGBClassifier(use_label_encoder=False).fit(X_synth, y_synth)\n",
    "y_pred = synth_clf.predict(X)\n",
    "print(\n",
    "    \"Trained on synthetic data - predicting on real\",\n",
    "    precision_score(y, y_pred),\n",
    "    recall_score(y, y_pred),\n",
    "    roc_auc_score(y, y_pred),\n",
    ")\n",
    "\n",
    "#Holdout set\n",
    "prop = .2\n",
    "length = int(prop*len(X))\n",
    "X_val, y_val = X[:length], y[:length]\n",
    "X_train, y_train = X[length:], y[length:]\n",
    "real_clf = XGBClassifier(use_label_encoder=False).fit(X_train, y_train)\n",
    "\n",
    "real_pred = real_clf.predict(X_val)\n",
    "print(\"Real Data on holdout set:\",\n",
    "    precision_score(y_val, real_pred),\n",
    "    recall_score(y_val, real_pred),\n",
    "    roc_auc_score(y_val, real_pred),\n",
    ")\n",
    "\n",
    "synth_pred = synth_clf.predict(X_val)\n",
    "print(\"Synth Data on holdout set:\",\n",
    "    precision_score(y_val, synth_pred),\n",
    "    recall_score(y_val, synth_pred),\n",
    "    roc_auc_score(y_val, synth_pred),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30162"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prop = .2\n",
    "length = int(prop*len(X))\n",
    "X_val, y_val = X[:length], y[:length]\n",
    "\n",
    "dataloader = DataModule()\n",
    "\n",
    "\n",
    "\n",
    "X_train, y_train = X[length:], y[length:]\n",
    "real_clf = XGBClassifier(use_label_encoder=False).fit(X_train, y_train)\n",
    "\n",
    "real_pred = real_clf.predict(X_val)\n",
    "print(\"Real Data on holdout set:\",\n",
    "    precision_score(y_val, real_pred),\n",
    "    recall_score(y_val, real_pred),\n",
    "    roc_auc_score(y_val, real_pred),\n",
    ")\n",
    "\n",
    "synth_pred = synth_clf.predict(X_val)\n",
    "print(\"Synth Data on holdout set:\",\n",
    "    precision_score(y_val, synth_pred),\n",
    "    recall_score(y_val, synth_pred),\n",
    "    roc_auc_score(y_val, synth_pred),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Some EDA\n",
    "path = \"https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.data\"\n",
    "names = [\n",
    "    \"age\",\n",
    "    \"workclass\",\n",
    "    \"fnlwgt\",\n",
    "    \"education\",\n",
    "    \"education-num\",\n",
    "    \"marital-status\",\n",
    "    \"occupation\",\n",
    "    \"relationship\",\n",
    "    \"race\",\n",
    "    \"sex\",\n",
    "    \"capital-gain\",\n",
    "    \"capital-loss\",\n",
    "    \"hours-per-week\",\n",
    "    \"native-country\",\n",
    "    \"label\",\n",
    "]\n",
    "df = pd.read_csv(path, names=names, index_col=False)\n",
    "df = df.applymap(lambda x: x.strip() if type(x) is str else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>workclass</th>\n",
       "      <th>fnlwgt</th>\n",
       "      <th>education</th>\n",
       "      <th>education-num</th>\n",
       "      <th>marital-status</th>\n",
       "      <th>occupation</th>\n",
       "      <th>relationship</th>\n",
       "      <th>race</th>\n",
       "      <th>sex</th>\n",
       "      <th>capital-gain</th>\n",
       "      <th>capital-loss</th>\n",
       "      <th>hours-per-week</th>\n",
       "      <th>native-country</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>39</td>\n",
       "      <td>State-gov</td>\n",
       "      <td>77516</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Adm-clerical</td>\n",
       "      <td>Not-in-family</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>2174</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>50</td>\n",
       "      <td>Self-emp-not-inc</td>\n",
       "      <td>83311</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Exec-managerial</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>38</td>\n",
       "      <td>Private</td>\n",
       "      <td>215646</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>9</td>\n",
       "      <td>Divorced</td>\n",
       "      <td>Handlers-cleaners</td>\n",
       "      <td>Not-in-family</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>53</td>\n",
       "      <td>Private</td>\n",
       "      <td>234721</td>\n",
       "      <td>11th</td>\n",
       "      <td>7</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Handlers-cleaners</td>\n",
       "      <td>Husband</td>\n",
       "      <td>Black</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "      <td>Private</td>\n",
       "      <td>338409</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Prof-specialty</td>\n",
       "      <td>Wife</td>\n",
       "      <td>Black</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>Cuba</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age         workclass  fnlwgt  education  education-num  \\\n",
       "0   39         State-gov   77516  Bachelors             13   \n",
       "1   50  Self-emp-not-inc   83311  Bachelors             13   \n",
       "2   38           Private  215646    HS-grad              9   \n",
       "3   53           Private  234721       11th              7   \n",
       "4   28           Private  338409  Bachelors             13   \n",
       "\n",
       "       marital-status         occupation   relationship   race     sex  \\\n",
       "0       Never-married       Adm-clerical  Not-in-family  White    Male   \n",
       "1  Married-civ-spouse    Exec-managerial        Husband  White    Male   \n",
       "2            Divorced  Handlers-cleaners  Not-in-family  White    Male   \n",
       "3  Married-civ-spouse  Handlers-cleaners        Husband  Black    Male   \n",
       "4  Married-civ-spouse     Prof-specialty           Wife  Black  Female   \n",
       "\n",
       "   capital-gain  capital-loss  hours-per-week native-country  label  \n",
       "0          2174             0              40  United-States  <=50K  \n",
       "1             0             0              13  United-States  <=50K  \n",
       "2             0             0              40  United-States  <=50K  \n",
       "3             0             0              40  United-States  <=50K  \n",
       "4             0             0              40           Cuba  <=50K  "
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = [\n",
    "            \"Tech-support\",\n",
    "            \"Craft-repair\",\n",
    "            \"Other-service\",\n",
    "            \"Sales\",\n",
    "            \"Exec-managerial\",\n",
    "            \"Prof-specialty\",\n",
    "            \"Handlers-cleaners\",\n",
    "            \"Machine-op-inspct\",\n",
    "            \"Adm-clerical\",\n",
    "            \"Farming-fishing\",\n",
    "            \"Transport-moving\",\n",
    "            \"Priv-house-serv\",\n",
    "            \"Protective-serv\",\n",
    "            \"Armed-Forces\",\n",
    "        ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.data\"\n",
    "names = [\n",
    "    \"age\",\n",
    "    \"workclass\",\n",
    "    \"fnlwgt\",\n",
    "    \"education\",\n",
    "    \"education-num\",\n",
    "    \"marital-status\",\n",
    "    \"occupation\",\n",
    "    \"relationship\",\n",
    "    \"race\",\n",
    "    \"sex\",\n",
    "    \"capital-gain\",\n",
    "    \"capital-loss\",\n",
    "    \"hours-per-week\",\n",
    "    \"native-country\",\n",
    "    \"label\",\n",
    "]\n",
    "df = pd.read_csv(path, names=names, index_col=False)\n",
    "df = df.applymap(lambda x: x.strip() if type(x) is str else x)\n",
    "\n",
    "for col in df:\n",
    "    if df[col].dtype == \"object\":\n",
    "        df = df[df[col] != \"?\"]\n",
    "\n",
    "replace = [\n",
    "    [\n",
    "        \"Private\",\n",
    "        \"Self-emp-not-inc\",\n",
    "        \"Self-emp-inc\",\n",
    "        \"Federal-gov\",\n",
    "        \"Local-gov\",\n",
    "        \"State-gov\",\n",
    "        \"Without-pay\",\n",
    "        \"Never-worked\",\n",
    "    ],\n",
    "    [\n",
    "        \"Bachelors\",\n",
    "        \"Some-college\",\n",
    "        \"11th\",\n",
    "        \"HS-grad\",\n",
    "        \"Prof-school\",\n",
    "        \"Assoc-acdm\",\n",
    "        \"Assoc-voc\",\n",
    "        \"9th\",\n",
    "        \"7th-8th\",\n",
    "        \"12th\",\n",
    "        \"Masters\",\n",
    "        \"1st-4th\",\n",
    "        \"10th\",\n",
    "        \"Doctorate\",\n",
    "        \"5th-6th\",\n",
    "        \"Preschool\",\n",
    "    ],\n",
    "    [\n",
    "        \"Married-civ-spouse\",\n",
    "        \"Divorced\",\n",
    "        \"Never-married\",\n",
    "        \"Separated\",\n",
    "        \"Widowed\",\n",
    "        \"Married-spouse-absent\",\n",
    "        \"Married-AF-spouse\",\n",
    "    ],\n",
    "    [\n",
    "        \"Tech-support\",\n",
    "        \"Craft-repair\",\n",
    "        \"Other-service\",\n",
    "        \"Sales\",\n",
    "        \"Exec-managerial\",\n",
    "        \"Prof-specialty\",\n",
    "        \"Handlers-cleaners\",\n",
    "        \"Machine-op-inspct\",\n",
    "        \"Adm-clerical\",\n",
    "        \"Farming-fishing\",\n",
    "        \"Transport-moving\",\n",
    "        \"Priv-house-serv\",\n",
    "        \"Protective-serv\",\n",
    "        \"Armed-Forces\",\n",
    "    ],\n",
    "    [\n",
    "        \"Wife\",\n",
    "        \"Own-child\",\n",
    "        \"Husband\",\n",
    "        \"Not-in-family\",\n",
    "        \"Other-relative\",\n",
    "        \"Unmarried\",\n",
    "    ],\n",
    "    [\"White\", \"Asian-Pac-Islander\", \"Amer-Indian-Eskimo\", \"Other\", \"Black\"],\n",
    "    [\"Female\", \"Male\"],\n",
    "    [\n",
    "        \"United-States\",\n",
    "        \"Cambodia\",\n",
    "        \"England\",\n",
    "        \"Puerto-Rico\",\n",
    "        \"Canada\",\n",
    "        \"Germany\",\n",
    "        \"Outlying-US(Guam-USVI-etc)\",\n",
    "        \"India\",\n",
    "        \"Japan\",\n",
    "        \"Greece\",\n",
    "        \"South\",\n",
    "        \"China\",\n",
    "        \"Cuba\",\n",
    "        \"Iran\",\n",
    "        \"Honduras\",\n",
    "        \"Philippines\",\n",
    "        \"Italy\",\n",
    "        \"Poland\",\n",
    "        \"Jamaica\",\n",
    "        \"Vietnam\",\n",
    "        \"Mexico\",\n",
    "        \"Portugal\",\n",
    "        \"Ireland\",\n",
    "        \"France\",\n",
    "        \"Dominican-Republic\",\n",
    "        \"Laos\",\n",
    "        \"Ecuador\",\n",
    "        \"Taiwan\",\n",
    "        \"Haiti\",\n",
    "        \"Columbia\",\n",
    "        \"Hungary\",\n",
    "        \"Guatemala\",\n",
    "        \"Nicaragua\",\n",
    "        \"Scotland\",\n",
    "        \"Thailand\",\n",
    "        \"Yugoslavia\",\n",
    "        \"El-Salvador\",\n",
    "        \"Trinadad&Tobago\",\n",
    "        \"Peru\",\n",
    "        \"Hong\",\n",
    "        \"Holand-Netherlands\",\n",
    "    ],\n",
    "    [\">50K\", \"<=50K\"],\n",
    "]\n",
    "\n",
    "for row in replace:\n",
    "    df = df.replace(row, range(len(row)))\n",
    "\n",
    "continuous = [\"education-num\", \"hours-per-week\", \"age\"]\n",
    "for col in continuous:\n",
    "    c_range = df[col].max() - df[col].min()\n",
    "    df[col] = df[col] / c_range\n",
    "\n",
    "df = df[[\"race\", \"age\", \"sex\", \"native-country\", \"marital-status\", \"education-num\", \"occupation\", \"hours-per-week\", \"workclass\", \"relationship\", \"label\"]]\n",
    "# df = pd.get_dummies(df, columns=[\"race\", \"native-country\", \"marital-status\", \"occupation\", \"workclass\", \"relationship\"])\n",
    "# df = df.values\n",
    "\n",
    "feature_num = {0: 5, 1: 1, 2:1, 3: 41, 4: 7, 5: 1, 6:14, 7: 1, 8:7, 9:6, 10:1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical = [\"race\", \"native-country\", \"marital-status\", \"occupation\", \"workclass\", \"relationship\"]\n",
    "df = pd.concat([pd.get_dummies(df[col]) if col in categorical else df[col] for col in df.columns], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index([               0,                1,                2,                3,\n",
       "                      4,            'age',            'sex',                0,\n",
       "                      1,                2,                3,                4,\n",
       "                      5,                6,                7,                8,\n",
       "                      9,               10,               11,               12,\n",
       "                     13,               14,               15,               16,\n",
       "                     17,               18,               19,               20,\n",
       "                     21,               22,               23,               24,\n",
       "                     25,               26,               27,               28,\n",
       "                     29,               30,               31,               32,\n",
       "                     33,               34,               35,               36,\n",
       "                     37,               38,               39,               40,\n",
       "                      0,                1,                2,                3,\n",
       "                      4,                5,                6,  'education-num',\n",
       "                      0,                1,                2,                3,\n",
       "                      4,                5,                6,                7,\n",
       "                      8,                9,               10,               11,\n",
       "                     12,               13, 'hours-per-week',                0,\n",
       "                      1,                2,                3,                4,\n",
       "                      5,                6,                0,                1,\n",
       "                      2,                3,                4,                5,\n",
       "                'label'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.concat(cols, axis=1).columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(30162, 85)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "86"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(feature_num.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "8fb698c318dae44a87bd23781ca3ad9e8055dc4595579aab63f6bc0d37003a58"
  },
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}