ImageAugment3.model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Image Data Augmentation Using Deep Convolutional Generative Adversarial Network (DCGAN)\n",
    "# Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import numpy as np\n",
    "import keras\n",
    "from keras import layers\n",
    "import myUtilities as myUtils\n",
    "import sklearn.utils"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generator Model\n",
    "\n",
    "Follows the DCGAN model from Deep Learning with Python by Francis Chollet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\nLayer (type)                 Output Shape              Param #   \n=================================================================\ninput_1 (InputLayer)         (None, 128)               0         \n_________________________________________________________________\ndense_1 (Dense)              (None, 1048576)           135266304 \n_________________________________________________________________\nleaky_re_lu_1 (LeakyReLU)    (None, 1048576)           0         \n_________________________________________________________________\nreshape_1 (Reshape)          (None, 64, 64, 256)       0         \n_________________________________________________________________\nconv2d_1 (Conv2D)            (None, 64, 64, 256)       1638656   \n_________________________________________________________________\nleaky_re_lu_2 (LeakyReLU)    (None, 64, 64, 256)       0         \n_________________________________________________________________\nconv2d_transpose_1 (Conv2DTr (None, 128, 128, 256)     1048832   \n_________________________________________________________________\nleaky_re_lu_3 (LeakyReLU)    (None, 128, 128, 256)     0         \n_________________________________________________________________\nconv2d_2 (Conv2D)            (None, 128, 128, 256)     1638656   \n_________________________________________________________________\nleaky_re_lu_4 (LeakyReLU)    (None, 128, 128, 256)     0         \n_________________________________________________________________\nconv2d_3 (Conv2D)            (None, 128, 128, 256)     1638656   \n_________________________________________________________________\nleaky_re_lu_5 (LeakyReLU)    (None, 128, 128, 256)     0         \n_________________________________________________________________\nconv2d_4 (Conv2D)            (None, 128, 128, 5)       62725     \n=================================================================\nTotal params: 141,293,829\nTrainable params: 141,293,829\nNon-trainable params: 0\n_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "latent_dim = 128\n",
    "height = 128\n",
    "width = 128\n",
    "channels = 5\n",
    "\n",
    "generator_input = keras.Input(shape=(latent_dim,))\n",
    "\n",
    "# First, transform the input into a 16x16 128-channels feature map\n",
    "x = layers.Dense(256 * 64 * 64)(generator_input)\n",
    "x = layers.LeakyReLU()(x)\n",
    "x = layers.Reshape((64, 64, 256))(x)\n",
    "\n",
    "# Then, add a convolution layer\n",
    "x = layers.Conv2D(256, 5, padding='same')(x)\n",
    "x = layers.LeakyReLU()(x)\n",
    "\n",
    "# Upsample to 32x32\n",
    "x = layers.Conv2DTranspose(256, 4, strides=2, padding='same')(x)\n",
    "x = layers.LeakyReLU()(x)\n",
    "\n",
    "# Few more conv layers\n",
    "x = layers.Conv2D(256, 5, padding='same')(x)\n",
    "x = layers.LeakyReLU()(x)\n",
    "x = layers.Conv2D(256, 5, padding='same')(x)\n",
    "x = layers.LeakyReLU()(x)\n",
    "\n",
    "# Produce a 32x32 1-channel feature map\n",
    "x = layers.Conv2D(channels, 7, activation='tanh', padding='same')(x)\n",
    "generator = keras.models.Model(generator_input, x)\n",
    "generator.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Discriminator Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\nLayer (type)                 Output Shape              Param #   \n=================================================================\ninput_2 (InputLayer)         (None, 128, 128, 5)       0         \n_________________________________________________________________\nconv2d_5 (Conv2D)            (None, 126, 126, 128)     5888      \n_________________________________________________________________\nleaky_re_lu_6 (LeakyReLU)    (None, 126, 126, 128)     0         \n_________________________________________________________________\nconv2d_6 (Conv2D)            (None, 62, 62, 128)       262272    \n_________________________________________________________________\nleaky_re_lu_7 (LeakyReLU)    (None, 62, 62, 128)       0         \n_________________________________________________________________\nconv2d_7 (Conv2D)            (None, 30, 30, 128)       262272    \n_________________________________________________________________\nleaky_re_lu_8 (LeakyReLU)    (None, 30, 30, 128)       0         \n_________________________________________________________________\nconv2d_8 (Conv2D)            (None, 14, 14, 128)       262272    \n_________________________________________________________________\nleaky_re_lu_9 (LeakyReLU)    (None, 14, 14, 128)       0         \n_________________________________________________________________\nconv2d_9 (Conv2D)            (None, 6, 6, 128)         262272    \n_________________________________________________________________\nleaky_re_lu_10 (LeakyReLU)   (None, 6, 6, 128)         0         \n_________________________________________________________________\nconv2d_10 (Conv2D)           (None, 2, 2, 128)         262272    \n_________________________________________________________________\nleaky_re_lu_11 (LeakyReLU)   (None, 2, 2, 128)         0         \n_________________________________________________________________\nflatten_1 (Flatten)          (None, 512)               0         \n_________________________________________________________________\ndropout_1 (Dropout)          (None, 512)               0         \n_________________________________________________________________\ndense_2 (Dense)              (None, 1)                 513       \n=================================================================\nTotal params: 1,317,761\nTrainable params: 1,317,761\nNon-trainable params: 0\n_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "discriminator_input = layers.Input(shape=(height, width, channels))\n",
    "x = layers.Conv2D(128, 3)(discriminator_input)\n",
    "x = layers.LeakyReLU()(x)\n",
    "x = layers.Conv2D(128, 4, strides=2)(x)\n",
    "x = layers.LeakyReLU()(x)\n",
    "x = layers.Conv2D(128, 4, strides=2)(x)\n",
    "x = layers.LeakyReLU()(x)\n",
    "x = layers.Conv2D(128, 4, strides=2)(x)\n",
    "x = layers.LeakyReLU()(x)\n",
    "\n",
    "# Here, I have added additional convolutions layers with stride 2 to further reduce\n",
    "# the size of the output before the final dense layer.\n",
    "x = layers.Conv2D(128, 4, strides=2)(x)\n",
    "x = layers.LeakyReLU()(x)\n",
    "x = layers.Conv2D(128, 4, strides=2)(x)\n",
    "x = layers.LeakyReLU()(x)\n",
    "\n",
    "x = layers.Flatten()(x)\n",
    "\n",
    "# One dropout layer - important trick!\n",
    "x = layers.Dropout(0.4)(x)\n",
    "\n",
    "# Classification layer\n",
    "x = layers.Dense(1, activation='sigmoid')(x)\n",
    "\n",
    "discriminator = keras.models.Model(discriminator_input, x)\n",
    "discriminator.summary()\n",
    "\n",
    "# To stabilize training, we use learning rate decay\n",
    "# and gradient clipping (by value) in the optimizer.\n",
    "# discriminator_optimizer = keras.optimizers.RMSprop(lr=0.0008, clipvalue=1.0, decay=1e-8)\n",
    "# Reduce the learning rate so that the discriminator and generator does not overpower\n",
    "# each other quickly. At larger learning rates, the loss of one drops to zero while the\n",
    "# other shows no change. \n",
    "discriminator_optimizer = keras.optimizers.RMSprop(lr=0.0004, clipvalue=1.0, decay=1e-8)\n",
    "discriminator.compile(optimizer=discriminator_optimizer, loss='binary_crossentropy')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The Adversarial Network Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set discriminator weights to non-trainable\n",
    "# (will only apply to the `gan` model)\n",
    "discriminator.trainable = False\n",
    "\n",
    "gan_input = keras.Input(shape=(latent_dim,))\n",
    "gan_output = discriminator(generator(gan_input))\n",
    "gan = keras.models.Model(gan_input, gan_output)\n",
    "\n",
    "# Reduce the learning rate to prevent the networks loss from settling\n",
    "# into a local minimum. See discriminator learning rate.\n",
    "# gan_optimizer = keras.optimizers.RMSprop(lr=0.0004, clipvalue=1.0, decay=1e-8)\n",
    "gan_optimizer = keras.optimizers.RMSprop(lr=0.0002, clipvalue=1.0, decay=1e-8)\n",
    "gan.compile(optimizer=gan_optimizer, loss='binary_crossentropy')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\Python36_64\\lib\\site-packages\\keras\\engine\\training.py:479: UserWarning: Discrepancy between trainable weights and collected trainable weights, did you set `model.trainable` without calling `model.compile` after ?\n  'Discrepancy between trainable weights and collected trainable'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 0: 0.6886867\nadversarial loss at step 0: 0.6797119\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 50: 0.66255397\nadversarial loss at step 50: 0.66550463\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 100: 0.5520548\nadversarial loss at step 100: 0.7829113\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 150: 0.7019726\nadversarial loss at step 150: 0.71089953\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 200: 0.50612116\nadversarial loss at step 200: 1.5045776\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 250: 0.5845079\nadversarial loss at step 250: 0.9743932\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 300: 0.5690267\nadversarial loss at step 300: 1.1145437\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 350: 0.61759007\nadversarial loss at step 350: 1.0783365\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 400: 0.5918338\nadversarial loss at step 400: 2.2250662\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 450: 0.51140964\nadversarial loss at step 450: 2.3456376\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 500: 0.5578655\nadversarial loss at step 500: 1.2269628\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 550: 0.5582696\nadversarial loss at step 550: 5.5591884\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 600: 0.571164\nadversarial loss at step 600: 1.9302366\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 650: 0.343031\nadversarial loss at step 650: 1.4735945\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 700: 0.37929395\nadversarial loss at step 700: 2.582828\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 750: 0.53215826\nadversarial loss at step 750: 1.184698\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 800: 0.55654967\nadversarial loss at step 800: 1.7203987\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 850: 0.44739923\nadversarial loss at step 850: 1.9230007\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 900: 0.31007543\nadversarial loss at step 900: 3.7031815\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 950: 0.40189376\nadversarial loss at step 950: 1.2991921\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 1000: 0.44719258\nadversarial loss at step 1000: 3.8684068\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 1050: 0.71324795\nadversarial loss at step 1050: 2.7462296\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 1100: 0.49053183\nadversarial loss at step 1100: 1.0793657\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 1150: 0.12570615\nadversarial loss at step 1150: 2.8071253\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 1200: 0.38426515\nadversarial loss at step 1200: 1.6524099\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 1250: 0.24576493\nadversarial loss at step 1250: 2.3542194\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 1300: 0.16932783\nadversarial loss at step 1300: 4.265878\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 1350: 0.6776765\nadversarial loss at step 1350: 0.7491705\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 1400: 0.2258697\nadversarial loss at step 1400: 1.9097849\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 1450: 0.42113775\nadversarial loss at step 1450: 0.7661966\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 1500: 0.3195902\nadversarial loss at step 1500: 2.458496\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 1550: 0.37553328\nadversarial loss at step 1550: 3.4029279\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 1600: 1.2593176\nadversarial loss at step 1600: 8.959753\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 1650: 0.034171857\nadversarial loss at step 1650: 6.386036\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 1700: 0.1351354\nadversarial loss at step 1700: 3.5472386\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 1750: 0.31200913\nadversarial loss at step 1750: 2.6248524\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 1800: 0.077965125\nadversarial loss at step 1800: 5.357026\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 1850: 0.6413084\nadversarial loss at step 1850: 8.6896305\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 1900: 0.30707914\nadversarial loss at step 1900: 1.9498408\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 1950: 0.15742859\nadversarial loss at step 1950: 2.566113\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 2000: 0.26890337\nadversarial loss at step 2000: 3.9821808\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 2050: 0.28248057\nadversarial loss at step 2050: 2.9827843\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 2100: 0.13925016\nadversarial loss at step 2100: 1.9263325\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 2150: 0.2958749\nadversarial loss at step 2150: 4.8849325\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 2200: 0.52016675\nadversarial loss at step 2200: 1.3058003\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 2250: 0.31215295\nadversarial loss at step 2250: 2.0286756\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 2300: 0.30134133\nadversarial loss at step 2300: 2.6750627\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 2350: 0.9699839\nadversarial loss at step 2350: 1.2052076\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 2400: 0.41420546\nadversarial loss at step 2400: 2.8063898\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 2450: 0.104497194\nadversarial loss at step 2450: 3.5302627\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 2500: 0.21295805\nadversarial loss at step 2500: 3.6395469\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 2550: 0.21591642\nadversarial loss at step 2550: 2.3661017\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 2600: 0.089581944\nadversarial loss at step 2600: 5.4424496\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 2650: 0.090595864\nadversarial loss at step 2650: 4.42334\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 2700: 1.005263\nadversarial loss at step 2700: 1.472844\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 2750: 1.131143\nadversarial loss at step 2750: 1.6175655\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 2800: 0.18692681\nadversarial loss at step 2800: 3.4430778\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 2850: 0.057644308\nadversarial loss at step 2850: 7.9026093\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 2900: 0.9104276\nadversarial loss at step 2900: 3.06831\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 2950: 0.05880462\nadversarial loss at step 2950: 4.227351\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 3000: 0.13787344\nadversarial loss at step 3000: 3.522792\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 3050: 0.44273406\nadversarial loss at step 3050: 1.9589891\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 3100: 0.46445614\nadversarial loss at step 3100: 1.8361504\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 3150: -0.032310445\nadversarial loss at step 3150: 9.611931\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 3200: 0.05233114\nadversarial loss at step 3200: 4.2381015\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 3250: 0.053356003\nadversarial loss at step 3250: 6.4009733\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 3300: 0.14153007\nadversarial loss at step 3300: 3.1771908\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 3350: 0.123171866\nadversarial loss at step 3350: 8.369621\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 3400: 0.056289367\nadversarial loss at step 3400: 4.5884314\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 3450: 0.06564996\nadversarial loss at step 3450: 4.792053\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 3500: 0.14032009\nadversarial loss at step 3500: 3.4242873\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 3550: 0.17862196\nadversarial loss at step 3550: 5.1823907\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 3600: 0.50212306\nadversarial loss at step 3600: 13.610845\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 3650: 0.06214906\nadversarial loss at step 3650: 5.2497897\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 3700: 0.45937213\nadversarial loss at step 3700: 2.879615\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 3750: 0.01617353\nadversarial loss at step 3750: 8.307514\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 3800: 0.07209712\nadversarial loss at step 3800: 4.1415033\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 3850: -0.09369274\nadversarial loss at step 3850: 14.410154\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 3900: 0.037338637\nadversarial loss at step 3900: 7.0338287\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 3950: -0.026012685\nadversarial loss at step 3950: 8.583669\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 4000: 0.0029409616\nadversarial loss at step 4000: 8.017131\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 4050: 0.031909645\nadversarial loss at step 4050: 8.374807\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 4100: 0.025087107\nadversarial loss at step 4100: 4.7271585\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 4150: 0.06306456\nadversarial loss at step 4150: 5.616501\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 4200: -0.077784196\nadversarial loss at step 4200: 15.596006\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 4250: 0.6536231\nadversarial loss at step 4250: 3.4238458\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 4300: 0.6596316\nadversarial loss at step 4300: 1.9477329\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 4350: 0.126106\nadversarial loss at step 4350: 3.5857856\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 4400: 0.4200154\nadversarial loss at step 4400: 1.6374702\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 4450: 0.08732456\nadversarial loss at step 4450: 7.77249\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 4500: 0.36911118\nadversarial loss at step 4500: 2.2154202\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 4550: 0.26192373\nadversarial loss at step 4550: 2.8304768\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 4600: 0.34447291\nadversarial loss at step 4600: 2.2491386\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 4650: 0.24937388\nadversarial loss at step 4650: 2.5484583\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 4700: 0.3205627\nadversarial loss at step 4700: 1.7059835\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 4750: 0.11272428\nadversarial loss at step 4750: 6.2630897\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 4800: 0.30498078\nadversarial loss at step 4800: 2.63156\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 4850: 0.033144705\nadversarial loss at step 4850: 5.5313287\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 4900: 2.2362056\nadversarial loss at step 4900: 1.8859863\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 4950: 0.08086579\nadversarial loss at step 4950: 3.4424565\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 5000: 0.1762152\nadversarial loss at step 5000: 3.442382\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 5050: 0.81528723\nadversarial loss at step 5050: 2.509658\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 5100: 0.60765296\nadversarial loss at step 5100: 2.6584537\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 5150: 0.045700073\nadversarial loss at step 5150: 7.3375497\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 5200: 0.14043005\nadversarial loss at step 5200: 3.796706\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 5250: 0.03758212\nadversarial loss at step 5250: 9.328768\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 5300: 0.020768289\nadversarial loss at step 5300: 9.133642\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 5350: 0.16729814\nadversarial loss at step 5350: 3.096107\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 5400: 0.047843087\nadversarial loss at step 5400: 4.035048\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator loss at step 5450: 0.015765142\nadversarial loss at step 5450: 6.402471\n"
     ]
    },
    {
     "ename": "InternalError",
     "evalue": "GPU sync failed",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mInternalError\u001b[0m                             Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-5-097859f9b481>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     41\u001b[0m     \u001b[1;31m# Train the generator (via the gan model,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     42\u001b[0m     \u001b[1;31m# where the discriminator weights are frozen)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 43\u001b[1;33m     \u001b[0ma_loss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgan\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain_on_batch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrandom_latent_vectors\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmisleading_targets\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     44\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     45\u001b[0m     \u001b[0mstart\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\Python36_64\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[1;34m(self, x, y, sample_weight, class_weight)\u001b[0m\n\u001b[0;32m   1213\u001b[0m             \u001b[0mins\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mx\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0my\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0msample_weights\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1214\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_make_train_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1215\u001b[1;33m         \u001b[0moutputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mins\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1216\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0munpack_singleton\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1217\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\Python36_64\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m   2664\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_legacy_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2665\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2666\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2667\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2668\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mis_tensor\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[1;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\Python36_64\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\u001b[0m in \u001b[0;36m_call\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m   2634\u001b[0m                                 \u001b[0msymbol_vals\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2635\u001b[0m                                 session)\n\u001b[1;32m-> 2636\u001b[1;33m         \u001b[0mfetched\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2637\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mfetched\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2638\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\Python36_64\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1380\u001b[0m           ret = tf_session.TF_SessionRunCallable(\n\u001b[0;32m   1381\u001b[0m               \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1382\u001b[1;33m               run_metadata_ptr)\n\u001b[0m\u001b[0;32m   1383\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1384\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\Python36_64\\lib\\site-packages\\tensorflow\\python\\framework\\errors_impl.py\u001b[0m in \u001b[0;36m__exit__\u001b[1;34m(self, type_arg, value_arg, traceback_arg)\u001b[0m\n\u001b[0;32m    517\u001b[0m             \u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    518\u001b[0m             \u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mc_api\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_Message\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstatus\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstatus\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 519\u001b[1;33m             c_api.TF_GetCode(self.status.status))\n\u001b[0m\u001b[0;32m    520\u001b[0m     \u001b[1;31m# Delete the underlying status object from memory otherwise it stays alive\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    521\u001b[0m     \u001b[1;31m# as there is a reference to status from this from the traceback due to\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mInternalError\u001b[0m: GPU sync failed"
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "# Load data\n",
    "x_train = myUtils.load_preprocessed_data()\n",
    "\n",
    "# Normalize data\n",
    "x_train = x_train.astype('float32') / 255.\n",
    "\n",
    "iterations = 10000\n",
    "batch_size = 20\n",
    "save_dir = '.\\data\\\\gan_images\\\\'\n",
    "\n",
    "# Start training loop\n",
    "start = 0\n",
    "for step in range(iterations):\n",
    "    # Sample random points in the latent space\n",
    "    random_latent_vectors = np.random.normal(size=(batch_size, latent_dim))\n",
    "\n",
    "    # Decode them to fake images\n",
    "    generated_images = generator.predict(random_latent_vectors)\n",
    "\n",
    "    # Combine them with real images\n",
    "    stop = start + batch_size\n",
    "    real_images = x_train[start: stop]\n",
    "    combined_images = np.concatenate([generated_images, real_images])\n",
    "\n",
    "    # Assemble labels discriminating real from fake images\n",
    "    labels = np.concatenate([np.ones((batch_size, 1)),\n",
    "                             np.zeros((batch_size, 1))])\n",
    "\n",
    "    # Add random noise to the labels - important trick!\n",
    "    labels += 0.05 * np.random.random(labels.shape)\n",
    "\n",
    "    # Train the discriminator\n",
    "    d_loss = discriminator.train_on_batch(combined_images, labels)\n",
    "\n",
    "    # sample random points in the latent space\n",
    "    random_latent_vectors = np.random.normal(size=(batch_size, latent_dim))\n",
    "\n",
    "    # Assemble labels that say \"all real images\"\n",
    "    misleading_targets = np.zeros((batch_size, 1))\n",
    "\n",
    "    # Train the generator (via the gan model,\n",
    "    # where the discriminator weights are frozen)\n",
    "    a_loss = gan.train_on_batch(random_latent_vectors, misleading_targets)\n",
    "    \n",
    "    start += batch_size\n",
    "    if start > len(x_train) - batch_size:\n",
    "        start = 0\n",
    "        # Is shuffling needed for stochastic gradient descent?\n",
    "        #x_train = sklearn.utils.shuffle(x_train)\n",
    "\n",
    "    # Occasionally save / plot\n",
    "    if step % 50 == 0:\n",
    "\n",
    "        # Print metrics\n",
    "        print('discriminator loss at step %s: %s' % (step, d_loss))\n",
    "        print('adversarial loss at step %s: %s' % (step, a_loss))\n",
    "\n",
    "        # Save one generated image\n",
    "        # img = image.array_to_img(generated_images[0] * 255., scale=False)\n",
    "        # img.save(os.path.join(save_dir, 'generated_frog' + str(step) + '.png'))\n",
    "        np.save(save_dir + 'generated_' + str(step), generated_images[0] * 255.)\n",
    "\n",
    "        # Save one real image, for comparison\n",
    "        # img = image.array_to_img(real_images[0] * 255., scale=False)\n",
    "        # img.save(os.path.join(save_dir, 'real_frog' + str(step) + '.png'))\n",
    "        np.save(save_dir + 'real_' + str(step), real_images[0] * 255.)\n",
    "        \n",
    "    if step % 1000 == 0:\n",
    "\n",
    "        # Save model weights\n",
    "        gan.save_weights('.\\data\\\\gan_model\\\\gan' + str(step) + '.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}