4_convolutions.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "4embtkV0pNxM"
   },
   "source": [
    "Deep Learning\n",
    "=============\n",
    "\n",
    "Assignment 4\n",
    "------------\n",
    "\n",
    "Previously in `2_fullyconnected.ipynb` and `3_regularization.ipynb`, we trained fully connected networks to classify [notMNIST](http://yaroslavvb.blogspot.com/2011/09/notmnist-dataset.html) characters.\n",
    "\n",
    "The goal of this assignment is make the neural network convolutional."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "tm2CQN_Cpwj0"
   },
   "outputs": [],
   "source": [
    "# These are all the modules we'll be using later. Make sure you can import them\n",
    "# before proceeding further.\n",
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from six.moves import cPickle as pickle\n",
    "from six.moves import range"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 11948,
     "status": "ok",
     "timestamp": 1446658914837,
     "user": {
      "color": "",
      "displayName": "",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "",
      "photoUrl": "",
      "sessionId": "0",
      "userId": ""
     },
     "user_tz": 480
    },
    "id": "y3-cj1bpmuxc",
    "outputId": "016b1a51-0290-4b08-efdb-8c95ffc3cd01"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set (200000, 28, 28) (200000,)\n",
      "Validation set (10000, 28, 28) (10000,)\n",
      "Test set (10000, 28, 28) (10000,)\n"
     ]
    }
   ],
   "source": [
    "pickle_file = './notMNIST.pickle'\n",
    "\n",
    "with open(pickle_file, 'rb') as f:\n",
    "  save = pickle.load(f)\n",
    "  train_dataset = save['train_dataset']\n",
    "  train_labels = save['train_labels']\n",
    "  valid_dataset = save['valid_dataset']\n",
    "  valid_labels = save['valid_labels']\n",
    "  test_dataset = save['test_dataset']\n",
    "  test_labels = save['test_labels']\n",
    "  del save  # hint to help gc free up memory\n",
    "  print('Training set', train_dataset.shape, train_labels.shape)\n",
    "  print('Validation set', valid_dataset.shape, valid_labels.shape)\n",
    "  print('Test set', test_dataset.shape, test_labels.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "L7aHrm6nGDMB"
   },
   "source": [
    "Reformat into a TensorFlow-friendly shape:\n",
    "- convolutions need the image data formatted as a cube (width by height by #channels)\n",
    "- labels as float 1-hot encodings."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 11952,
     "status": "ok",
     "timestamp": 1446658914857,
     "user": {
      "color": "",
      "displayName": "",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "",
      "photoUrl": "",
      "sessionId": "0",
      "userId": ""
     },
     "user_tz": 480
    },
    "id": "IRSyYiIIGIzS",
    "outputId": "650a208c-8359-4852-f4f5-8bf10e80ef6c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set (200000, 28, 28, 1) (200000, 10)\n",
      "Validation set (10000, 28, 28, 1) (10000, 10)\n",
      "Test set (10000, 28, 28, 1) (10000, 10)\n"
     ]
    }
   ],
   "source": [
    "image_size = 28\n",
    "num_labels = 10\n",
    "num_channels = 1 # grayscale\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "def reformat(dataset, labels):\n",
    "  dataset = dataset.reshape(\n",
    "    (-1, image_size, image_size, num_channels)).astype(np.float32)\n",
    "  labels = (np.arange(num_labels) == labels[:,None]).astype(np.float32)\n",
    "  return dataset, labels\n",
    "train_dataset, train_labels = reformat(train_dataset, train_labels)\n",
    "valid_dataset, valid_labels = reformat(valid_dataset, valid_labels)\n",
    "test_dataset, test_labels = reformat(test_dataset, test_labels)\n",
    "print('Training set', train_dataset.shape, train_labels.shape)\n",
    "print('Validation set', valid_dataset.shape, valid_labels.shape)\n",
    "print('Test set', test_dataset.shape, test_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "AgQDIREv02p1"
   },
   "outputs": [],
   "source": [
    "def accuracy(predictions, labels):\n",
    "  return (100.0 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1))\n",
    "          / predictions.shape[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "5rhgjmROXu2O"
   },
   "source": [
    "Let's build a small network with two convolutional layers, followed by one fully connected layer. Convolutional networks are more expensive computationally, so we'll limit its depth and number of fully connected nodes."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 两层卷积，未池化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "IZYv70SvvOan"
   },
   "outputs": [],
   "source": [
    "batch_size = 16\n",
    "patch_size = 5\n",
    "depth = 16\n",
    "num_hidden = 64\n",
    "\n",
    "graph = tf.Graph()\n",
    "\n",
    "with graph.as_default():\n",
    "\n",
    "  # Input data.\n",
    "  tf_train_dataset = tf.placeholder(\n",
    "    tf.float32, shape=(batch_size, image_size, image_size, num_channels))\n",
    "  tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))\n",
    "  tf_valid_dataset = tf.constant(valid_dataset)\n",
    "  tf_test_dataset = tf.constant(test_dataset)\n",
    "  \n",
    "  # Variables.\n",
    "  layer1_weights = tf.Variable(tf.truncated_normal(\n",
    "      [patch_size, patch_size, num_channels, depth], stddev=0.1))\n",
    "  layer1_biases = tf.Variable(tf.zeros([depth]))\n",
    "  layer2_weights = tf.Variable(tf.truncated_normal(\n",
    "      [patch_size, patch_size, depth, depth], stddev=0.1))\n",
    "  layer2_biases = tf.Variable(tf.constant(1.0, shape=[depth]))\n",
    "  layer3_weights = tf.Variable(tf.truncated_normal(\n",
    "      [image_size // 4 * image_size // 4 * depth, num_hidden], stddev=0.1))\n",
    "  layer3_biases = tf.Variable(tf.constant(1.0, shape=[num_hidden]))\n",
    "  layer4_weights = tf.Variable(tf.truncated_normal(\n",
    "      [num_hidden, num_labels], stddev=0.1))\n",
    "  layer4_biases = tf.Variable(tf.constant(1.0, shape=[num_labels]))\n",
    "  \n",
    "  # Model.\n",
    "  def model(data):\n",
    "    # strides: A list of ints. 1-D of length 4. The stride of the sliding window for each dimension of input\n",
    "    conv = tf.nn.conv2d(data, layer1_weights, strides=[1, 2, 2, 1], padding='SAME')\n",
    "    hidden = tf.nn.relu(conv + layer1_biases)\n",
    "    conv = tf.nn.conv2d(hidden, layer2_weights, strides=[1, 2, 2, 1], padding='SAME')\n",
    "    hidden = tf.nn.relu(conv + layer2_biases)\n",
    "    shape = hidden.get_shape().as_list()\n",
    "    reshape = tf.reshape(hidden, [shape[0], shape[1] * shape[2] * shape[3]])\n",
    "    hidden = tf.nn.relu(tf.matmul(reshape, layer3_weights) + layer3_biases)\n",
    "    return tf.matmul(hidden, layer4_weights) + layer4_biases\n",
    "  \n",
    "  # Training computation.\n",
    "  logits = model(tf_train_dataset)\n",
    "  loss = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=tf_train_labels, logits=logits))\n",
    "    \n",
    "  # Optimizer.\n",
    "  optimizer = tf.train.GradientDescentOptimizer(0.05).minimize(loss)\n",
    "  \n",
    "  # Predictions for the training, validation, and test data.\n",
    "  train_prediction = tf.nn.softmax(logits)\n",
    "  valid_prediction = tf.nn.softmax(model(tf_valid_dataset))\n",
    "  test_prediction = tf.nn.softmax(model(tf_test_dataset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 37
      }
     ]
    },
    "colab_type": "code",
    "collapsed": true,
    "executionInfo": {
     "elapsed": 63292,
     "status": "ok",
     "timestamp": 1446658966251,
     "user": {
      "color": "",
      "displayName": "",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "",
      "photoUrl": "",
      "sessionId": "0",
      "userId": ""
     },
     "user_tz": 480
    },
    "id": "noKFb2UovVFR",
    "outputId": "28941338-2ef9-4088-8bd1-44295661e628"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Minibatch loss at step 0: 3.277067\n",
      "Minibatch accuracy: 0.0%\n",
      "Validation accuracy: 10.0%\n",
      "Minibatch loss at step 50: 0.944812\n",
      "Minibatch accuracy: 62.5%\n",
      "Validation accuracy: 50.5%\n",
      "Minibatch loss at step 100: 1.059388\n",
      "Minibatch accuracy: 56.2%\n",
      "Validation accuracy: 73.5%\n",
      "Minibatch loss at step 150: 1.150371\n",
      "Minibatch accuracy: 62.5%\n",
      "Validation accuracy: 72.0%\n",
      "Minibatch loss at step 200: 1.054395\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 78.0%\n",
      "Minibatch loss at step 250: 0.822889\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 78.1%\n",
      "Minibatch loss at step 300: 0.907918\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 79.8%\n",
      "Minibatch loss at step 350: 0.350143\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 80.5%\n",
      "Minibatch loss at step 400: 0.695514\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 80.1%\n",
      "Minibatch loss at step 450: 1.028287\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 78.8%\n",
      "Minibatch loss at step 500: 0.632704\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 81.6%\n",
      "Minibatch loss at step 550: 0.432547\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 80.9%\n",
      "Minibatch loss at step 600: 0.220484\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 82.3%\n",
      "Minibatch loss at step 650: 0.480207\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 82.2%\n",
      "Minibatch loss at step 700: 0.657629\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 82.1%\n",
      "Minibatch loss at step 750: 0.621178\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 82.5%\n",
      "Minibatch loss at step 800: 0.878932\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 81.3%\n",
      "Minibatch loss at step 850: 0.624281\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 83.1%\n",
      "Minibatch loss at step 900: 0.589018\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 82.8%\n",
      "Minibatch loss at step 950: 0.382989\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 82.2%\n",
      "Minibatch loss at step 1000: 0.329264\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 83.7%\n",
      "Test accuracy: 90.3%\n"
     ]
    }
   ],
   "source": [
    "num_steps = 1001\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "  tf.global_variables_initializer().run()\n",
    "  print('Initialized')\n",
    "  for step in range(num_steps):\n",
    "    offset = (step * batch_size) % (train_labels.shape[0] - batch_size)\n",
    "    batch_data = train_dataset[offset:(offset + batch_size), :, :, :]\n",
    "    batch_labels = train_labels[offset:(offset + batch_size), :]\n",
    "    feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}\n",
    "    _, l, predictions = session.run(\n",
    "      [optimizer, loss, train_prediction], feed_dict=feed_dict)\n",
    "    if (step % 50 == 0):\n",
    "      print('Minibatch loss at step %d: %f' % (step, l))\n",
    "      print('Minibatch accuracy: %.1f%%' % accuracy(predictions, batch_labels))\n",
    "      print('Validation accuracy: %.1f%%' % accuracy(\n",
    "        valid_prediction.eval(), valid_labels))\n",
    "  print('Test accuracy: %.1f%%' % accuracy(test_prediction.eval(), test_labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "KedKkn4EutIK"
   },
   "source": [
    "---\n",
    "Problem 1\n",
    "---------\n",
    "\n",
    "The convolutional model above uses convolutions with stride 2 to reduce the dimensionality. Replace the strides by a max pooling operation (`nn.max_pool()`) of stride 2 and kernel size 2.\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.5\n",
      "64\n"
     ]
    }
   ],
   "source": [
    "import math\n",
    "\n",
    "print(7/2)\n",
    "print(math.ceil(math.ceil(image_size//2/2)//2/2)**2*depth)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 两层卷积+两次pooling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "IZYv70SvvOan"
   },
   "outputs": [],
   "source": [
    "batch_size = 16\n",
    "patch_size = 5\n",
    "depth = 16\n",
    "num_hidden = 64\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "\n",
    "  # Input data.\n",
    "  tf_train_dataset = tf.placeholder(\n",
    "    tf.float32, shape=(batch_size, image_size, image_size, num_channels))\n",
    "  tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))\n",
    "  tf_valid_dataset = tf.constant(valid_dataset)\n",
    "  tf_test_dataset = tf.constant(test_dataset)\n",
    "  \n",
    "  # Variables.\n",
    "  layer1_weights = tf.Variable(tf.truncated_normal(\n",
    "      [patch_size, patch_size, num_channels, depth], stddev=0.1))\n",
    "  layer1_biases = tf.Variable(tf.zeros([depth]))\n",
    "  layer2_weights = tf.Variable(tf.truncated_normal(\n",
    "      [patch_size, patch_size, depth, depth], stddev=0.1))\n",
    "  layer2_biases = tf.Variable(tf.constant(1.0, shape=[depth]))\n",
    "  layer3_weights = tf.Variable(tf.truncated_normal(\n",
    "      [floor(floor(image_size//2/2)//2/2)**2*deepth , num_hidden], stddev=0.1))\n",
    "  layer3_biases = tf.Variable(tf.constant(1.0, shape=[num_hidden]))\n",
    "  layer4_weights = tf.Variable(tf.truncated_normal(\n",
    "      [num_hidden, num_labels], stddev=0.1))\n",
    "  layer4_biases = tf.Variable(tf.constant(1.0, shape=[num_labels]))\n",
    "  \n",
    "  def max_pool_2x2(x):\n",
    "    \"\"\"\n",
    "    padding = 'SAME': Round down (only full size windows are considered).\n",
    "    padding = 'VALID': Round up (partial windows are included).\n",
    "    \"\"\"\n",
    "    return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],\n",
    "                            strides=[1, 2, 2, 1], padding='SAME')\n",
    "\n",
    "  # Model.\n",
    "  def model(data):\n",
    "    # strides: A list of ints. 1-D of length 4. The stride of the sliding window for each dimension of input\n",
    "    conv = tf.nn.conv2d(data, layer1_weights, strides=[1, 1, 1, 1], padding='SAME')\n",
    "    hidden = tf.nn.relu(conv + layer1_biases)\n",
    "    max_pool = max_pool_2x2(hidden)\n",
    "    conv = tf.nn.conv2d(max_pool, layer2_weights, strides=[1, 1, 1, 1], padding='SAME')\n",
    "    hidden = tf.nn.relu(conv + layer2_biases)\n",
    "    max_pool = max_pool_2x2(hidden)\n",
    "    shape = max_pool.get_shape().as_list()\n",
    "    reshape = tf.reshape(max_pool, [shape[0], shape[1] * shape[2] * shape[3]])\n",
    "    hidden = tf.nn.relu(tf.matmul(reshape, layer3_weights) + layer3_biases)\n",
    "    return tf.matmul(hidden, layer4_weights) + layer4_biases\n",
    "  \n",
    "  # Training computation.\n",
    "  logits = model(tf_train_dataset)\n",
    "  loss = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=tf_train_labels, logits=logits))\n",
    "    \n",
    "  # Optimizer.\n",
    "  optimizer = tf.train.GradientDescentOptimizer(0.05).minimize(loss)\n",
    "  \n",
    "  # Predictions for the training, validation, and test data.\n",
    "  train_prediction = tf.nn.softmax(logits)\n",
    "  valid_prediction = tf.nn.softmax(model(tf_valid_dataset))\n",
    "  test_prediction = tf.nn.softmax(model(tf_test_dataset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 37
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 63292,
     "status": "ok",
     "timestamp": 1446658966251,
     "user": {
      "color": "",
      "displayName": "",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "",
      "photoUrl": "",
      "sessionId": "0",
      "userId": ""
     },
     "user_tz": 480
    },
    "id": "noKFb2UovVFR",
    "outputId": "28941338-2ef9-4088-8bd1-44295661e628",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Minibatch loss at step 0: 2.245908\n",
      "Minibatch accuracy: 18.8%\n",
      "Validation accuracy: 10.2%\n",
      "Minibatch loss at step 50: 1.209020\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 35.3%\n",
      "Minibatch loss at step 100: 1.081415\n",
      "Minibatch accuracy: 62.5%\n",
      "Validation accuracy: 65.3%\n",
      "Minibatch loss at step 150: 1.404632\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 65.5%\n",
      "Minibatch loss at step 200: 1.074799\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 77.5%\n",
      "Minibatch loss at step 250: 0.892156\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 73.6%\n",
      "Minibatch loss at step 300: 1.045174\n",
      "Minibatch accuracy: 62.5%\n",
      "Validation accuracy: 80.0%\n",
      "Minibatch loss at step 350: 0.463666\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 81.0%\n",
      "Minibatch loss at step 400: 0.827460\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 80.0%\n",
      "Minibatch loss at step 450: 1.030784\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 80.6%\n",
      "Minibatch loss at step 500: 0.843199\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 82.4%\n",
      "Minibatch loss at step 550: 0.308153\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 81.6%\n",
      "Minibatch loss at step 600: 0.095665\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 83.1%\n",
      "Minibatch loss at step 650: 0.400784\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 82.5%\n",
      "Minibatch loss at step 700: 0.592804\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 82.7%\n",
      "Minibatch loss at step 750: 0.645150\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.0%\n",
      "Minibatch loss at step 800: 1.359351\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 82.2%\n",
      "Minibatch loss at step 850: 0.650528\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 83.9%\n",
      "Minibatch loss at step 900: 0.679671\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 83.5%\n",
      "Minibatch loss at step 950: 0.499311\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 82.9%\n",
      "Minibatch loss at step 1000: 0.329878\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.3%\n",
      "Minibatch loss at step 1050: 0.475150\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.1%\n",
      "Minibatch loss at step 1100: 0.315644\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.1%\n",
      "Minibatch loss at step 1150: 0.819901\n",
      "Minibatch accuracy: 62.5%\n",
      "Validation accuracy: 83.1%\n",
      "Minibatch loss at step 1200: 0.096507\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 84.9%\n",
      "Minibatch loss at step 1250: 0.669541\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 1300: 0.623500\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.6%\n",
      "Minibatch loss at step 1350: 0.595296\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 1400: 0.635622\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 1450: 0.861819\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.0%\n",
      "Minibatch loss at step 1500: 0.446232\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 1550: 0.572067\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.5%\n",
      "Minibatch loss at step 1600: 0.200211\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 84.3%\n",
      "Minibatch loss at step 1650: 0.200968\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 1700: 0.263933\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 1750: 0.233538\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.4%\n",
      "Minibatch loss at step 1800: 0.448073\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.8%\n",
      "Minibatch loss at step 1850: 0.473885\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.2%\n",
      "Minibatch loss at step 1900: 0.750093\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 86.3%\n",
      "Minibatch loss at step 1950: 0.330217\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 2000: 0.612095\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.7%\n",
      "Minibatch loss at step 2050: 0.523626\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 2100: 0.411102\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.5%\n",
      "Minibatch loss at step 2150: 0.358303\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.7%\n",
      "Minibatch loss at step 2200: 0.255994\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 2250: 0.570838\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 2300: 0.392989\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.5%\n",
      "Minibatch loss at step 2350: 0.642448\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 2400: 0.202259\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.8%\n",
      "Minibatch loss at step 2450: 0.486015\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.5%\n",
      "Minibatch loss at step 2500: 0.391964\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.0%\n",
      "Minibatch loss at step 2550: 0.116222\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 2600: 0.201227\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.9%\n",
      "Minibatch loss at step 2650: 0.724193\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 2700: 0.611380\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 2750: 0.024032\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 86.8%\n",
      "Minibatch loss at step 2800: 0.163433\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 2850: 0.877795\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 86.2%\n",
      "Minibatch loss at step 2900: 0.057701\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 2950: 0.669788\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 3000: 0.306808\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 3050: 0.265149\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.1%\n",
      "Minibatch loss at step 3100: 0.282581\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.5%\n",
      "Minibatch loss at step 3150: 0.512961\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.5%\n",
      "Minibatch loss at step 3200: 0.308915\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 3250: 0.479315\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 3300: 0.285250\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.1%\n",
      "Minibatch loss at step 3350: 0.314332\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.8%\n",
      "Minibatch loss at step 3400: 0.462077\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 3450: 0.233312\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 3500: 0.426697\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 3550: 0.739429\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 3600: 0.719658\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 3650: 0.389150\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 3700: 0.170461\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 3750: 0.323007\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.2%\n",
      "Minibatch loss at step 3800: 0.534843\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.8%\n",
      "Minibatch loss at step 3850: 0.422596\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.4%\n",
      "Minibatch loss at step 3900: 0.309292\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 3950: 0.156567\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 4000: 0.379103\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.9%\n",
      "Minibatch loss at step 4050: 0.511346\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.2%\n",
      "Minibatch loss at step 4100: 0.137023\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.0%\n",
      "Minibatch loss at step 4150: 0.525313\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.1%\n",
      "Minibatch loss at step 4200: 0.295604\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.9%\n",
      "Minibatch loss at step 4250: 0.388546\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 4300: 0.108106\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 4350: 0.181506\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.1%\n",
      "Minibatch loss at step 4400: 0.129664\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.4%\n",
      "Minibatch loss at step 4450: 0.492655\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.1%\n",
      "Minibatch loss at step 4500: 0.162888\n",
      "Minibatch accuracy: 93.8%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation accuracy: 88.2%\n",
      "Minibatch loss at step 4550: 0.425939\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.5%\n",
      "Minibatch loss at step 4600: 0.536162\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.2%\n",
      "Minibatch loss at step 4650: 0.422463\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.9%\n",
      "Minibatch loss at step 4700: 0.642050\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 88.3%\n",
      "Minibatch loss at step 4750: 0.403506\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 4800: 0.342163\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.2%\n",
      "Minibatch loss at step 4850: 0.621787\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.4%\n",
      "Minibatch loss at step 4900: 0.085086\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.3%\n",
      "Minibatch loss at step 4950: 1.073914\n",
      "Minibatch accuracy: 62.5%\n",
      "Validation accuracy: 88.1%\n",
      "Minibatch loss at step 5000: 0.901499\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 87.8%\n",
      "Test accuracy: 93.5%\n"
     ]
    }
   ],
   "source": [
    "num_steps = 5001\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "  tf.global_variables_initializer().run()\n",
    "  print('Initialized')\n",
    "  for step in range(num_steps):\n",
    "    offset = (step * batch_size) % (train_labels.shape[0] - batch_size)\n",
    "    batch_data = train_dataset[offset:(offset + batch_size), :, :, :]\n",
    "    batch_labels = train_labels[offset:(offset + batch_size), :]\n",
    "    feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}\n",
    "    _, l, predictions = session.run(\n",
    "      [optimizer, loss, train_prediction], feed_dict=feed_dict)\n",
    "    if (step % 50 == 0):\n",
    "      print('Minibatch loss at step %d: %f' % (step, l))\n",
    "      print('Minibatch accuracy: %.1f%%' % accuracy(predictions, batch_labels))\n",
    "      print('Validation accuracy: %.1f%%' % accuracy(valid_prediction.eval(), valid_labels))\n",
    "  print('Test accuracy: %.1f%%' % accuracy(test_prediction.eval(), test_labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "klf21gpbAgb-"
   },
   "source": [
    "---\n",
    "Problem 2\n",
    "---------\n",
    "\n",
    "Try to get the best performance you can using a convolutional net. Look for example at the classic [LeNet5](http://yann.lecun.com/exdb/lenet/) architecture, adding Dropout, and/or adding learning rate decay.\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 两层卷积+两次pooling\n",
    "- learning rate decay\n",
    "- dropout (本次实验在对valid set和test set进行预测时也进行了dropout，导致结果低于预期)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "IZYv70SvvOan"
   },
   "outputs": [],
   "source": [
    "batch_size = 16\n",
    "patch_size = 5\n",
    "depth = 16\n",
    "num_hidden = 64\n",
    "keep_prob = 0.5\n",
    "graph = tf.Graph()\n",
    "\n",
    "with graph.as_default():\n",
    "\n",
    "  # Input data.\n",
    "  tf_train_dataset = tf.placeholder(\n",
    "    tf.float32, shape=(batch_size, image_size, image_size, num_channels))\n",
    "  tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))\n",
    "  tf_valid_dataset = tf.constant(valid_dataset)\n",
    "  tf_test_dataset = tf.constant(test_dataset)\n",
    "  \n",
    "  # Variables.\n",
    "  layer1_weights = tf.Variable(tf.truncated_normal(\n",
    "      [patch_size, patch_size, num_channels, depth], stddev=0.1))\n",
    "  layer1_biases = tf.Variable(tf.zeros([depth]))\n",
    "  layer2_weights = tf.Variable(tf.truncated_normal(\n",
    "      [patch_size, patch_size, depth, depth], stddev=0.1))\n",
    "  layer2_biases = tf.Variable(tf.constant(1.0, shape=[depth]))\n",
    "  \n",
    "  layer3_weights = tf.Variable(tf.truncated_normal(\n",
    "      [math.ceil(math.ceil(image_size//2/2)//2/2)**2*depth, num_hidden], stddev=0.1))\n",
    "  layer3_biases = tf.Variable(tf.constant(1.0, shape=[num_hidden]))\n",
    "  layer4_weights = tf.Variable(tf.truncated_normal(\n",
    "      [num_hidden, num_labels], stddev=0.1))\n",
    "  layer4_biases = tf.Variable(tf.constant(1.0, shape=[num_labels]))\n",
    "  \n",
    "  def max_pool_2x2(x):\n",
    "    return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],\n",
    "                            strides=[1, 2, 2, 1], padding='SAME')\n",
    "\n",
    "  # Model.\n",
    "  def model(data):\n",
    "    # strides: A list of ints. 1-D of length 4. The stride of the sliding window for each dimension of input\n",
    "    conv = tf.nn.conv2d(data, layer1_weights, strides=[1, 2, 2, 1], padding='SAME')\n",
    "    hidden = tf.nn.relu(conv + layer1_biases)\n",
    "    max_pool = max_pool_2x2(hidden)\n",
    "    conv = tf.nn.conv2d(max_pool, layer2_weights, strides=[1, 2, 2, 1], padding='SAME')\n",
    "    hidden = tf.nn.relu(conv + layer2_biases)\n",
    "    max_pool = max_pool_2x2(hidden)\n",
    "    shape = max_pool.get_shape().as_list()\n",
    "    reshape = tf.reshape(max_pool, [shape[0], shape[1] * shape[2] * shape[3]])\n",
    "    hidden = tf.nn.dropout(tf.nn.relu(tf.matmul(reshape, layer3_weights) + layer3_biases), keep_prob=keep_prob)\n",
    "    return tf.matmul(hidden, layer4_weights) + layer4_biases\n",
    "  \n",
    "  # Training computation.\n",
    "  logits = model(tf_train_dataset)\n",
    "  loss = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=tf_train_labels, logits=logits))\n",
    "    \n",
    "  # Optimizer.\n",
    "  \"\"\"\n",
    "  decayed_learning_rate = learning_rate *\n",
    "                        decay_rate ^ (global_step / decay_steps)\n",
    "  \"\"\"\n",
    "  global_step = tf.Variable(0, trainable=False)\n",
    "  starter_learning_rate = 0.05\n",
    "  learning_rate = tf.train.exponential_decay(starter_learning_rate, global_step, 1000, 0.7, staircase=True, name=None)\n",
    "  optimizer = tf.train.MomentumOptimizer(learning_rate, 0.6).minimize(loss)\n",
    "  \n",
    "  # Predictions for the training, validation, and test data.\n",
    "  train_prediction = tf.nn.softmax(logits)\n",
    "  valid_prediction = tf.nn.softmax(model(tf_valid_dataset))\n",
    "  test_prediction = tf.nn.softmax(model(tf_test_dataset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 37
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 63292,
     "status": "ok",
     "timestamp": 1446658966251,
     "user": {
      "color": "",
      "displayName": "",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "",
      "photoUrl": "",
      "sessionId": "0",
      "userId": ""
     },
     "user_tz": 480
    },
    "id": "noKFb2UovVFR",
    "outputId": "28941338-2ef9-4088-8bd1-44295661e628"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Minibatch loss at step 0: 2.235619\n",
      "Minibatch accuracy: 31.2%\n",
      "Validation accuracy: 9.8%\n",
      "Minibatch loss at step 50: 2.284733\n",
      "Minibatch accuracy: 6.2%\n",
      "Validation accuracy: 16.4%\n",
      "Minibatch loss at step 100: 1.961445\n",
      "Minibatch accuracy: 31.2%\n",
      "Validation accuracy: 24.2%\n",
      "Minibatch loss at step 150: 1.859371\n",
      "Minibatch accuracy: 62.5%\n",
      "Validation accuracy: 41.7%\n",
      "Minibatch loss at step 200: 2.233802\n",
      "Minibatch accuracy: 50.0%\n",
      "Validation accuracy: 54.5%\n",
      "Minibatch loss at step 250: 1.622989\n",
      "Minibatch accuracy: 43.8%\n",
      "Validation accuracy: 64.4%\n",
      "Minibatch loss at step 300: 1.368005\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 68.4%\n",
      "Minibatch loss at step 350: 0.610750\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 71.1%\n",
      "Minibatch loss at step 400: 0.944678\n",
      "Minibatch accuracy: 62.5%\n",
      "Validation accuracy: 72.1%\n",
      "Minibatch loss at step 450: 1.260144\n",
      "Minibatch accuracy: 56.2%\n",
      "Validation accuracy: 70.1%\n",
      "Minibatch loss at step 500: 1.256263\n",
      "Minibatch accuracy: 50.0%\n",
      "Validation accuracy: 73.0%\n",
      "Minibatch loss at step 550: 0.352455\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 72.8%\n",
      "Minibatch loss at step 600: 0.376505\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 75.9%\n",
      "Minibatch loss at step 650: 0.631303\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 77.9%\n",
      "Minibatch loss at step 700: 0.865404\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 78.2%\n",
      "Minibatch loss at step 750: 0.768189\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 78.5%\n",
      "Minibatch loss at step 800: 0.910604\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 78.1%\n",
      "Minibatch loss at step 850: 0.681066\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 79.5%\n",
      "Minibatch loss at step 900: 0.900412\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 79.2%\n",
      "Minibatch loss at step 950: 0.559118\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 78.6%\n",
      "Minibatch loss at step 1000: 0.429743\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 79.0%\n",
      "Minibatch loss at step 1050: 0.690756\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 79.5%\n",
      "Minibatch loss at step 1100: 0.480211\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 79.7%\n",
      "Minibatch loss at step 1150: 1.080285\n",
      "Minibatch accuracy: 56.2%\n",
      "Validation accuracy: 73.0%\n",
      "Minibatch loss at step 1200: 0.297516\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 78.0%\n",
      "Minibatch loss at step 1250: 0.565548\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 79.4%\n",
      "Minibatch loss at step 1300: 0.583179\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 80.0%\n",
      "Minibatch loss at step 1350: 0.782088\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 79.2%\n",
      "Minibatch loss at step 1400: 0.846590\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 79.7%\n",
      "Minibatch loss at step 1450: 1.020481\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 79.3%\n",
      "Minibatch loss at step 1500: 0.815932\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 80.0%\n",
      "Minibatch loss at step 1550: 0.528763\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 80.0%\n",
      "Minibatch loss at step 1600: 0.380372\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 80.4%\n",
      "Minibatch loss at step 1650: 0.485980\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 80.2%\n",
      "Minibatch loss at step 1700: 0.385496\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 80.5%\n",
      "Minibatch loss at step 1750: 0.387119\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 80.9%\n",
      "Minibatch loss at step 1800: 0.472867\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 81.8%\n",
      "Minibatch loss at step 1850: 0.705389\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 81.7%\n",
      "Minibatch loss at step 1900: 0.954256\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 81.6%\n",
      "Minibatch loss at step 1950: 0.169290\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 81.4%\n",
      "Minibatch loss at step 2000: 0.684597\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 80.2%\n",
      "Minibatch loss at step 2050: 0.610504\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 81.8%\n",
      "Minibatch loss at step 2100: 0.431250\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 81.8%\n",
      "Minibatch loss at step 2150: 0.619446\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 81.3%\n",
      "Minibatch loss at step 2200: 0.574782\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 81.0%\n",
      "Minibatch loss at step 2250: 0.776420\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 81.6%\n",
      "Minibatch loss at step 2300: 0.570985\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 82.8%\n",
      "Minibatch loss at step 2350: 0.610716\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 83.0%\n",
      "Minibatch loss at step 2400: 0.612912\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 82.4%\n",
      "Minibatch loss at step 2450: 0.702124\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 83.0%\n",
      "Minibatch loss at step 2500: 0.475843\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 81.2%\n",
      "Minibatch loss at step 2550: 0.370202\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 81.9%\n",
      "Minibatch loss at step 2600: 0.392474\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 82.2%\n",
      "Minibatch loss at step 2650: 1.139796\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 82.1%\n",
      "Minibatch loss at step 2700: 1.110504\n",
      "Minibatch accuracy: 62.5%\n",
      "Validation accuracy: 81.8%\n",
      "Minibatch loss at step 2750: 0.215954\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 82.7%\n",
      "Minibatch loss at step 2800: 0.486704\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 82.6%\n",
      "Minibatch loss at step 2850: 0.858987\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 82.1%\n",
      "Minibatch loss at step 2900: 0.099790\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 83.2%\n",
      "Minibatch loss at step 2950: 0.566049\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 81.8%\n",
      "Minibatch loss at step 3000: 0.649025\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 82.4%\n",
      "Minibatch loss at step 3050: 0.498530\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 82.2%\n",
      "Minibatch loss at step 3100: 0.232228\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 83.0%\n",
      "Minibatch loss at step 3150: 0.368403\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 82.3%\n",
      "Minibatch loss at step 3200: 0.347478\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 83.6%\n",
      "Minibatch loss at step 3250: 0.613566\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 83.4%\n",
      "Minibatch loss at step 3300: 0.563505\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 83.3%\n",
      "Minibatch loss at step 3350: 0.574451\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 81.8%\n",
      "Minibatch loss at step 3400: 0.865328\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 82.5%\n",
      "Minibatch loss at step 3450: 0.393722\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 81.5%\n",
      "Minibatch loss at step 3500: 0.551046\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 83.0%\n",
      "Minibatch loss at step 3550: 0.953592\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 82.3%\n",
      "Minibatch loss at step 3600: 0.997743\n",
      "Minibatch accuracy: 62.5%\n",
      "Validation accuracy: 83.8%\n",
      "Minibatch loss at step 3650: 0.505091\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 83.5%\n",
      "Minibatch loss at step 3700: 0.460699\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 83.1%\n",
      "Minibatch loss at step 3750: 0.436601\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.0%\n",
      "Minibatch loss at step 3800: 0.931299\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 83.1%\n",
      "Minibatch loss at step 3850: 0.830092\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 83.4%\n",
      "Minibatch loss at step 3900: 0.517966\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 83.4%\n",
      "Minibatch loss at step 3950: 0.156424\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 83.1%\n",
      "Minibatch loss at step 4000: 0.569764\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.2%\n",
      "Minibatch loss at step 4050: 0.854796\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 83.4%\n",
      "Minibatch loss at step 4100: 0.310626\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 83.5%\n",
      "Minibatch loss at step 4150: 0.665245\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 83.3%\n",
      "Minibatch loss at step 4200: 0.693851\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 83.7%\n",
      "Minibatch loss at step 4250: 0.441908\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 82.8%\n",
      "Minibatch loss at step 4300: 0.514641\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 83.2%\n",
      "Minibatch loss at step 4350: 0.274810\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 82.2%\n",
      "Minibatch loss at step 4400: 0.212389\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 83.9%\n",
      "Minibatch loss at step 4450: 0.479501\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 83.4%\n",
      "Minibatch loss at step 4500: 0.168860\n",
      "Minibatch accuracy: 93.8%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation accuracy: 83.2%\n",
      "Minibatch loss at step 4550: 0.625353\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 83.8%\n",
      "Minibatch loss at step 4600: 0.618900\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 83.7%\n",
      "Minibatch loss at step 4650: 0.440837\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.0%\n",
      "Minibatch loss at step 4700: 0.777104\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.1%\n",
      "Minibatch loss at step 4750: 0.401248\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 84.1%\n",
      "Minibatch loss at step 4800: 0.316511\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.1%\n",
      "Minibatch loss at step 4850: 1.213853\n",
      "Minibatch accuracy: 62.5%\n",
      "Validation accuracy: 82.6%\n",
      "Minibatch loss at step 4900: 0.226186\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 83.8%\n",
      "Minibatch loss at step 4950: 1.406403\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 83.0%\n",
      "Minibatch loss at step 5000: 1.080698\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 82.9%\n",
      "Minibatch loss at step 5050: 0.503354\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 5100: 1.982932\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 84.7%\n",
      "Minibatch loss at step 5150: 0.158554\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 84.2%\n",
      "Minibatch loss at step 5200: 0.968942\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 82.1%\n",
      "Minibatch loss at step 5250: 0.488268\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 83.4%\n",
      "Minibatch loss at step 5300: 0.614116\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 83.5%\n",
      "Minibatch loss at step 5350: 0.845804\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 84.3%\n",
      "Minibatch loss at step 5400: 0.326443\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 82.8%\n",
      "Minibatch loss at step 5450: 0.617128\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 83.3%\n",
      "Minibatch loss at step 5500: 0.493691\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.5%\n",
      "Minibatch loss at step 5550: 1.191608\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 5600: 0.612997\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 84.3%\n",
      "Minibatch loss at step 5650: 0.654427\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.1%\n",
      "Minibatch loss at step 5700: 0.688572\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 84.0%\n",
      "Minibatch loss at step 5750: 0.290691\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.0%\n",
      "Minibatch loss at step 5800: 0.268252\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 83.9%\n",
      "Minibatch loss at step 5850: 0.748609\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 84.0%\n",
      "Minibatch loss at step 5900: 0.458182\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.6%\n",
      "Minibatch loss at step 5950: 0.073069\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 84.6%\n",
      "Minibatch loss at step 6000: 0.679694\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.1%\n",
      "Minibatch loss at step 6050: 0.487792\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 83.5%\n",
      "Minibatch loss at step 6100: 0.180940\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 82.6%\n",
      "Minibatch loss at step 6150: 0.336750\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 83.6%\n",
      "Minibatch loss at step 6200: 0.992227\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 84.6%\n",
      "Minibatch loss at step 6250: 0.188067\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 84.7%\n",
      "Minibatch loss at step 6300: 1.526442\n",
      "Minibatch accuracy: 56.2%\n",
      "Validation accuracy: 83.6%\n",
      "Minibatch loss at step 6350: 0.209617\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 83.6%\n",
      "Minibatch loss at step 6400: 0.660999\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.0%\n",
      "Minibatch loss at step 6450: 0.429818\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.3%\n",
      "Minibatch loss at step 6500: 0.085739\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 84.7%\n",
      "Minibatch loss at step 6550: 0.369556\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.3%\n",
      "Minibatch loss at step 6600: 1.346669\n",
      "Minibatch accuracy: 62.5%\n",
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 6650: 0.319956\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.2%\n",
      "Minibatch loss at step 6700: 0.188237\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 83.4%\n",
      "Minibatch loss at step 6750: 0.526388\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.0%\n",
      "Minibatch loss at step 6800: 0.727979\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 82.9%\n",
      "Minibatch loss at step 6850: 0.850823\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.6%\n",
      "Minibatch loss at step 6900: 0.381899\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.9%\n",
      "Minibatch loss at step 6950: 0.293288\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.7%\n",
      "Minibatch loss at step 7000: 0.338373\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 83.1%\n",
      "Minibatch loss at step 7050: 0.122724\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 84.7%\n",
      "Minibatch loss at step 7100: 0.690798\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 83.3%\n",
      "Minibatch loss at step 7150: 0.908104\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 7200: 0.019264\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 84.5%\n",
      "Minibatch loss at step 7250: 0.192547\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 84.3%\n",
      "Minibatch loss at step 7300: 0.264594\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 7350: 1.358626\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 84.3%\n",
      "Minibatch loss at step 7400: 0.530508\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 81.8%\n",
      "Minibatch loss at step 7450: 1.101585\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 82.9%\n",
      "Minibatch loss at step 7500: 1.383607\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 84.1%\n",
      "Minibatch loss at step 7550: 0.608068\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.9%\n",
      "Minibatch loss at step 7600: 0.730083\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 7650: 0.412077\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 7700: 0.863108\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 84.6%\n",
      "Minibatch loss at step 7750: 0.367812\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.7%\n",
      "Minibatch loss at step 7800: 0.841317\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 84.1%\n",
      "Minibatch loss at step 7850: 0.261328\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 84.6%\n",
      "Minibatch loss at step 7900: 0.736192\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 7950: 0.100203\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 84.7%\n",
      "Minibatch loss at step 8000: 0.506440\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.7%\n",
      "Minibatch loss at step 8050: 0.312787\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 83.3%\n",
      "Minibatch loss at step 8100: 0.359051\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 83.7%\n",
      "Minibatch loss at step 8150: 0.371427\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.9%\n",
      "Minibatch loss at step 8200: 0.324789\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.3%\n",
      "Minibatch loss at step 8250: 0.568747\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.2%\n",
      "Minibatch loss at step 8300: 0.548661\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 83.2%\n",
      "Minibatch loss at step 8350: 0.801410\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 84.5%\n",
      "Minibatch loss at step 8400: 0.250598\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 84.2%\n",
      "Minibatch loss at step 8450: 0.557454\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.9%\n",
      "Minibatch loss at step 8500: 0.388657\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 8550: 0.535114\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 84.3%\n",
      "Minibatch loss at step 8600: 1.273828\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 84.0%\n",
      "Minibatch loss at step 8650: 0.249514\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 8700: 0.843125\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 8750: 0.419421\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 8800: 0.611296\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 8850: 0.505473\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.3%\n",
      "Minibatch loss at step 8900: 0.333733\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 8950: 1.153563\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 9000: 0.200538\n",
      "Minibatch accuracy: 100.0%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation accuracy: 85.3%\n",
      "Minibatch loss at step 9050: 0.656280\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 84.9%\n",
      "Minibatch loss at step 9100: 0.509453\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 9150: 0.105551\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 84.9%\n",
      "Minibatch loss at step 9200: 0.354775\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.7%\n",
      "Minibatch loss at step 9250: 0.422752\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.9%\n",
      "Minibatch loss at step 9300: 0.196326\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 84.7%\n",
      "Minibatch loss at step 9350: 0.273613\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.9%\n",
      "Minibatch loss at step 9400: 0.441600\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 9450: 0.194682\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 84.9%\n",
      "Minibatch loss at step 9500: 0.076287\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 84.4%\n",
      "Minibatch loss at step 9550: 0.552548\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 9600: 0.476038\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 9650: 0.482449\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 9700: 0.659781\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.9%\n",
      "Minibatch loss at step 9750: 0.308645\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.3%\n",
      "Minibatch loss at step 9800: 0.363910\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 9850: 0.465613\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.3%\n",
      "Minibatch loss at step 9900: 0.153353\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 83.9%\n",
      "Minibatch loss at step 9950: 0.726402\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 83.6%\n",
      "Minibatch loss at step 10000: 0.431461\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.1%\n",
      "Minibatch loss at step 10050: 0.149897\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 10100: 0.602456\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.3%\n",
      "Minibatch loss at step 10150: 0.640202\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 10200: 1.073947\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 84.5%\n",
      "Minibatch loss at step 10250: 0.462644\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 10300: 0.282292\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 10350: 0.197108\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 10400: 0.300342\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 10450: 0.724911\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 10500: 0.220977\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 10550: 0.389580\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.3%\n",
      "Minibatch loss at step 10600: 0.689481\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.9%\n",
      "Minibatch loss at step 10650: 0.362631\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.4%\n",
      "Minibatch loss at step 10700: 0.607751\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 10750: 0.296821\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 10800: 0.862858\n",
      "Minibatch accuracy: 56.2%\n",
      "Validation accuracy: 83.7%\n",
      "Minibatch loss at step 10850: 0.503394\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.1%\n",
      "Minibatch loss at step 10900: 0.693747\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 83.7%\n",
      "Minibatch loss at step 10950: 0.015466\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 85.1%\n",
      "Minibatch loss at step 11000: 0.663036\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.3%\n",
      "Minibatch loss at step 11050: 0.227912\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 11100: 0.298836\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.1%\n",
      "Minibatch loss at step 11150: 0.562318\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.0%\n",
      "Minibatch loss at step 11200: 1.031596\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.0%\n",
      "Minibatch loss at step 11250: 0.302258\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 11300: 0.883712\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 11350: 0.164025\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 84.6%\n",
      "Minibatch loss at step 11400: 0.872483\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 11450: 0.682500\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.1%\n",
      "Minibatch loss at step 11500: 0.565476\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.0%\n",
      "Minibatch loss at step 11550: 0.446696\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 11600: 0.281015\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 11650: 0.549608\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 11700: 0.612428\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.4%\n",
      "Minibatch loss at step 11750: 0.511991\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 11800: 0.441893\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.1%\n",
      "Minibatch loss at step 11850: 0.179836\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 11900: 0.129831\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 11950: 0.080957\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 85.3%\n",
      "Minibatch loss at step 12000: 0.346630\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.0%\n",
      "Minibatch loss at step 12050: 0.209117\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.1%\n",
      "Minibatch loss at step 12100: 0.274949\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 12150: 0.365944\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.0%\n",
      "Minibatch loss at step 12200: 0.047579\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 12250: 0.281942\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 84.4%\n",
      "Minibatch loss at step 12300: 0.208123\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 84.1%\n",
      "Minibatch loss at step 12350: 0.773248\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 12400: 0.232958\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 12450: 0.714710\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 85.0%\n",
      "Minibatch loss at step 12500: 0.267085\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 12550: 0.423042\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 12600: 1.020636\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 12650: 0.141357\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 84.4%\n",
      "Minibatch loss at step 12700: 0.641790\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 82.7%\n",
      "Minibatch loss at step 12750: 0.243805\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 12800: 0.811956\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 12850: 0.699074\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 12900: 0.464547\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 12950: 0.853365\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.7%\n",
      "Minibatch loss at step 13000: 0.448608\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 13050: 0.164233\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 84.9%\n",
      "Minibatch loss at step 13100: 0.439172\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 13150: 0.658286\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 13200: 0.553054\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 13250: 0.826905\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 13300: 0.536055\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 13350: 0.550764\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 86.3%\n",
      "Minibatch loss at step 13400: 0.412002\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 13450: 0.416705\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.4%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Minibatch loss at step 13500: 0.879190\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 13550: 0.018949\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 85.0%\n",
      "Minibatch loss at step 13600: 0.275998\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 13650: 0.459190\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 13700: 0.178370\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 86.2%\n",
      "Minibatch loss at step 13750: 0.548113\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 13800: 0.445429\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 13850: 0.410725\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.3%\n",
      "Minibatch loss at step 13900: 0.378451\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.7%\n",
      "Minibatch loss at step 13950: 0.354918\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.7%\n",
      "Minibatch loss at step 14000: 0.203081\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 14050: 0.356606\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 14100: 0.024684\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 14150: 0.439400\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 14200: 0.566897\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 14250: 0.355126\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 14300: 0.061674\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 14350: 0.623549\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 14400: 0.933606\n",
      "Minibatch accuracy: 62.5%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 14450: 0.558762\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 14500: 0.312154\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.3%\n",
      "Minibatch loss at step 14550: 0.282668\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 14600: 0.416596\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 14650: 0.530900\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 14700: 0.333920\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 14750: 0.338355\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 14800: 1.428597\n",
      "Minibatch accuracy: 62.5%\n",
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 14850: 0.617814\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 14900: 0.683974\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 14950: 0.236792\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 15000: 0.211769\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 15050: 0.527193\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.0%\n",
      "Minibatch loss at step 15100: 0.227599\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 15150: 0.461971\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.1%\n",
      "Minibatch loss at step 15200: 0.941461\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 85.3%\n",
      "Minibatch loss at step 15250: 0.560002\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 15300: 0.494718\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 15350: 0.323714\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 15400: 0.445652\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 15450: 0.042285\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 15500: 0.502259\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 15550: 0.493355\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.9%\n",
      "Minibatch loss at step 15600: 0.194169\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.2%\n",
      "Minibatch loss at step 15650: 0.064346\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 15700: 0.527622\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.3%\n",
      "Minibatch loss at step 15750: 0.455815\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.3%\n",
      "Minibatch loss at step 15800: 0.689122\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.7%\n",
      "Minibatch loss at step 15850: 1.021639\n",
      "Minibatch accuracy: 62.5%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 15900: 0.257492\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 15950: 0.566605\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.6%\n",
      "Minibatch loss at step 16000: 0.333634\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.4%\n",
      "Minibatch loss at step 16050: 0.609116\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 16100: 0.681841\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.0%\n",
      "Minibatch loss at step 16150: 0.404835\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.0%\n",
      "Minibatch loss at step 16200: 0.193443\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.5%\n",
      "Minibatch loss at step 16250: 0.752061\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 16300: 0.164453\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 84.1%\n",
      "Minibatch loss at step 16350: 0.297213\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 16400: 0.426768\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.4%\n",
      "Minibatch loss at step 16450: 0.532768\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.3%\n",
      "Minibatch loss at step 16500: 0.800459\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 16550: 0.913721\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 85.3%\n",
      "Minibatch loss at step 16600: 0.023409\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 16650: 0.427951\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 16700: 0.739817\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.1%\n",
      "Minibatch loss at step 16750: 0.572428\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 83.2%\n",
      "Minibatch loss at step 16800: 0.443929\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.4%\n",
      "Minibatch loss at step 16850: 0.579061\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 16900: 0.389420\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 16950: 0.386602\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.0%\n",
      "Minibatch loss at step 17000: 0.252725\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 17050: 0.342372\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 17100: 0.037798\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 17150: 0.596879\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 17200: 0.213891\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.3%\n",
      "Minibatch loss at step 17250: 0.376850\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 17300: 0.391941\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 17350: 0.493300\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 17400: 1.738647\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 17450: 0.423441\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 17500: 0.554807\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 17550: 0.444226\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 17600: 0.315602\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 17650: 0.385024\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 17700: 0.270947\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 17750: 0.198746\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.3%\n",
      "Minibatch loss at step 17800: 0.678014\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 82.2%\n",
      "Minibatch loss at step 17850: 0.187097\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 17900: 0.274312\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.1%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Minibatch loss at step 17950: 0.970277\n",
      "Minibatch accuracy: 62.5%\n",
      "Validation accuracy: 85.0%\n",
      "Minibatch loss at step 18000: 0.465509\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 18050: 0.400290\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 18100: 0.418137\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 18150: 1.555319\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 18200: 0.556370\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.5%\n",
      "Minibatch loss at step 18250: 0.231180\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 18300: 0.399547\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 18350: 0.303111\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 18400: 0.503002\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 18450: 0.589072\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 18500: 0.290172\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 18550: 0.597942\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 18600: 0.156909\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 18650: 0.253694\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 18700: 0.296774\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 18750: 0.903018\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 18800: 0.094745\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 18850: 0.505749\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 18900: 0.803597\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.0%\n",
      "Minibatch loss at step 18950: 0.048114\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 19000: 0.314821\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 84.6%\n",
      "Minibatch loss at step 19050: 0.132620\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 19100: 0.870005\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 19150: 0.153441\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.3%\n",
      "Minibatch loss at step 19200: 0.338537\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 19250: 0.857274\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 86.3%\n",
      "Minibatch loss at step 19300: 0.078861\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 82.0%\n",
      "Minibatch loss at step 19350: 0.240948\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 19400: 0.557952\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 19450: 0.592234\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 83.7%\n",
      "Minibatch loss at step 19500: 0.466992\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 19550: 0.073401\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 19600: 1.133110\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 19650: 0.223331\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 19700: 0.446915\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 19750: 0.423471\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.0%\n",
      "Minibatch loss at step 19800: 0.327729\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 19850: 0.420204\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 19900: 0.165306\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.1%\n",
      "Minibatch loss at step 19950: 0.379123\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 20000: 0.814937\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 20050: 0.937189\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 20100: 0.606337\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 20150: 0.424822\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 20200: 0.264329\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 20250: 0.757814\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 20300: 0.274875\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 20350: 0.730126\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 20400: 0.213837\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 20450: 0.434147\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 20500: 0.738010\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.3%\n",
      "Minibatch loss at step 20550: 0.267690\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 20600: 0.222324\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.2%\n",
      "Minibatch loss at step 20650: 0.998606\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 20700: 0.343079\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 20750: 0.858093\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 20800: 0.318095\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 20850: 0.538677\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 86.3%\n",
      "Minibatch loss at step 20900: 0.626064\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 20950: 0.341454\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 21000: 0.236950\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.2%\n",
      "Minibatch loss at step 21050: 0.565221\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 21100: 0.535379\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 21150: 0.261666\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.4%\n",
      "Minibatch loss at step 21200: 0.835443\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 21250: 0.460451\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.5%\n",
      "Minibatch loss at step 21300: 0.809336\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 21350: 0.381804\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 21400: 0.410116\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.3%\n",
      "Minibatch loss at step 21450: 0.734804\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 21500: 0.595295\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 86.8%\n",
      "Minibatch loss at step 21550: 0.652171\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 21600: 0.226985\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 21650: 0.297348\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 21700: 0.870589\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 21750: 0.447934\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 21800: 2.039473\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 21850: 0.370195\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 21900: 0.272405\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 21950: 0.506643\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.7%\n",
      "Minibatch loss at step 22000: 0.845934\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 86.2%\n",
      "Minibatch loss at step 22050: 0.451427\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 22100: 0.453787\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 22150: 0.734889\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 22200: 0.648221\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.1%\n",
      "Minibatch loss at step 22250: 0.794836\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 22300: 0.118104\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 86.2%\n",
      "Minibatch loss at step 22350: 0.373012\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.9%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Minibatch loss at step 22400: 0.036640\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 22450: 0.223064\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 22500: 0.391017\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 22550: 0.157726\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 22600: 0.277832\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 84.9%\n",
      "Minibatch loss at step 22650: 0.365843\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 22700: 0.968999\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.1%\n",
      "Minibatch loss at step 22750: 0.250326\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 22800: 0.398456\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 22850: 0.329451\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.2%\n",
      "Minibatch loss at step 22900: 0.164185\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 22950: 0.593206\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.2%\n",
      "Minibatch loss at step 23000: 0.625843\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 86.3%\n",
      "Minibatch loss at step 23050: 0.466252\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 23100: 0.529231\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 23150: 0.164904\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.3%\n",
      "Minibatch loss at step 23200: 0.226777\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 23250: 0.532567\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 23300: 0.518216\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.7%\n",
      "Minibatch loss at step 23350: 0.269855\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 23400: 0.372163\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 23450: 0.264201\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.3%\n",
      "Minibatch loss at step 23500: 0.828365\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 23550: 0.377030\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 23600: 2.728842\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 23650: 0.637706\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 81.9%\n",
      "Minibatch loss at step 23700: 0.899294\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 23750: 0.452121\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 23800: 0.694915\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 23850: 0.920102\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 23900: 0.513972\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.1%\n",
      "Minibatch loss at step 23950: 0.583965\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 24000: 0.410197\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 24050: 0.290744\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 84.9%\n",
      "Minibatch loss at step 24100: 0.530864\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.0%\n",
      "Minibatch loss at step 24150: 0.219064\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 24200: 0.298812\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 24250: 0.175132\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 24300: 0.148972\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 24350: 0.582776\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 24400: 0.192694\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 85.0%\n",
      "Minibatch loss at step 24450: 0.173929\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 24500: 0.487679\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 24550: 0.169116\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 24600: 0.015161\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 24650: 0.525607\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 24700: 0.428188\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 24750: 0.119012\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 24800: 0.326927\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.3%\n",
      "Minibatch loss at step 24850: 0.806271\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.2%\n",
      "Minibatch loss at step 24900: 0.599873\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 84.9%\n",
      "Minibatch loss at step 24950: 0.227969\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 25000: 0.238966\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 25050: 1.625549\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 25100: 0.458800\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 25150: 0.715091\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 25200: 0.254902\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 25250: 0.335817\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 25300: 0.208498\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 25350: 0.474326\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 25400: 0.115880\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 25450: 0.352605\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 25500: 0.504969\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 25550: 0.624117\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 25600: 0.379095\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 25650: 0.254717\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 25700: 0.329709\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.5%\n",
      "Minibatch loss at step 25750: 0.322992\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 25800: 0.757755\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 25850: 0.516911\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 25900: 0.230422\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 25950: 0.467925\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 26000: 0.629798\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.1%\n",
      "Minibatch loss at step 26050: 0.285755\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 26100: 0.610378\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 26150: 0.179378\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 26200: 0.147957\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.7%\n",
      "Minibatch loss at step 26250: 0.155306\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.7%\n",
      "Minibatch loss at step 26300: 0.583012\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.0%\n",
      "Minibatch loss at step 26350: 0.378861\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 26400: 0.331609\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 26450: 0.454334\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 26500: 1.024967\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 26550: 0.425297\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.2%\n",
      "Minibatch loss at step 26600: 0.296000\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 26650: 1.029576\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 85.0%\n",
      "Minibatch loss at step 26700: 0.253695\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 26750: 0.360315\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.2%\n",
      "Minibatch loss at step 26800: 0.410504\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.1%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Minibatch loss at step 26850: 0.261990\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.3%\n",
      "Minibatch loss at step 26900: 0.242819\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.5%\n",
      "Minibatch loss at step 26950: 0.624164\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 27000: 0.432986\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.3%\n",
      "Minibatch loss at step 27050: 0.400469\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 27100: 0.230709\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 27150: 0.346738\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 27200: 1.252655\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.1%\n",
      "Minibatch loss at step 27250: 1.256744\n",
      "Minibatch accuracy: 62.5%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 27300: 0.368849\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 27350: 0.677890\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 27400: 0.476858\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 27450: 0.404956\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.4%\n",
      "Minibatch loss at step 27500: 0.852760\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 27550: 0.233865\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 27600: 0.867276\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 27650: 0.330580\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.2%\n",
      "Minibatch loss at step 27700: 0.340638\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 27750: 0.728167\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 27800: 0.184685\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 86.7%\n",
      "Minibatch loss at step 27850: 0.531934\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.2%\n",
      "Minibatch loss at step 27900: 0.112355\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 27950: 0.273347\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 28000: 0.294571\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 28050: 0.362812\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.3%\n",
      "Minibatch loss at step 28100: 0.695903\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 28150: 0.430044\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.6%\n",
      "Minibatch loss at step 28200: 0.734708\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 28250: 0.417508\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 28300: 0.451697\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 28350: 0.254502\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 28400: 0.444757\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 28450: 0.495160\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.9%\n",
      "Minibatch loss at step 28500: 0.585592\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 28550: 0.218902\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 28600: 0.079025\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.8%\n",
      "Minibatch loss at step 28650: 0.429561\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.3%\n",
      "Minibatch loss at step 28700: 0.336182\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 28750: 0.617073\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 28800: 0.732648\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 28850: 0.805935\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 28900: 0.756809\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 28950: 0.390625\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 29000: 0.444963\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.6%\n",
      "Minibatch loss at step 29050: 0.398314\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.5%\n",
      "Minibatch loss at step 29100: 0.149827\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.7%\n",
      "Minibatch loss at step 29150: 0.188682\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 29200: 0.496888\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.1%\n",
      "Minibatch loss at step 29250: 0.342874\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 29300: 0.593106\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.9%\n",
      "Minibatch loss at step 29350: 0.390009\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 29400: 0.832002\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 29450: 0.583232\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.2%\n",
      "Minibatch loss at step 29500: 0.353765\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 29550: 0.639501\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 29600: 0.014889\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 29650: 0.844025\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 29700: 0.679339\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 29750: 0.249423\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 84.1%\n",
      "Minibatch loss at step 29800: 0.575206\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.3%\n",
      "Minibatch loss at step 29850: 0.952782\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 85.0%\n",
      "Minibatch loss at step 29900: 0.260304\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.3%\n",
      "Minibatch loss at step 29950: 0.745361\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 30000: 0.677678\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 84.8%\n",
      "Test accuracy: 91.1%\n"
     ]
    }
   ],
   "source": [
    "num_steps = 30001\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "  tf.global_variables_initializer().run()\n",
    "  print('Initialized')\n",
    "  for step in range(num_steps):\n",
    "    offset = (step译文 * batch_size) % (train_labels.shape[0] - batch_size)\n",
    "    batch_data = train_dataset[offset:(offset + batch_size), :, :, :]\n",
    "    batch_labels = train_labels[offset:(offset + batch_size), :]\n",
    "    feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}\n",
    "    _, l, predictions = session.run(\n",
    "      [optimizer, loss, train_prediction], feed_dict=feed_dict)\n",
    "    if (step % 50 == 0):\n",
    "      print('Minibatch loss at step %d: %f' % (step, l))\n",
    "      print('Minibatch accuracy: %.1f%%' % accuracy(predictions, batch_labels))\n",
    "      print('Validation accuracy: %.1f%%' % accuracy(\n",
    "        valid_prediction.eval(), valid_labels))\n",
    "  print('Test accuracy: %.1f%%' % accuracy(test_prediction.eval(), test_labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 两层卷积+两次pooling¶\n",
    "- learning rate decay\n",
    "- dropout (本次实验在对valid set和test set进行预测时,去掉dropout了（理应如此）)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "IZYv70SvvOan"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\n"
     ]
    }
   ],
   "source": [
    "batch_size = 16\n",
    "patch_size = 5\n",
    "depth = 16\n",
    "num_hidden = 64\n",
    "keep_prob = 0.8\n",
    "graph = tf.Graph()\n",
    "\n",
    "with graph.as_default():\n",
    "\n",
    "  # Input data.\n",
    "  tf_train_dataset = tf.placeholder(\n",
    "    tf.float32, shape=(batch_size, image_size, image_size, num_channels))\n",
    "  tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))\n",
    "  tf_valid_dataset = tf.constant(valid_dataset)\n",
    "  tf_test_dataset = tf.constant(test_dataset)\n",
    "  \n",
    "  # Variables.\n",
    "  layer1_weights = tf.Variable(tf.truncated_normal(\n",
    "      [patch_size, patch_size, num_channels, depth], stddev=0.1))\n",
    "  layer1_biases = tf.Variable(tf.zeros([depth]))\n",
    "  layer2_weights = tf.Variable(tf.truncated_normal(\n",
    "      [patch_size, patch_size, depth, depth], stddev=0.1))\n",
    "  layer2_biases = tf.Variable(tf.constant(1.0, shape=[depth]))\n",
    "  \n",
    "  new_size = math.ceil((((math.ceil(((image_size-patch_size+1)-2+1)/2)-patch_size+1)) - 2 + 1) / 2)\n",
    "  print(new_size)\n",
    "  layer3_weights = tf.Variable(tf.truncated_normal(\n",
    "      [new_size**2*depth, num_hidden], stddev=0.1))\n",
    "  layer3_biases = tf.Variable(tf.constant(1.0, shape=[num_hidden]))\n",
    "  layer4_weights = tf.Variable(tf.truncated_normal(\n",
    "      [num_hidden, num_labels], stddev=0.1))\n",
    "  layer4_biases = tf.Variable(tf.constant(1.0, shape=[num_labels]))\n",
    "  \n",
    "  def max_pool_2x2(x):\n",
    "    return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],\n",
    "                            strides=[1, 2, 2, 1], padding='VALID')\n",
    "\n",
    "  # Model.\n",
    "  def model(data):\n",
    "    # strides: A list of ints. 1-D of length 4. The stride of the sliding window for each dimension of input\n",
    "    # input 28 * 28\n",
    "    conv = tf.nn.conv2d(data, layer1_weights, strides=[1, 1, 1, 1], padding='VALID')\n",
    "    hidden = tf.nn.relu(conv + layer1_biases)\n",
    "    # 28 - 5(patch_size) + 1 = 24, input 24 * 24\n",
    "    max_pool = max_pool_2x2(hidden)\n",
    "    # ceil((24 - 2 + 1) / 2) = 12, input 12 * 12 \n",
    "    conv = tf.nn.conv2d(max_pool, layer2_weights, strides=[1, 1, 1, 1], padding='VALID')\n",
    "    hidden = tf.nn.relu(conv + layer2_biases)\n",
    "    # 12 - 5 + 1 = 8, input 8 * 8\n",
    "    max_pool = max_pool_2x2(hidden)\n",
    "    # ceil((8 - 2 + 1) / 2) = 4, input 4 * 4 \n",
    "    shape = max_pool.get_shape().as_list()\n",
    "    reshape = tf.reshape(max_pool, [shape[0], shape[1] * shape[2] * shape[3]])\n",
    "    hidden = tf.nn.dropout(tf.nn.relu(tf.matmul(reshape, layer3_weights) + layer3_biases), keep_prob=keep_prob)\n",
    "    return tf.matmul(hidden, layer4_weights) + layer4_biases\n",
    "  \n",
    "  # Training computation.\n",
    "  logits = model(tf_train_dataset)\n",
    "  loss = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=tf_train_labels, logits=logits))\n",
    "    \n",
    "  # Optimizer.\n",
    "  \"\"\"\n",
    "  decayed_learning_rate = learning_rate *\n",
    "                        decay_rate ^ (global_step / decay_steps)\n",
    "  \"\"\"\n",
    "  global_step = tf.Variable(0, trainable=False)\n",
    "  starter_learning_rate = 0.05\n",
    "  learning_rate = tf.train.exponential_decay(starter_learning_rate, global_step, 100, 0.37, staircase=True, name=None)\n",
    "  optimizer = tf.train.MomentumOptimizer(learning_rate, 0.6).minimize(loss)\n",
    "  \n",
    "  # Predictions for the training, validation, and test data.\n",
    "  train_prediction = tf.nn.softmax(logits)\n",
    "  valid_prediction = tf.nn.softmax(model(tf_valid_dataset))\n",
    "  test_prediction = tf.nn.softmax(model(tf_test_dataset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 37
      }
     ]
    },
    "colab_type": "code",
    "collapsed": true,
    "executionInfo": {
     "elapsed": 63292,
     "status": "ok",
     "timestamp": 1446658966251,
     "user": {
      "color": "",
      "displayName": "",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "",
      "photoUrl": "",
      "sessionId": "0",
      "userId": ""
     },
     "user_tz": 480
    },
    "id": "noKFb2UovVFR",
    "outputId": "28941338-2ef9-4088-8bd1-44295661e628"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Minibatch loss at step 0: 2.759915\n",
      "Minibatch accuracy: 18.8%\n",
      "Validation accuracy: 10.2%\n",
      "Minibatch loss at step 50: 0.980603\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 44.3%\n",
      "Minibatch loss at step 100: 1.773895\n",
      "Minibatch accuracy: 37.5%\n",
      "Validation accuracy: 61.5%\n",
      "Minibatch loss at step 150: 1.408789\n",
      "Minibatch accuracy: 50.0%\n",
      "Validation accuracy: 69.8%\n",
      "Minibatch loss at step 200: 1.232012\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 74.8%\n",
      "Minibatch loss at step 250: 1.298495\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 74.0%\n",
      "Minibatch loss at step 300: 1.401020\n",
      "Minibatch accuracy: 62.5%\n",
      "Validation accuracy: 76.3%\n",
      "Minibatch loss at step 350: 0.533752\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 78.7%\n",
      "Minibatch loss at step 400: 1.027910\n",
      "Minibatch accuracy: 62.5%\n",
      "Validation accuracy: 79.3%\n",
      "Minibatch loss at step 450: 1.132483\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 75.7%\n",
      "Minibatch loss at step 500: 0.957476\n",
      "Minibatch accuracy: 62.5%\n",
      "Validation accuracy: 79.0%\n",
      "Minibatch loss at step 550: 0.401292\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 80.6%\n",
      "Minibatch loss at step 600: 0.236477\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 80.8%\n",
      "Minibatch loss at step 650: 0.926551\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 81.7%\n",
      "Minibatch loss at step 700: 0.601172\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 80.7%\n",
      "Minibatch loss at step 750: 0.764005\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 81.8%\n",
      "Minibatch loss at step 800: 1.188225\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 80.5%\n",
      "Minibatch loss at step 850: 0.661960\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 83.0%\n",
      "Minibatch loss at step 900: 0.681843\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 81.6%\n",
      "Minibatch loss at step 950: 0.405050\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 81.1%\n",
      "Minibatch loss at step 1000: 0.247783\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 82.9%\n",
      "Minibatch loss at step 1050: 0.856828\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 82.4%\n",
      "Minibatch loss at step 1100: 0.499757\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 82.8%\n",
      "Minibatch loss at step 1150: 0.552871\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 82.9%\n",
      "Minibatch loss at step 1200: 0.086052\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 84.0%\n",
      "Minibatch loss at step 1250: 0.634734\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 83.0%\n",
      "Minibatch loss at step 1300: 0.706561\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 83.8%\n",
      "Minibatch loss at step 1350: 0.816751\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 83.0%\n",
      "Minibatch loss at step 1400: 0.764521\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 82.5%\n",
      "Minibatch loss at step 1450: 1.109110\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 81.5%\n",
      "Minibatch loss at step 1500: 0.683487\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 83.7%\n",
      "Minibatch loss at step 1550: 0.561501\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 83.8%\n",
      "Minibatch loss at step 1600: 0.252817\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 81.9%\n",
      "Minibatch loss at step 1650: 0.304851\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 83.1%\n",
      "Minibatch loss at step 1700: 0.415075\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 82.0%\n",
      "Minibatch loss at step 1750: 0.286359\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 83.9%\n",
      "Minibatch loss at step 1800: 0.556151\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 81.8%\n",
      "Minibatch loss at step 1850: 0.521484\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 83.9%\n",
      "Minibatch loss at step 1900: 0.717920\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 83.7%\n",
      "Minibatch loss at step 1950: 0.270111\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 83.7%\n",
      "Minibatch loss at step 2000: 0.802419\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 84.7%\n",
      "Minibatch loss at step 2050: 0.986103\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 83.5%\n",
      "Minibatch loss at step 2100: 0.300909\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 84.4%\n",
      "Minibatch loss at step 2150: 0.632537\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.1%\n",
      "Minibatch loss at step 2200: 0.366906\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.3%\n",
      "Minibatch loss at step 2250: 1.019583\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 83.3%\n",
      "Minibatch loss at step 2300: 0.596697\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.2%\n",
      "Minibatch loss at step 2350: 0.752132\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 82.8%\n",
      "Minibatch loss at step 2400: 0.400854\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 82.8%\n",
      "Minibatch loss at step 2450: 0.413662\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 2500: 0.397650\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 83.9%\n",
      "Minibatch loss at step 2550: 0.212567\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 84.0%\n",
      "Minibatch loss at step 2600: 0.351052\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 84.7%\n",
      "Minibatch loss at step 2650: 0.625029\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.4%\n",
      "Minibatch loss at step 2700: 0.678806\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.5%\n",
      "Minibatch loss at step 2750: 0.013380\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 85.0%\n",
      "Minibatch loss at step 2800: 0.215786\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 2850: 1.244095\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 83.2%\n",
      "Minibatch loss at step 2900: 0.089368\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 83.9%\n",
      "Minibatch loss at step 2950: 0.476764\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.0%\n",
      "Minibatch loss at step 3000: 0.528736\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 3050: 0.326208\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.1%\n",
      "Minibatch loss at step 3100: 0.425554\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 84.9%\n",
      "Minibatch loss at step 3150: 0.472587\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.4%\n",
      "Minibatch loss at step 3200: 0.278164\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 3250: 0.368150\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.3%\n",
      "Minibatch loss at step 3300: 0.404771\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.0%\n",
      "Minibatch loss at step 3350: 0.292715\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 85.1%\n",
      "Minibatch loss at step 3400: 0.568562\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 3450: 0.243556\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 3500: 0.800758\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 3550: 0.528237\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 3600: 0.627735\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 3650: 0.558238\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 3700: 0.541189\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 3750: 0.519894\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.3%\n",
      "Minibatch loss at step 3800: 0.688858\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 3850: 0.360396\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 3900: 0.348717\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.3%\n",
      "Minibatch loss at step 3950: 0.244156\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.4%\n",
      "Minibatch loss at step 4000: 0.427176\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.2%\n",
      "Minibatch loss at step 4050: 0.882019\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 4100: 0.307813\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 4150: 0.592996\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 4200: 0.786783\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.0%\n",
      "Minibatch loss at step 4250: 0.602602\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.3%\n",
      "Minibatch loss at step 4300: 0.158391\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 4350: 0.209691\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 4400: 0.208307\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 4450: 0.365568\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 4500: 0.204148\n",
      "Minibatch accuracy: 93.8%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 4550: 0.318774\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.2%\n",
      "Minibatch loss at step 4600: 0.576124\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 4650: 0.838885\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.4%\n",
      "Minibatch loss at step 4700: 0.677872\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 4750: 0.515390\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.4%\n",
      "Minibatch loss at step 4800: 0.125188\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 4850: 0.782236\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 4900: 0.304234\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.4%\n",
      "Minibatch loss at step 4950: 1.137106\n",
      "Minibatch accuracy: 56.2%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 5000: 0.708375\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 5050: 0.555777\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.1%\n",
      "Minibatch loss at step 5100: 1.340143\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 5150: 0.157840\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 84.7%\n",
      "Minibatch loss at step 5200: 0.521451\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 5250: 0.492681\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 5300: 0.253611\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 5350: 1.049592\n",
      "Minibatch accuracy: 62.5%\n",
      "Validation accuracy: 86.5%\n",
      "Minibatch loss at step 5400: 0.179072\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 5450: 0.523921\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 5500: 0.429296\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.2%\n",
      "Minibatch loss at step 5550: 1.025104\n",
      "Minibatch accuracy: 62.5%\n",
      "Validation accuracy: 86.2%\n",
      "Minibatch loss at step 5600: 0.636787\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 5650: 0.591387\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 5700: 0.790583\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 5750: 0.131213\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 5800: 0.388379\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.5%\n",
      "Minibatch loss at step 5850: 0.715573\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 86.5%\n",
      "Minibatch loss at step 5900: 0.244276\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.6%\n",
      "Minibatch loss at step 5950: 0.060044\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 86.2%\n",
      "Minibatch loss at step 6000: 0.723543\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 85.0%\n",
      "Minibatch loss at step 6050: 0.093305\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 86.7%\n",
      "Minibatch loss at step 6100: 0.072889\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 86.3%\n",
      "Minibatch loss at step 6150: 0.232287\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 6200: 0.758188\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 6250: 0.155826\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 6300: 0.692558\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 86.5%\n",
      "Minibatch loss at step 6350: 0.141339\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.0%\n",
      "Minibatch loss at step 6400: 0.463456\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.5%\n",
      "Minibatch loss at step 6450: 0.658656\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.3%\n",
      "Minibatch loss at step 6500: 0.240787\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.0%\n",
      "Minibatch loss at step 6550: 0.416044\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.7%\n",
      "Minibatch loss at step 6600: 0.632111\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 86.3%\n",
      "Minibatch loss at step 6650: 0.218707\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.4%\n",
      "Minibatch loss at step 6700: 0.344889\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.5%\n",
      "Minibatch loss at step 6750: 0.902787\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 6800: 0.820122\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 86.5%\n",
      "Minibatch loss at step 6850: 0.325283\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.6%\n",
      "Minibatch loss at step 6900: 0.401023\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 6950: 0.230397\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.4%\n",
      "Minibatch loss at step 7000: 0.210052\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.6%\n",
      "Minibatch loss at step 7050: 0.228036\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 7100: 0.395953\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 7150: 0.520641\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.5%\n",
      "Minibatch loss at step 7200: 0.020366\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 7250: 0.249454\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.4%\n",
      "Minibatch loss at step 7300: 0.420894\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.6%\n",
      "Minibatch loss at step 7350: 0.668939\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.5%\n",
      "Minibatch loss at step 7400: 0.092814\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 86.3%\n",
      "Minibatch loss at step 7450: 0.782889\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.9%\n",
      "Minibatch loss at step 7500: 0.683371\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.8%\n",
      "Minibatch loss at step 7550: 0.357813\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 7600: 0.785632\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 86.6%\n",
      "Minibatch loss at step 7650: 0.489436\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.5%\n",
      "Minibatch loss at step 7700: 0.633043\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 86.2%\n",
      "Minibatch loss at step 7750: 0.114211\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.6%\n",
      "Minibatch loss at step 7800: 0.649271\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 7850: 0.379286\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.0%\n",
      "Minibatch loss at step 7900: 0.978705\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 86.7%\n",
      "Minibatch loss at step 7950: 0.018537\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 8000: 0.261287\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.0%\n",
      "Minibatch loss at step 8050: 0.341234\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.3%\n",
      "Minibatch loss at step 8100: 0.441181\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 8150: 0.356614\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 87.1%\n",
      "Minibatch loss at step 8200: 0.067460\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 86.4%\n",
      "Minibatch loss at step 8250: 0.678472\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 8300: 0.521904\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.2%\n",
      "Minibatch loss at step 8350: 0.656939\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.6%\n",
      "Minibatch loss at step 8400: 0.124237\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 86.4%\n",
      "Minibatch loss at step 8450: 0.587474\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.9%\n",
      "Minibatch loss at step 8500: 0.301042\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.0%\n",
      "Minibatch loss at step 8550: 0.598354\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.1%\n",
      "Minibatch loss at step 8600: 0.461318\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.8%\n",
      "Minibatch loss at step 8650: 0.333784\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.6%\n",
      "Minibatch loss at step 8700: 1.099809\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 87.0%\n",
      "Minibatch loss at step 8750: 0.527502\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.7%\n",
      "Minibatch loss at step 8800: 0.548433\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.5%\n",
      "Minibatch loss at step 8850: 0.476959\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.1%\n",
      "Minibatch loss at step 8900: 0.103194\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 8950: 1.082359\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 87.1%\n",
      "Minibatch loss at step 9000: 0.067547\n",
      "Minibatch accuracy: 93.8%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 9050: 0.401951\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.8%\n",
      "Minibatch loss at step 9100: 0.279899\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.5%\n",
      "Minibatch loss at step 9150: 0.011444\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.1%\n",
      "Minibatch loss at step 9200: 0.514852\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.6%\n",
      "Minibatch loss at step 9250: 0.436558\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.0%\n",
      "Minibatch loss at step 9300: 0.267190\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 9350: 0.324063\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.4%\n",
      "Minibatch loss at step 9400: 0.339728\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 9450: 0.126341\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.1%\n",
      "Minibatch loss at step 9500: 0.149018\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.1%\n",
      "Minibatch loss at step 9550: 0.696271\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.0%\n",
      "Minibatch loss at step 9600: 0.472898\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.2%\n",
      "Minibatch loss at step 9650: 0.400660\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.8%\n",
      "Minibatch loss at step 9700: 0.391154\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.9%\n",
      "Minibatch loss at step 9750: 0.240715\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.0%\n",
      "Minibatch loss at step 9800: 0.230723\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.0%\n",
      "Minibatch loss at step 9850: 0.322132\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.1%\n",
      "Minibatch loss at step 9900: 0.114271\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.2%\n",
      "Minibatch loss at step 9950: 0.936836\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 10000: 0.365583\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 10050: 0.081455\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 86.8%\n",
      "Minibatch loss at step 10100: 0.518379\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.8%\n",
      "Minibatch loss at step 10150: 0.488702\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 10200: 1.071624\n",
      "Minibatch accuracy: 62.5%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 10250: 0.514503\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.5%\n",
      "Minibatch loss at step 10300: 0.512922\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.8%\n",
      "Minibatch loss at step 10350: 0.246571\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.4%\n",
      "Minibatch loss at step 10400: 0.059614\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 86.2%\n",
      "Minibatch loss at step 10450: 0.483851\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 10500: 0.204697\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.5%\n",
      "Minibatch loss at step 10550: 0.238218\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 10600: 0.504840\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 10650: 0.108406\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.5%\n",
      "Minibatch loss at step 10700: 0.609771\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 87.5%\n",
      "Minibatch loss at step 10750: 0.348562\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 10800: 0.501359\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.9%\n",
      "Minibatch loss at step 10850: 0.134827\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 10900: 0.432991\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.8%\n",
      "Minibatch loss at step 10950: 0.149058\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 86.8%\n",
      "Minibatch loss at step 11000: 0.329141\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.0%\n",
      "Minibatch loss at step 11050: 0.343788\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.6%\n",
      "Minibatch loss at step 11100: 0.213095\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 84.9%\n",
      "Minibatch loss at step 11150: 0.484259\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.5%\n",
      "Minibatch loss at step 11200: 0.638335\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.5%\n",
      "Minibatch loss at step 11250: 0.340044\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.4%\n",
      "Minibatch loss at step 11300: 0.603413\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 11350: 0.106484\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 86.9%\n",
      "Minibatch loss at step 11400: 1.369937\n",
      "Minibatch accuracy: 56.2%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 11450: 0.767358\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 87.1%\n",
      "Minibatch loss at step 11500: 0.526123\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 11550: 0.390081\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 11600: 0.180011\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.7%\n",
      "Minibatch loss at step 11650: 0.673476\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.5%\n",
      "Minibatch loss at step 11700: 0.258079\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.8%\n",
      "Minibatch loss at step 11750: 0.515419\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.9%\n",
      "Minibatch loss at step 11800: 0.202371\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 11850: 0.053904\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 86.9%\n",
      "Minibatch loss at step 11900: 0.383900\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.9%\n",
      "Minibatch loss at step 11950: 0.251142\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 12000: 0.427602\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.1%\n",
      "Minibatch loss at step 12050: 0.401481\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.3%\n",
      "Minibatch loss at step 12100: 0.311651\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 12150: 0.332751\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.7%\n",
      "Minibatch loss at step 12200: 0.017127\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 12250: 0.484528\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.5%\n",
      "Minibatch loss at step 12300: 0.272341\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.8%\n",
      "Minibatch loss at step 12350: 0.409428\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.1%\n",
      "Minibatch loss at step 12400: 0.236797\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.9%\n",
      "Minibatch loss at step 12450: 0.506610\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 87.5%\n",
      "Minibatch loss at step 12500: 0.333453\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 12550: 0.275431\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 12600: 0.867217\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 12650: 0.020227\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.0%\n",
      "Minibatch loss at step 12700: 0.441903\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 12750: 0.205111\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.4%\n",
      "Minibatch loss at step 12800: 0.662325\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 86.6%\n",
      "Minibatch loss at step 12850: 0.867773\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 12900: 0.391809\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.4%\n",
      "Minibatch loss at step 12950: 0.758461\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.7%\n",
      "Minibatch loss at step 13000: 0.212987\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.0%\n",
      "Minibatch loss at step 13050: 0.312250\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.8%\n",
      "Minibatch loss at step 13100: 0.371805\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 13150: 0.521922\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 13200: 0.687609\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 86.3%\n",
      "Minibatch loss at step 13250: 0.382951\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.4%\n",
      "Minibatch loss at step 13300: 0.093142\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.9%\n",
      "Minibatch loss at step 13350: 0.660818\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 13400: 0.351124\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 13450: 0.564901\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.5%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Minibatch loss at step 13500: 0.269771\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.9%\n",
      "Minibatch loss at step 13550: 0.024404\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 13600: 0.288799\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.1%\n",
      "Minibatch loss at step 13650: 0.376066\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 13700: 0.128913\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.0%\n",
      "Minibatch loss at step 13750: 0.393597\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 13800: 0.348619\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 13850: 0.411365\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.4%\n",
      "Minibatch loss at step 13900: 0.307156\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.1%\n",
      "Minibatch loss at step 13950: 0.343587\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.3%\n",
      "Minibatch loss at step 14000: 0.236785\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 14050: 0.216224\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 14100: 0.041438\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.5%\n",
      "Minibatch loss at step 14150: 1.044129\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 14200: 0.441682\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 14250: 0.424271\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.5%\n",
      "Minibatch loss at step 14300: 0.089508\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 14350: 0.578382\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 14400: 1.079329\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 87.5%\n",
      "Minibatch loss at step 14450: 0.832520\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.9%\n",
      "Minibatch loss at step 14500: 0.503852\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.1%\n",
      "Minibatch loss at step 14550: 0.416562\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.4%\n",
      "Minibatch loss at step 14600: 0.331757\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.5%\n",
      "Minibatch loss at step 14650: 0.654827\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 14700: 0.168361\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.1%\n",
      "Minibatch loss at step 14750: 0.682424\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 14800: 0.956490\n",
      "Minibatch accuracy: 62.5%\n",
      "Validation accuracy: 87.0%\n",
      "Minibatch loss at step 14850: 0.471185\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.7%\n",
      "Minibatch loss at step 14900: 0.416260\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 14950: 0.074792\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 15000: 0.214320\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 15050: 0.308113\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 15100: 0.255630\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.8%\n",
      "Minibatch loss at step 15150: 0.337410\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 15200: 0.967074\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 87.1%\n",
      "Minibatch loss at step 15250: 0.311278\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 15300: 0.331917\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 15350: 0.158531\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.5%\n",
      "Minibatch loss at step 15400: 0.229400\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.4%\n",
      "Minibatch loss at step 15450: 0.048232\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 15500: 0.337424\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.4%\n",
      "Minibatch loss at step 15550: 0.615755\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 86.5%\n",
      "Minibatch loss at step 15600: 0.636658\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.0%\n",
      "Minibatch loss at step 15650: 0.057548\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.4%\n",
      "Minibatch loss at step 15700: 0.342901\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.5%\n",
      "Minibatch loss at step 15750: 0.322000\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 15800: 0.475888\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 15850: 0.369649\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.9%\n",
      "Minibatch loss at step 15900: 0.125058\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.4%\n",
      "Minibatch loss at step 15950: 0.531150\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 16000: 0.290750\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.5%\n",
      "Minibatch loss at step 16050: 0.429978\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 16100: 0.739981\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 16150: 0.258292\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 16200: 0.018667\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 86.5%\n",
      "Minibatch loss at step 16250: 0.364929\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 16300: 0.126919\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.6%\n",
      "Minibatch loss at step 16350: 0.302313\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.6%\n",
      "Minibatch loss at step 16400: 0.376123\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 16450: 0.315560\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.0%\n",
      "Minibatch loss at step 16500: 0.107994\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 86.7%\n",
      "Minibatch loss at step 16550: 0.768517\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 16600: 0.089047\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 88.0%\n",
      "Minibatch loss at step 16650: 0.222832\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.3%\n",
      "Minibatch loss at step 16700: 0.523222\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.0%\n",
      "Minibatch loss at step 16750: 0.525878\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 16800: 0.940157\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.9%\n",
      "Minibatch loss at step 16850: 0.526437\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 87.4%\n",
      "Minibatch loss at step 16900: 0.284450\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.9%\n",
      "Minibatch loss at step 16950: 0.402491\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 17000: 0.073866\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 17050: 0.192636\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 17100: 0.226377\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.0%\n",
      "Minibatch loss at step 17150: 0.108482\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 17200: 0.199317\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.1%\n",
      "Minibatch loss at step 17250: 0.340293\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 17300: 0.206543\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.4%\n",
      "Minibatch loss at step 17350: 0.426912\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.5%\n",
      "Minibatch loss at step 17400: 0.838379\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.7%\n",
      "Minibatch loss at step 17450: 0.291910\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 17500: 0.969894\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.9%\n",
      "Minibatch loss at step 17550: 0.833393\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 17600: 0.283263\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 17650: 0.206587\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.5%\n",
      "Minibatch loss at step 17700: 0.229503\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 17750: 0.060838\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 17800: 0.779327\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 87.1%\n",
      "Minibatch loss at step 17850: 0.187382\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.5%\n",
      "Minibatch loss at step 17900: 0.317729\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.1%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Minibatch loss at step 17950: 0.474897\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 18000: 0.408802\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.2%\n",
      "Minibatch loss at step 18050: 0.339443\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 18100: 0.545194\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.3%\n",
      "Minibatch loss at step 18150: 0.899432\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 18200: 0.473348\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.0%\n",
      "Minibatch loss at step 18250: 0.597344\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.9%\n",
      "Minibatch loss at step 18300: 0.297456\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.0%\n",
      "Minibatch loss at step 18350: 0.712859\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 87.4%\n",
      "Minibatch loss at step 18400: 0.419558\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 18450: 0.594173\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 88.3%\n",
      "Minibatch loss at step 18500: 0.326487\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 18550: 0.515895\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 18600: 0.291511\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 18650: 0.189020\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.9%\n",
      "Minibatch loss at step 18700: 0.336793\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.4%\n",
      "Minibatch loss at step 18750: 0.880560\n",
      "Minibatch accuracy: 62.5%\n",
      "Validation accuracy: 87.5%\n",
      "Minibatch loss at step 18800: 0.058924\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 18850: 0.490768\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.0%\n",
      "Minibatch loss at step 18900: 0.856475\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 18950: 0.023663\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.4%\n",
      "Minibatch loss at step 19000: 0.314627\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 19050: 0.027794\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 19100: 0.553019\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.0%\n",
      "Minibatch loss at step 19150: 0.080498\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 19200: 0.050390\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 88.0%\n",
      "Minibatch loss at step 19250: 0.346845\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 19300: 0.233491\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.0%\n",
      "Minibatch loss at step 19350: 0.252510\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 19400: 0.300921\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 19450: 0.632786\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 19500: 0.486466\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.9%\n",
      "Minibatch loss at step 19550: 0.030664\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.5%\n",
      "Minibatch loss at step 19600: 0.462033\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 19650: 0.281843\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 19700: 0.483808\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.4%\n",
      "Minibatch loss at step 19750: 0.635547\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 19800: 0.327076\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 19850: 0.604724\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.1%\n",
      "Minibatch loss at step 19900: 0.085201\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 19950: 0.644225\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 20000: 0.600537\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.5%\n",
      "Test accuracy: 92.7%\n"
     ]
    }
   ],
   "source": [
    "num_steps = 20001\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "  tf.global_variables_initializer().run()\n",
    "  print('Initialized')\n",
    "  for step in range(num_steps):\n",
    "    offset = (step * batch_size) % (train_labels.shape[0] - batch_size)\n",
    "    batch_data = train_dataset[offset:(offset + batch_size), :, :, :]\n",
    "    batch_labels = train_labels[offset:(offset + batch_size), :]\n",
    "    feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}\n",
    "    _, l, predictions = session.run(\n",
    "      [optimizer, loss, train_prediction], feed_dict=feed_dict)\n",
    "    if (step % 50 == 0):\n",
    "      print('Minibatch loss at step %d: %f' % (step, l))\n",
    "      print('Minibatch accuracy: %.1f%%' % accuracy(predictions, batch_labels))\n",
    "      print('Validation accuracy: %.1f%%' % accuracy(\n",
    "        valid_prediction.eval(), valid_labels))\n",
    "  print('Test accuracy: %.1f%%' % accuracy(test_prediction.eval(), test_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "IZYv70SvvOan"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\n"
     ]
    }
   ],
   "source": [
    "batch_size = 16\n",
    "patch_size = 5\n",
    "depth = 16\n",
    "num_hidden = 64\n",
    "keep_prob = 0.8\n",
    "graph = tf.Graph()\n",
    "import math\n",
    "with graph.as_default():\n",
    "\n",
    "  # Input data.\n",
    "  tf_train_dataset = tf.placeholder(\n",
    "    tf.float32, shape=(batch_size, image_size, image_size, num_channels))\n",
    "  tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))\n",
    "  tf_valid_dataset = tf.constant(valid_dataset)\n",
    "  tf_test_dataset = tf.constant(test_dataset)\n",
    "  \n",
    "  # Variables.\n",
    "  layer1_weights = tf.Variable(tf.truncated_normal(\n",
    "      [patch_size, patch_size, num_channels, depth], stddev=0.1))\n",
    "  layer1_biases = tf.Variable(tf.zeros([depth]))\n",
    "  layer2_weights = tf.Variable(tf.truncated_normal(\n",
    "      [patch_size, patch_size, depth, depth], stddev=0.1))\n",
    "  layer2_biases = tf.Variable(tf.constant(1.0, shape=[depth]))\n",
    "  \n",
    "  new_size = math.ceil((((math.ceil(((image_size-patch_size+1)-2+1)/2)-patch_size+1)) - 2 + 1) / 2)\n",
    "  print(new_size)\n",
    "  layer3_weights = tf.Variable(tf.truncated_normal(\n",
    "      [new_size**2*depth, num_hidden], stddev=0.1))\n",
    "  layer3_biases = tf.Variable(tf.constant(1.0, shape=[num_hidden]))\n",
    "  layer4_weights = tf.Variable(tf.truncated_normal(\n",
    "      [num_hidden, num_labels], stddev=0.1))\n",
    "  layer4_biases = tf.Variable(tf.constant(1.0, shape=[num_labels]))\n",
    "  \n",
    "  def max_pool_2x2(x):\n",
    "    return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],\n",
    "                            strides=[1, 2, 2, 1], padding='VALID')\n",
    "\n",
    "  # Model.\n",
    "  def model(data):\n",
    "    # strides: A list of ints. 1-D of length 4. The stride of the sliding window for each dimension of input\n",
    "    # input 28 * 28\n",
    "    conv = tf.nn.conv2d(data, layer1_weights, strides=[1, 1, 1, 1], padding='VALID')\n",
    "    hidden = tf.nn.relu(conv + layer1_biases)\n",
    "    # 28 - 5(patch_size) + 1 = 24, input 24 * 24\n",
    "    max_pool = max_pool_2x2(hidden)\n",
    "    # ceil((24 - 2 + 1) / 2) = 12, input 12 * 12 \n",
    "    conv = tf.nn.conv2d(max_pool, layer2_weights, strides=[1, 1, 1, 1], padding='VALID')\n",
    "    hidden = tf.nn.relu(conv + layer2_biases)\n",
    "    # 12 - 5 + 1 = 8, input 8 * 8\n",
    "    max_pool = max_pool_2x2(hidden)\n",
    "    # ceil((8 - 2 + 1) / 2) = 4, input 4 * 4 \n",
    "    shape = max_pool.get_shape().as_list()\n",
    "    reshape = tf.reshape(max_pool, [shape[0], shape[1] * shape[2] * shape[3]])\n",
    "    hidden = tf.nn.dropout(tf.nn.relu(tf.matmul(reshape, layer3_weights) + layer3_biases), keep_prob=keep_prob)\n",
    "    return tf.matmul(hidden, layer4_weights) + layer4_biases\n",
    "\n",
    "  def predict(data):\n",
    "    # strides: A list of ints. 1-D of length 4. The stride of the sliding window for each dimension of input\n",
    "    # input 28 * 28\n",
    "    conv = tf.nn.conv2d(data, layer1_weights, strides=[1, 1, 1, 1], padding='VALID')\n",
    "    hidden = tf.nn.relu(conv + layer1_biases)\n",
    "    # 28 - 5(patch_size) + 1 = 24, input 24 * 24\n",
    "    max_pool = max_pool_2x2(hidden)\n",
    "    # ceil((24 - 2 + 1) / 2) = 12, input 12 * 12 \n",
    "    conv = tf.nn.conv2d(max_pool, layer2_weights, strides=[1, 1, 1, 1], padding='VALID')\n",
    "    hidden = tf.nn.relu(conv + layer2_biases)\n",
    "    # 12 - 5 + 1 = 8, input 8 * 8\n",
    "    max_pool = max_pool_2x2(hidden)\n",
    "    # ceil((8 - 2 + 1) / 2) = 4, input 4 * 4 \n",
    "    shape = max_pool.get_shape().as_list()\n",
    "    reshape = tf.reshape(max_pool, [shape[0], shape[1] * shape[2] * shape[3]])\n",
    "    hidden = tf.nn.relu(tf.matmul(reshape, layer3_weights) + layer3_biases)\n",
    "    return tf.matmul(hidden, layer4_weights) + layer4_biases\n",
    "  # Training computation.\n",
    "  logits = model(tf_train_dataset)\n",
    "  loss = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=tf_train_labels, logits=logits))\n",
    "    \n",
    "  # Optimizer.\n",
    "  \"\"\"\n",
    "  decayed_learning_rate = learning_rate *\n",
    "                        decay_rate ^ (global_step / decay_steps)\n",
    "  \"\"\"\n",
    "  global_step = tf.Variable(0, trainable=False)\n",
    "  starter_learning_rate = 0.05\n",
    "  learning_rate = tf.train.exponential_decay(starter_learning_rate, global_step, 100, 0.37, staircase=True, name=None)\n",
    "  optimizer = tf.train.MomentumOptimizer(learning_rate, 0.6).minimize(loss)\n",
    "  \n",
    "  # Predictions for the training, validation, and test data.\n",
    "  train_prediction = tf.nn.softmax(logits)\n",
    "  valid_prediction = tf.nn.softmax(predict(tf_valid_dataset))\n",
    "  test_prediction = tf.nn.softmax(predict(tf_test_dataset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 37
      }
     ]
    },
    "colab_type": "code",
    "collapsed": true,
    "executionInfo": {
     "elapsed": 63292,
     "status": "ok",
     "timestamp": 1446658966251,
     "user": {
      "color": "",
      "displayName": "",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "",
      "photoUrl": "",
      "sessionId": "0",
      "userId": ""
     },
     "user_tz": 480
    },
    "id": "noKFb2UovVFR",
    "outputId": "28941338-2ef9-4088-8bd1-44295661e628"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Minibatch loss at step 0: 3.803486\n",
      "Minibatch accuracy: 6.2%\n",
      "Validation accuracy: 10.0%\n",
      "Minibatch loss at step 50: 0.927324\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 61.6%\n",
      "Minibatch loss at step 100: 1.348211\n",
      "Minibatch accuracy: 43.8%\n",
      "Validation accuracy: 68.2%\n",
      "Minibatch loss at step 150: 1.423715\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 73.3%\n",
      "Minibatch loss at step 200: 1.243949\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 73.4%\n",
      "Minibatch loss at step 250: 1.026381\n",
      "Minibatch accuracy: 56.2%\n",
      "Validation accuracy: 79.0%\n",
      "Minibatch loss at step 300: 0.820717\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 80.7%\n",
      "Minibatch loss at step 350: 0.253694\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 80.6%\n",
      "Minibatch loss at step 400: 0.599398\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 80.0%\n",
      "Minibatch loss at step 450: 0.949975\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 77.2%\n",
      "Minibatch loss at step 500: 0.699220\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 82.5%\n",
      "Minibatch loss at step 550: 0.282925\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 82.4%\n",
      "Minibatch loss at step 600: 0.199812\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 83.2%\n",
      "Minibatch loss at step 650: 0.450176\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 82.7%\n",
      "Minibatch loss at step 700: 0.965787\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 83.7%\n",
      "Minibatch loss at step 750: 0.574953\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 83.4%\n",
      "Minibatch loss at step 800: 1.389534\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 82.2%\n",
      "Minibatch loss at step 850: 0.639437\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 83.8%\n",
      "Minibatch loss at step 900: 0.743285\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 83.8%\n",
      "Minibatch loss at step 950: 0.489952\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 80.6%\n",
      "Minibatch loss at step 1000: 0.251943\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 84.4%\n",
      "Minibatch loss at step 1050: 0.531867\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 83.3%\n",
      "Minibatch loss at step 1100: 0.675906\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 83.1%\n",
      "Minibatch loss at step 1150: 0.558023\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.4%\n",
      "Minibatch loss at step 1200: 0.107836\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 1250: 0.688683\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 85.0%\n",
      "Minibatch loss at step 1300: 0.292482\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 83.4%\n",
      "Minibatch loss at step 1350: 0.850742\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 84.3%\n",
      "Minibatch loss at step 1400: 1.083486\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 84.3%\n",
      "Minibatch loss at step 1450: 0.712690\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 84.0%\n",
      "Minibatch loss at step 1500: 0.715001\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 84.5%\n",
      "Minibatch loss at step 1550: 0.661686\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.0%\n",
      "Minibatch loss at step 1600: 0.272107\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 83.4%\n",
      "Minibatch loss at step 1650: 0.516562\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 83.7%\n",
      "Minibatch loss at step 1700: 0.304294\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 1750: 0.454177\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.1%\n",
      "Minibatch loss at step 1800: 0.618673\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 84.7%\n",
      "Minibatch loss at step 1850: 0.584768\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 1900: 0.829529\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 1950: 0.034404\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 2000: 0.684086\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 2050: 0.531395\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 2100: 0.307471\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 2150: 0.607336\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 2200: 0.488351\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.1%\n",
      "Minibatch loss at step 2250: 0.544165\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 84.9%\n",
      "Minibatch loss at step 2300: 0.587790\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 2350: 0.702319\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 86.2%\n",
      "Minibatch loss at step 2400: 0.407942\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 2450: 0.368799\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 2500: 0.367273\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.3%\n",
      "Minibatch loss at step 2550: 0.098452\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 2600: 0.274750\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 2650: 0.958138\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 2700: 0.761634\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 2750: 0.052568\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 86.2%\n",
      "Minibatch loss at step 2800: 0.241534\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.2%\n",
      "Minibatch loss at step 2850: 1.166207\n",
      "Minibatch accuracy: 62.5%\n",
      "Validation accuracy: 85.9%\n",
      "Minibatch loss at step 2900: 0.360345\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.2%\n",
      "Minibatch loss at step 2950: 0.503370\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.3%\n",
      "Minibatch loss at step 3000: 0.299438\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.8%\n",
      "Minibatch loss at step 3050: 0.298749\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.5%\n",
      "Minibatch loss at step 3100: 0.174648\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.6%\n",
      "Minibatch loss at step 3150: 0.557225\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 85.3%\n",
      "Minibatch loss at step 3200: 0.244828\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.5%\n",
      "Minibatch loss at step 3250: 0.355215\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 3300: 0.278343\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.8%\n",
      "Minibatch loss at step 3350: 0.454848\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.2%\n",
      "Minibatch loss at step 3400: 0.177493\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 86.5%\n",
      "Minibatch loss at step 3450: 0.206647\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.2%\n",
      "Minibatch loss at step 3500: 0.429730\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.0%\n",
      "Minibatch loss at step 3550: 0.492233\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 85.6%\n",
      "Minibatch loss at step 3600: 0.704985\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 86.9%\n",
      "Minibatch loss at step 3650: 0.508696\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.6%\n",
      "Minibatch loss at step 3700: 0.327902\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.4%\n",
      "Minibatch loss at step 3750: 0.260459\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.9%\n",
      "Minibatch loss at step 3800: 0.678656\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 3850: 0.309116\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.8%\n",
      "Minibatch loss at step 3900: 0.341244\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 3950: 0.457035\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.0%\n",
      "Minibatch loss at step 4000: 0.564625\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.1%\n",
      "Minibatch loss at step 4050: 0.528920\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 4100: 0.220693\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.9%\n",
      "Minibatch loss at step 4150: 0.846470\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 87.0%\n",
      "Minibatch loss at step 4200: 0.401843\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.7%\n",
      "Minibatch loss at step 4250: 0.513002\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.4%\n",
      "Minibatch loss at step 4300: 0.252551\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.6%\n",
      "Minibatch loss at step 4350: 0.248077\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.8%\n",
      "Minibatch loss at step 4400: 0.139399\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.5%\n",
      "Minibatch loss at step 4450: 0.568076\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.0%\n",
      "Minibatch loss at step 4500: 0.174351\n",
      "Minibatch accuracy: 93.8%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation accuracy: 86.1%\n",
      "Minibatch loss at step 4550: 0.440966\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 4600: 0.661960\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.6%\n",
      "Minibatch loss at step 4650: 0.548963\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.7%\n",
      "Minibatch loss at step 4700: 0.519950\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 4750: 0.556362\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.1%\n",
      "Minibatch loss at step 4800: 0.256407\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.5%\n",
      "Minibatch loss at step 4850: 0.654070\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 87.5%\n",
      "Minibatch loss at step 4900: 0.146518\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 4950: 1.175735\n",
      "Minibatch accuracy: 62.5%\n",
      "Validation accuracy: 86.9%\n",
      "Minibatch loss at step 5000: 0.901807\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 5050: 0.654801\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 86.8%\n",
      "Minibatch loss at step 5100: 1.125203\n",
      "Minibatch accuracy: 62.5%\n",
      "Validation accuracy: 86.4%\n",
      "Minibatch loss at step 5150: 0.201960\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 5200: 0.327944\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 5250: 0.769028\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 86.9%\n",
      "Minibatch loss at step 5300: 0.396815\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.1%\n",
      "Minibatch loss at step 5350: 0.804218\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 87.0%\n",
      "Minibatch loss at step 5400: 0.162105\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.7%\n",
      "Minibatch loss at step 5450: 0.621064\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.0%\n",
      "Minibatch loss at step 5500: 0.598733\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 87.1%\n",
      "Minibatch loss at step 5550: 0.962603\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 5600: 0.279203\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 86.9%\n",
      "Minibatch loss at step 5650: 0.407130\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 5700: 0.629450\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 86.3%\n",
      "Minibatch loss at step 5750: 0.263586\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.0%\n",
      "Minibatch loss at step 5800: 0.288091\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 5850: 0.609057\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 5900: 0.807586\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 5950: 0.178675\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.0%\n",
      "Minibatch loss at step 6000: 0.649592\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 6050: 0.196829\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.1%\n",
      "Minibatch loss at step 6100: 0.125079\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 86.6%\n",
      "Minibatch loss at step 6150: 0.208217\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.7%\n",
      "Minibatch loss at step 6200: 0.941682\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 6250: 0.249278\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 6300: 1.189225\n",
      "Minibatch accuracy: 62.5%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 6350: 0.273994\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 6400: 0.435939\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 6450: 0.410981\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 6500: 0.219415\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.5%\n",
      "Minibatch loss at step 6550: 0.454474\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.5%\n",
      "Minibatch loss at step 6600: 0.988013\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 6650: 0.380803\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 6700: 0.235238\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.5%\n",
      "Minibatch loss at step 6750: 0.558909\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 6800: 0.646399\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 6850: 0.763483\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.0%\n",
      "Minibatch loss at step 6900: 0.449524\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 6950: 0.249717\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 7000: 0.342281\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.9%\n",
      "Minibatch loss at step 7050: 0.800141\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 7100: 0.414967\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 7150: 0.636151\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 7200: 0.045686\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.4%\n",
      "Minibatch loss at step 7250: 0.219186\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 7300: 0.728517\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 7350: 0.497401\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 7400: 0.199732\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.8%\n",
      "Minibatch loss at step 7450: 0.757889\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.1%\n",
      "Minibatch loss at step 7500: 0.792522\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 7550: 0.366639\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.2%\n",
      "Minibatch loss at step 7600: 0.663112\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.6%\n",
      "Minibatch loss at step 7650: 0.715174\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 7700: 0.696061\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 7750: 0.262007\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 7800: 0.716107\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 7850: 0.287798\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 7900: 0.900664\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 7950: 0.106387\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 8000: 0.231131\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.9%\n",
      "Minibatch loss at step 8050: 0.346589\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.0%\n",
      "Minibatch loss at step 8100: 0.277188\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 8150: 0.326694\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.4%\n",
      "Minibatch loss at step 8200: 0.105561\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 88.1%\n",
      "Minibatch loss at step 8250: 0.493442\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 8300: 0.353619\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.1%\n",
      "Minibatch loss at step 8350: 0.700199\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 8400: 0.177722\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 8450: 0.617910\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 8500: 0.322868\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 8550: 0.653285\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.9%\n",
      "Minibatch loss at step 8600: 0.697994\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.9%\n",
      "Minibatch loss at step 8650: 0.133425\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 8700: 0.780537\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 86.0%\n",
      "Minibatch loss at step 8750: 0.442604\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 8800: 0.729075\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 8850: 0.483908\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.2%\n",
      "Minibatch loss at step 8900: 0.304891\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 8950: 1.075435\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 9000: 0.203667\n",
      "Minibatch accuracy: 100.0%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 9050: 0.287797\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 88.0%\n",
      "Minibatch loss at step 9100: 0.167920\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.0%\n",
      "Minibatch loss at step 9150: 0.039274\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 9200: 0.370156\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.5%\n",
      "Minibatch loss at step 9250: 0.386327\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 9300: 0.243265\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 9350: 0.083319\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.9%\n",
      "Minibatch loss at step 9400: 0.274899\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 9450: 0.162802\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 9500: 0.128826\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 9550: 0.243911\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.2%\n",
      "Minibatch loss at step 9600: 0.386925\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.3%\n",
      "Minibatch loss at step 9650: 0.365810\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.2%\n",
      "Minibatch loss at step 9700: 0.359997\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.0%\n",
      "Minibatch loss at step 9750: 0.269548\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.3%\n",
      "Minibatch loss at step 9800: 0.471912\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.3%\n",
      "Minibatch loss at step 9850: 0.324694\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.5%\n",
      "Minibatch loss at step 9900: 0.057214\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 88.1%\n",
      "Minibatch loss at step 9950: 1.035049\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 10000: 0.320792\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 10050: 0.041517\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 88.3%\n",
      "Minibatch loss at step 10100: 0.463620\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.1%\n",
      "Minibatch loss at step 10150: 0.496568\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 10200: 1.072018\n",
      "Minibatch accuracy: 62.5%\n",
      "Validation accuracy: 88.4%\n",
      "Minibatch loss at step 10250: 0.447212\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.3%\n",
      "Minibatch loss at step 10300: 0.717707\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 10350: 0.137353\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.4%\n",
      "Minibatch loss at step 10400: 0.583412\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.0%\n",
      "Minibatch loss at step 10450: 0.612383\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 88.4%\n",
      "Minibatch loss at step 10500: 0.194044\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.6%\n",
      "Minibatch loss at step 10550: 0.284953\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.9%\n",
      "Minibatch loss at step 10600: 0.552718\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.2%\n",
      "Minibatch loss at step 10650: 0.126130\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.5%\n",
      "Minibatch loss at step 10700: 0.454800\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.4%\n",
      "Minibatch loss at step 10750: 0.276060\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.4%\n",
      "Minibatch loss at step 10800: 0.610998\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.5%\n",
      "Minibatch loss at step 10850: 0.316989\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.4%\n",
      "Minibatch loss at step 10900: 0.747795\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.5%\n",
      "Minibatch loss at step 10950: 0.210195\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 86.8%\n",
      "Minibatch loss at step 11000: 0.626089\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.5%\n",
      "Minibatch loss at step 11050: 0.220341\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.2%\n",
      "Minibatch loss at step 11100: 0.281345\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.2%\n",
      "Minibatch loss at step 11150: 0.364163\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.4%\n",
      "Minibatch loss at step 11200: 0.356148\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 11250: 0.390855\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.0%\n",
      "Minibatch loss at step 11300: 0.662796\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 87.9%\n",
      "Minibatch loss at step 11350: 0.152490\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.7%\n",
      "Minibatch loss at step 11400: 1.026201\n",
      "Minibatch accuracy: 62.5%\n",
      "Validation accuracy: 88.6%\n",
      "Minibatch loss at step 11450: 0.746143\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.5%\n",
      "Minibatch loss at step 11500: 0.413206\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.9%\n",
      "Minibatch loss at step 11550: 0.178954\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.4%\n",
      "Minibatch loss at step 11600: 0.216967\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 11650: 0.638799\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.3%\n",
      "Minibatch loss at step 11700: 0.438860\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.1%\n",
      "Minibatch loss at step 11750: 0.447639\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.3%\n",
      "Minibatch loss at step 11800: 0.704071\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 88.1%\n",
      "Minibatch loss at step 11850: 0.105099\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.4%\n",
      "Minibatch loss at step 11900: 0.129632\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.2%\n",
      "Minibatch loss at step 11950: 0.029338\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 88.3%\n",
      "Minibatch loss at step 12000: 0.599813\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.3%\n",
      "Minibatch loss at step 12050: 0.199333\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 12100: 0.182437\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.2%\n",
      "Minibatch loss at step 12150: 0.266271\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.1%\n",
      "Minibatch loss at step 12200: 0.078994\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 88.1%\n",
      "Minibatch loss at step 12250: 0.308355\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.2%\n",
      "Minibatch loss at step 12300: 0.319585\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.0%\n",
      "Minibatch loss at step 12350: 0.395149\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 12400: 0.191194\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 12450: 0.434257\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.5%\n",
      "Minibatch loss at step 12500: 0.340421\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.6%\n",
      "Minibatch loss at step 12550: 0.222723\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.0%\n",
      "Minibatch loss at step 12600: 0.798896\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 88.0%\n",
      "Minibatch loss at step 12650: 0.021852\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 88.3%\n",
      "Minibatch loss at step 12700: 0.494310\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.9%\n",
      "Minibatch loss at step 12750: 0.379880\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.4%\n",
      "Minibatch loss at step 12800: 0.540152\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.1%\n",
      "Minibatch loss at step 12850: 0.477879\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.1%\n",
      "Minibatch loss at step 12900: 0.455264\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 87.9%\n",
      "Minibatch loss at step 12950: 0.883448\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 86.7%\n",
      "Minibatch loss at step 13000: 0.401997\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.2%\n",
      "Minibatch loss at step 13050: 0.163976\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 13100: 0.227926\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.0%\n",
      "Minibatch loss at step 13150: 0.629429\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 13200: 0.587309\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 13250: 0.198602\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.6%\n",
      "Minibatch loss at step 13300: 0.225977\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.0%\n",
      "Minibatch loss at step 13350: 0.664676\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.2%\n",
      "Minibatch loss at step 13400: 0.228678\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.9%\n",
      "Minibatch loss at step 13450: 0.737223\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.1%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Minibatch loss at step 13500: 0.191527\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.7%\n",
      "Minibatch loss at step 13550: 0.007345\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 88.3%\n",
      "Minibatch loss at step 13600: 0.338702\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.9%\n",
      "Minibatch loss at step 13650: 0.220080\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.4%\n",
      "Minibatch loss at step 13700: 0.066398\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 88.4%\n",
      "Minibatch loss at step 13750: 0.396385\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.7%\n",
      "Minibatch loss at step 13800: 0.179199\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.5%\n",
      "Minibatch loss at step 13850: 0.374941\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.3%\n",
      "Minibatch loss at step 13900: 0.356554\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.2%\n",
      "Minibatch loss at step 13950: 0.487772\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.9%\n",
      "Minibatch loss at step 14000: 0.409940\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 14050: 0.321667\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.2%\n",
      "Minibatch loss at step 14100: 0.022453\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.9%\n",
      "Minibatch loss at step 14150: 0.695501\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.5%\n",
      "Minibatch loss at step 14200: 0.463208\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 14250: 0.403423\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.7%\n",
      "Minibatch loss at step 14300: 0.079223\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.5%\n",
      "Minibatch loss at step 14350: 0.683466\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 88.3%\n",
      "Minibatch loss at step 14400: 0.656349\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 88.6%\n",
      "Minibatch loss at step 14450: 0.716121\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 14500: 0.369550\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.9%\n",
      "Minibatch loss at step 14550: 0.313627\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 14600: 0.132233\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.9%\n",
      "Minibatch loss at step 14650: 0.497304\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.6%\n",
      "Minibatch loss at step 14700: 0.199541\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.3%\n",
      "Minibatch loss at step 14750: 0.366687\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.2%\n",
      "Minibatch loss at step 14800: 1.769819\n",
      "Minibatch accuracy: 62.5%\n",
      "Validation accuracy: 88.5%\n",
      "Minibatch loss at step 14850: 0.390216\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.3%\n",
      "Minibatch loss at step 14900: 0.345922\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.7%\n",
      "Minibatch loss at step 14950: 0.065846\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 88.7%\n",
      "Minibatch loss at step 15000: 0.308530\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 15050: 0.173899\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 88.5%\n",
      "Minibatch loss at step 15100: 0.180800\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.5%\n",
      "Minibatch loss at step 15150: 0.580080\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.7%\n",
      "Minibatch loss at step 15200: 0.792373\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 15250: 0.427833\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.6%\n",
      "Minibatch loss at step 15300: 0.469403\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.7%\n",
      "Minibatch loss at step 15350: 0.055420\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 88.3%\n",
      "Minibatch loss at step 15400: 0.574793\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.4%\n",
      "Minibatch loss at step 15450: 0.185367\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.7%\n",
      "Minibatch loss at step 15500: 0.272948\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.4%\n",
      "Minibatch loss at step 15550: 0.704179\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.3%\n",
      "Minibatch loss at step 15600: 0.413422\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.5%\n",
      "Minibatch loss at step 15650: 0.075001\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 88.8%\n",
      "Minibatch loss at step 15700: 0.391286\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.8%\n",
      "Minibatch loss at step 15750: 0.307705\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.3%\n",
      "Minibatch loss at step 15800: 0.448355\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.4%\n",
      "Minibatch loss at step 15850: 0.115201\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.4%\n",
      "Minibatch loss at step 15900: 0.150566\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 88.5%\n",
      "Minibatch loss at step 15950: 0.596264\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.7%\n",
      "Minibatch loss at step 16000: 0.292121\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 89.1%\n",
      "Minibatch loss at step 16050: 0.244910\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.6%\n",
      "Minibatch loss at step 16100: 0.837257\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 89.0%\n",
      "Minibatch loss at step 16150: 0.203853\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.4%\n",
      "Minibatch loss at step 16200: 0.256436\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.7%\n",
      "Minibatch loss at step 16250: 0.325087\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.6%\n",
      "Minibatch loss at step 16300: 0.071764\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.0%\n",
      "Minibatch loss at step 16350: 0.368303\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 88.2%\n",
      "Minibatch loss at step 16400: 0.332238\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.8%\n",
      "Minibatch loss at step 16450: 0.258957\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.9%\n",
      "Minibatch loss at step 16500: 0.201200\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.1%\n",
      "Minibatch loss at step 16550: 0.790245\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.6%\n",
      "Minibatch loss at step 16600: 0.033662\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 88.9%\n",
      "Minibatch loss at step 16650: 0.264793\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.4%\n",
      "Minibatch loss at step 16700: 0.474429\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.3%\n",
      "Minibatch loss at step 16750: 0.686724\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 88.2%\n",
      "Minibatch loss at step 16800: 0.961011\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.4%\n",
      "Minibatch loss at step 16850: 0.237734\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 88.7%\n",
      "Minibatch loss at step 16900: 0.279083\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.9%\n",
      "Minibatch loss at step 16950: 0.439920\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.7%\n",
      "Minibatch loss at step 17000: 0.019890\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 88.8%\n",
      "Minibatch loss at step 17050: 0.176266\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.7%\n",
      "Minibatch loss at step 17100: 0.013597\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 88.3%\n",
      "Minibatch loss at step 17150: 0.088565\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 88.9%\n",
      "Minibatch loss at step 17200: 0.203694\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.8%\n",
      "Minibatch loss at step 17250: 0.585239\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.1%\n",
      "Minibatch loss at step 17300: 0.335221\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 89.0%\n",
      "Minibatch loss at step 17350: 0.407741\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.5%\n",
      "Minibatch loss at step 17400: 0.880580\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 88.5%\n",
      "Minibatch loss at step 17450: 0.407386\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 89.1%\n",
      "Minibatch loss at step 17500: 0.954119\n",
      "Minibatch accuracy: 68.8%\n",
      "Validation accuracy: 88.8%\n",
      "Minibatch loss at step 17550: 0.674027\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 89.2%\n",
      "Minibatch loss at step 17600: 0.335608\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.8%\n",
      "Minibatch loss at step 17650: 0.362967\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.1%\n",
      "Minibatch loss at step 17700: 0.303942\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.7%\n",
      "Minibatch loss at step 17750: 0.071998\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 88.8%\n",
      "Minibatch loss at step 17800: 0.556933\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.6%\n",
      "Minibatch loss at step 17850: 0.114420\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.1%\n",
      "Minibatch loss at step 17900: 0.154844\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.7%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Minibatch loss at step 17950: 0.762857\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.9%\n",
      "Minibatch loss at step 18000: 0.405371\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.5%\n",
      "Minibatch loss at step 18050: 0.454330\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.6%\n",
      "Minibatch loss at step 18100: 0.405725\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.6%\n",
      "Minibatch loss at step 18150: 0.646574\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 88.8%\n",
      "Minibatch loss at step 18200: 0.491613\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.4%\n",
      "Minibatch loss at step 18250: 0.562298\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 87.9%\n",
      "Minibatch loss at step 18300: 0.222044\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.5%\n",
      "Minibatch loss at step 18350: 0.224997\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.7%\n",
      "Minibatch loss at step 18400: 0.354835\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 89.2%\n",
      "Minibatch loss at step 18450: 0.430899\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.9%\n",
      "Minibatch loss at step 18500: 0.338681\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 89.0%\n",
      "Minibatch loss at step 18550: 0.664555\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 89.0%\n",
      "Minibatch loss at step 18600: 0.067903\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.2%\n",
      "Minibatch loss at step 18650: 0.150731\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 88.4%\n",
      "Minibatch loss at step 18700: 0.426756\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.2%\n",
      "Minibatch loss at step 18750: 0.746951\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 89.0%\n",
      "Minibatch loss at step 18800: 0.157644\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.3%\n",
      "Minibatch loss at step 18850: 0.485592\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.5%\n",
      "Minibatch loss at step 18900: 0.361196\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.4%\n",
      "Minibatch loss at step 18950: 0.018185\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.5%\n",
      "Minibatch loss at step 19000: 0.337686\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.2%\n",
      "Minibatch loss at step 19050: 0.246239\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.6%\n",
      "Minibatch loss at step 19100: 0.682476\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 88.3%\n",
      "Minibatch loss at step 19150: 0.201066\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 87.9%\n",
      "Minibatch loss at step 19200: 0.031328\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 89.2%\n",
      "Minibatch loss at step 19250: 0.308864\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.3%\n",
      "Minibatch loss at step 19300: 0.152190\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.7%\n",
      "Minibatch loss at step 19350: 0.229908\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.4%\n",
      "Minibatch loss at step 19400: 0.590805\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 88.6%\n",
      "Minibatch loss at step 19450: 0.614888\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 88.4%\n",
      "Minibatch loss at step 19500: 0.329370\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.6%\n",
      "Minibatch loss at step 19550: 0.093797\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 89.1%\n",
      "Minibatch loss at step 19600: 0.832113\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.3%\n",
      "Minibatch loss at step 19650: 0.306527\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.6%\n",
      "Minibatch loss at step 19700: 0.516681\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.6%\n",
      "Minibatch loss at step 19750: 0.398428\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.6%\n",
      "Minibatch loss at step 19800: 0.194241\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.9%\n",
      "Minibatch loss at step 19850: 0.426266\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 88.3%\n",
      "Minibatch loss at step 19900: 0.176538\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 87.8%\n",
      "Minibatch loss at step 19950: 0.507147\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 88.7%\n",
      "Minibatch loss at step 20000: 0.530069\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 88.4%\n",
      "Test accuracy: 93.8%\n"
     ]
    }
   ],
   "source": [
    "num_steps = 20001\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "  tf.global_variables_initializer().run()\n",
    "  print('Initialized')\n",
    "  for step in range(num_steps):\n",
    "    offset = (step * batch_size) % (train_labels.shape[0] - batch_size)\n",
    "    batch_data = train_dataset[offset:(offset + batch_size), :, :, :]\n",
    "    batch_labels = train_labels[offset:(offset + batch_size), :]\n",
    "    feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}\n",
    "    _, l, predictions = session.run(\n",
    "      [optimizer, loss, train_prediction], feed_dict=feed_dict)\n",
    "    if (step % 50 == 0):\n",
    "      print('Minibatch loss at step %d: %f' % (step, l))\n",
    "      print('Minibatch accuracy: %.1f%%' % accuracy(predictions, batch_labels))\n",
    "      print('Validation accuracy: %.1f%%' % accuracy(\n",
    "        valid_prediction.eval(), valid_labels))\n",
    "  print('Test accuracy: %.1f%%' % accuracy(test_prediction.eval(), test_labels))"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "default_view": {},
   "name": "4_convolutions.ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}