HW5_overfitting.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Homework 5\n",
    "In this homework you will improve your convolutional network to overfit less on supertux.\n",
    "\n",
    "Development notes: \n",
    "\n",
    "1) If you are doing your homework in a Jupyter/iPython notebook you may need to 'Restart & Clear Output' after making a change and re-running a cell.  TensorFlow will not allow you to create multiple variables with the same name, which is what you are doing when you run a cell that creates a variable twice.<br/><br/>\n",
    "2) Be careful with your calls to global_variables_initializer(). If you call it after training one network it will re-initialize your variables erasing your training.  In general, double check the outputs of your model after all training and before turning your model in. Ending a session will discard all your variable values.\n",
    "\n",
    "## Part 0: Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input shape: (12257, 64, 64, 3)\n",
      "Labels shape: (12257,)\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import util\n",
    "\n",
    "# Load the data we are giving you\n",
    "def load(filename, W=64, H=64):\n",
    "    data = np.fromfile(filename, dtype=np.uint8).reshape((-1, W*H*3+1))\n",
    "    images, labels = data[:, :-1].reshape((-1,H,W,3)), data[:, -1]\n",
    "    return images, labels\n",
    "\n",
    "image_data, label_data = load('tux_train.dat')\n",
    "\n",
    "print('Input shape: ' + str(image_data.shape))\n",
    "print('Labels shape: ' + str(label_data.shape))\n",
    "\n",
    "num_classes = 6"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 1: Define your convnet\n",
    "\n",
    "Make sure the total number of parameters is less than 100,000."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Lets clear the tensorflow graph, so that you don't have to restart the notebook every time you change the network\n",
    "tf.reset_default_graph()\n",
    "\n",
    "# Set up your input placeholder\n",
    "inputs = tf.placeholder(tf.float32, (None,64,64,3))\n",
    "\n",
    "# Step 1: Augment the training data (try the following, not all might improve the performance)\n",
    "#  * mirror the image\n",
    "#  * color augmentations (keep the values to small ranges first then try to expand):\n",
    "#    - brightness\n",
    "#    - hue\n",
    "#    - saturation\n",
    "#    - contrast\n",
    "\n",
    "def data_augmentation(I):\n",
    "    # TODO: Put your data augmentation here\n",
    "#     I= tf.image.random_brightness(I,20)\n",
    "#     I= tf.image.random_saturation(I,0,50)\n",
    "    I= tf.image.random_flip_left_right(I)\n",
    "#     I= tf.image.random_flip_up_down(I)\n",
    "#     I= tf.image.random_hue(I,0.2)\n",
    "#     I= tf.image.random_contrast(I,0,0.5)\n",
    "    return I\n",
    "\n",
    "# map_fn applies data_augmentation independently for each image in the batch, since we are not croping let's apply the augmentation before whitening, it does make evaluation easier\n",
    "aug_input = tf.map_fn(data_augmentation, inputs)\n",
    "\n",
    "# During evaluation we don't want data augmentation\n",
    "eval_inputs = tf.identity(aug_input, name='inputs')\n",
    "\n",
    "# Whenever you deal with image data it's important to mean center it first and subtract the standard deviation\n",
    "white_inputs = (eval_inputs - 100.) / 72.\n",
    "\n",
    "\n",
    "# Set up your label placeholders\n",
    "labels = tf.placeholder(tf.int64, (None), name='labels')\n",
    "lrate= tf.placeholder(tf.float32,name='learning_rate')\n",
    "outputs = []\n",
    "losses = []\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"model1/model1/conv1/Relu:0\", shape=(?, 32, 32, 19), dtype=float32)\n",
      "Tensor(\"model1/model1/conv2/Relu:0\", shape=(?, 16, 16, 30), dtype=float32)\n",
      "Tensor(\"model1/model1/conv3/Relu:0\", shape=(?, 8, 8, 50), dtype=float32)\n",
      "Tensor(\"model1/model1/conv4/Relu:0\", shape=(?, 4, 4, 100), dtype=float32)\n",
      "Tensor(\"model1/model1/pool4/MaxPool:0\", shape=(?, 2, 2, 100), dtype=float32)\n",
      "Tensor(\"model1/model1/conv5/Relu:0\", shape=(?, 1, 1, 6), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Step 4: Define multiple models in your ensemble. You should train an ensemble of 5 models.\n",
    "# Let's put all variables in a scope, this makes training ensembles easier. Make sure each model in your ensemble has it's own scope and produces an output and loss\n",
    "with tf.name_scope('model1'), tf.variable_scope('model1'):\n",
    "    # Step 2: define the compute graph of your CNN here (use your solution to HW4 here)\n",
    "    #   Add weight regularization (l2-loss)\n",
    "    \n",
    "    conv1 = tf.contrib.layers.conv2d(inputs=white_inputs,num_outputs=19,kernel_size=[5, 5],weights_regularizer=tf.nn.l2_loss,\n",
    "      stride=2,padding=\"same\",scope='conv1') # put in scope everywhere\n",
    "    print(conv1)\n",
    "    conv1_bn= tf.layers.batch_normalization(conv1,training=True)\n",
    "   \n",
    "    conv2 = tf.contrib.layers.conv2d(inputs=conv1_bn,num_outputs=30,kernel_size=[5, 5],weights_regularizer=tf.nn.l2_loss,\n",
    "      stride=2,padding=\"same\",scope='conv2') # put in scope everywhere\n",
    "    print(conv2)\n",
    "    conv2_bn= tf.layers.batch_normalization(conv2,training=True)\n",
    "    \n",
    "    \n",
    "    conv3 = tf.contrib.layers.conv2d(inputs=conv2_bn,num_outputs=50,kernel_size=[5,5],weights_regularizer=tf.nn.l2_loss,\n",
    "      stride=2,padding=\"same\",scope='conv3') # put in scope everywhere\n",
    "    print(conv3)\n",
    "    conv3_bn= tf.layers.batch_normalization(conv3,training=True)\n",
    "\n",
    "    \n",
    "    conv4 = tf.contrib.layers.conv2d(inputs=conv3_bn,num_outputs=100,kernel_size=[3, 3],weights_regularizer=tf.nn.l2_loss,\n",
    "      stride=2,padding=\"same\",scope='conv4') # put in scope everywhere\n",
    "    print(conv4)\n",
    "    conv4_bn= tf.layers.batch_normalization(conv4,training=True)\n",
    "    pool4 = tf.contrib.layers.max_pool2d(inputs=conv4_bn, kernel_size=[2,2], stride=2, scope='pool4')\n",
    "    print(pool4)\n",
    "    \n",
    "    conv5 = tf.contrib.layers.conv2d(inputs=pool4,num_outputs=6,kernel_size=[1,1],weights_regularizer=tf.nn.l2_loss,\n",
    "      stride=2,padding=\"same\",scope='conv5') # put in scope everywhere\n",
    "    print(conv5)\n",
    "    \n",
    "    \n",
    "    h= tf.identity(conv5, name='conv_out')\n",
    "        \n",
    "    #put it all together\n",
    "    h = tf.contrib.layers.flatten(h)\n",
    "    loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=h, labels=labels))\n",
    "    outputs.append(h)\n",
    "    losses.append(loss)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"model2/model2/conv1/Relu:0\", shape=(?, 32, 32, 19), dtype=float32)\n",
      "Tensor(\"model2/model2/conv2/Relu:0\", shape=(?, 16, 16, 30), dtype=float32)\n",
      "Tensor(\"model2/model2/conv3/Relu:0\", shape=(?, 8, 8, 50), dtype=float32)\n",
      "Tensor(\"model2/model2/conv4/Relu:0\", shape=(?, 4, 4, 100), dtype=float32)\n",
      "Tensor(\"model2/model2/pool4/MaxPool:0\", shape=(?, 2, 2, 100), dtype=float32)\n",
      "Tensor(\"model2/model2/conv5/Relu:0\", shape=(?, 1, 1, 6), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Step 4: Define multiple models in your ensemble. You should train an ensemble of 5 models.\n",
    "# Let's put all variables in a scope, this makes training ensembles easier. Make sure each model in your ensemble has it's own scope and produces an output and loss\n",
    "with tf.name_scope('model2'), tf.variable_scope('model2'):\n",
    "    # Step 2: define the compute graph of your CNN here (use your solution to HW4 here)\n",
    "    #   Add weight regularization (l2-loss)\n",
    "    \n",
    "    conv1 = tf.contrib.layers.conv2d(inputs=white_inputs,num_outputs=19,kernel_size=[5, 5],weights_regularizer=tf.nn.l2_loss,\n",
    "      stride=2,padding=\"same\",scope='conv1') # put in scope everywhere\n",
    "    print(conv1)\n",
    "    conv1_bn= tf.layers.batch_normalization(conv1,training=True)\n",
    "   \n",
    "    conv2 = tf.contrib.layers.conv2d(inputs=conv1_bn,num_outputs=30,kernel_size=[5, 5],weights_regularizer=tf.nn.l2_loss,\n",
    "      stride=2,padding=\"same\",scope='conv2') # put in scope everywhere\n",
    "    print(conv2)\n",
    "    conv2_bn= tf.layers.batch_normalization(conv2,training=True)\n",
    "    \n",
    "    \n",
    "    conv3 = tf.contrib.layers.conv2d(inputs=conv2_bn,num_outputs=50,kernel_size=[5,5],weights_regularizer=tf.nn.l2_loss,\n",
    "      stride=2,padding=\"same\",scope='conv3') # put in scope everywhere\n",
    "    print(conv3)\n",
    "    conv3_bn= tf.layers.batch_normalization(conv3,training=True)\n",
    "\n",
    "    \n",
    "    conv4 = tf.contrib.layers.conv2d(inputs=conv3_bn,num_outputs=100,kernel_size=[3, 3],weights_regularizer=tf.nn.l2_loss,\n",
    "      stride=2,padding=\"same\",scope='conv4') # put in scope everywhere\n",
    "    print(conv4)\n",
    "#     drop4= tf.contrib.layers.dropout(inputs=conv4,keep_prob=0.5)\n",
    "    conv4_bn= tf.layers.batch_normalization(conv4,training=True)\n",
    "    pool4 = tf.contrib.layers.max_pool2d(inputs=conv4_bn, kernel_size=[2,2], stride=2, scope='pool4')\n",
    "    print(pool4)\n",
    "    \n",
    "    conv5 = tf.contrib.layers.conv2d(inputs=pool4,num_outputs=6,kernel_size=[1,1],weights_regularizer=tf.nn.l2_loss,\n",
    "      stride=2,padding=\"same\",scope='conv5') # put in scope everywhere\n",
    "    print(conv5)\n",
    "    \n",
    "    \n",
    "    h= tf.identity(conv5, name='conv_out')\n",
    "        \n",
    "    #put it all together\n",
    "    h = tf.contrib.layers.flatten(h)\n",
    "    loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=h, labels=labels))\n",
    "    outputs.append(h)\n",
    "    losses.append(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"model3/model3/conv1/Relu:0\", shape=(?, 32, 32, 19), dtype=float32)\n",
      "Tensor(\"model3/model3/conv2/Relu:0\", shape=(?, 16, 16, 30), dtype=float32)\n",
      "Tensor(\"model3/model3/conv3/Relu:0\", shape=(?, 8, 8, 50), dtype=float32)\n",
      "Tensor(\"model3/model3/conv4/Relu:0\", shape=(?, 4, 4, 100), dtype=float32)\n",
      "Tensor(\"model3/model3/pool4/MaxPool:0\", shape=(?, 2, 2, 100), dtype=float32)\n",
      "Tensor(\"model3/model3/conv5/Relu:0\", shape=(?, 1, 1, 6), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Step 4: Define multiple models in your ensemble. You should train an ensemble of 5 models.\n",
    "# Let's put all variables in a scope, this makes training ensembles easier. Make sure each model in your ensemble has it's own scope and produces an output and loss\n",
    "with tf.name_scope('model3'), tf.variable_scope('model3'):\n",
    "    # Step 2: define the compute graph of your CNN here (use your solution to HW4 here)\n",
    "    #   Add weight regularization (l2-loss)\n",
    "    \n",
    "    conv1 = tf.contrib.layers.conv2d(inputs=white_inputs,num_outputs=19,kernel_size=[5, 5],weights_regularizer=tf.nn.l2_loss,\n",
    "      stride=2,padding=\"same\",scope='conv1') # put in scope everywhere\n",
    "    print(conv1)\n",
    "    conv1_bn= tf.layers.batch_normalization(conv1,training=True)\n",
    "   \n",
    "    conv2 = tf.contrib.layers.conv2d(inputs=conv1_bn,num_outputs=30,kernel_size=[5, 5],weights_regularizer=tf.nn.l2_loss,\n",
    "      stride=2,padding=\"same\",scope='conv2') # put in scope everywhere\n",
    "    print(conv2)\n",
    "    conv2_bn= tf.layers.batch_normalization(conv2,training=True)\n",
    "    \n",
    "    \n",
    "    conv3 = tf.contrib.layers.conv2d(inputs=conv2_bn,num_outputs=50,kernel_size=[5,5],weights_regularizer=tf.nn.l2_loss,\n",
    "      stride=2,padding=\"same\",scope='conv3') # put in scope everywhere\n",
    "    print(conv3)\n",
    "    conv3_bn= tf.layers.batch_normalization(conv3,training=True)\n",
    "\n",
    "    \n",
    "    conv4 = tf.contrib.layers.conv2d(inputs=conv3_bn,num_outputs=100,kernel_size=[3, 3],weights_regularizer=tf.nn.l2_loss,\n",
    "      stride=2,padding=\"same\",scope='conv4') # put in scope everywhere\n",
    "    print(conv4)\n",
    "#     drop4= tf.contrib.layers.dropout(inputs=conv4,keep_prob=0.5)\n",
    "    conv4_bn= tf.layers.batch_normalization(conv4,training=True)\n",
    "    pool4 = tf.contrib.layers.max_pool2d(inputs=conv4_bn, kernel_size=[2,2], stride=2, scope='pool4')\n",
    "    print(pool4)\n",
    "    \n",
    "    conv5 = tf.contrib.layers.conv2d(inputs=pool4,num_outputs=6,kernel_size=[1,1],weights_regularizer=tf.nn.l2_loss,\n",
    "      stride=2,padding=\"same\",scope='conv5') # put in scope everywhere\n",
    "    print(conv5)\n",
    "    \n",
    "    \n",
    "    h= tf.identity(conv5, name='conv_out')\n",
    "        \n",
    "    #put it all together\n",
    "    h = tf.contrib.layers.flatten(h)\n",
    "    loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=h, labels=labels))\n",
    "    outputs.append(h)\n",
    "    losses.append(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"model4/model4/conv1/Relu:0\", shape=(?, 32, 32, 19), dtype=float32)\n",
      "Tensor(\"model4/model4/conv2/Relu:0\", shape=(?, 16, 16, 30), dtype=float32)\n",
      "Tensor(\"model4/model4/conv3/Relu:0\", shape=(?, 8, 8, 50), dtype=float32)\n",
      "Tensor(\"model4/model4/conv4/Relu:0\", shape=(?, 4, 4, 100), dtype=float32)\n",
      "Tensor(\"model4/model4/pool4/MaxPool:0\", shape=(?, 2, 2, 100), dtype=float32)\n",
      "Tensor(\"model4/model4/conv5/Relu:0\", shape=(?, 1, 1, 6), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Step 4: Define multiple models in your ensemble. You should train an ensemble of 5 models.\n",
    "# Let's put all variables in a scope, this makes training ensembles easier. Make sure each model in your ensemble has it's own scope and produces an output and loss\n",
    "with tf.name_scope('model4'), tf.variable_scope('model4'):\n",
    "    # Step 2: define the compute graph of your CNN here (use your solution to HW4 here)\n",
    "    #   Add weight regularization (l2-loss)\n",
    "    \n",
    "    conv1 = tf.contrib.layers.conv2d(inputs=white_inputs,num_outputs=19,kernel_size=[5, 5],weights_regularizer=tf.nn.l2_loss,\n",
    "      stride=2,padding=\"same\",scope='conv1') # put in scope everywhere\n",
    "    print(conv1)\n",
    "    conv1_bn= tf.layers.batch_normalization(conv1,training=True)\n",
    "   \n",
    "    conv2 = tf.contrib.layers.conv2d(inputs=conv1_bn,num_outputs=30,kernel_size=[5, 5],weights_regularizer=tf.nn.l2_loss,\n",
    "      stride=2,padding=\"same\",scope='conv2') # put in scope everywhere\n",
    "    print(conv2)\n",
    "    conv2_bn= tf.layers.batch_normalization(conv2,training=True)\n",
    "    \n",
    "    \n",
    "    conv3 = tf.contrib.layers.conv2d(inputs=conv2_bn,num_outputs=50,kernel_size=[5,5],weights_regularizer=tf.nn.l2_loss,\n",
    "      stride=2,padding=\"same\",scope='conv3') # put in scope everywhere\n",
    "    print(conv3)\n",
    "    conv3_bn= tf.layers.batch_normalization(conv3,training=True)\n",
    "\n",
    "    \n",
    "    conv4 = tf.contrib.layers.conv2d(inputs=conv3_bn,num_outputs=100,kernel_size=[3, 3],weights_regularizer=tf.nn.l2_loss,\n",
    "      stride=2,padding=\"same\",scope='conv4') # put in scope everywhere\n",
    "    print(conv4)\n",
    "#     drop4= tf.contrib.layers.dropout(inputs=conv4,keep_prob=0.5)\n",
    "    conv4_bn= tf.layers.batch_normalization(conv4,training=True)\n",
    "    pool4 = tf.contrib.layers.max_pool2d(inputs=conv4_bn, kernel_size=[2,2], stride=2, scope='pool4')\n",
    "    print(pool4)\n",
    "    \n",
    "    conv5 = tf.contrib.layers.conv2d(inputs=pool4,num_outputs=6,kernel_size=[1,1],weights_regularizer=tf.nn.l2_loss,\n",
    "      stride=2,padding=\"same\",scope='conv5') # put in scope everywhere\n",
    "    print(conv5)\n",
    "    \n",
    "    \n",
    "    h= tf.identity(conv5, name='conv_out')\n",
    "        \n",
    "    #put it all together\n",
    "    h = tf.contrib.layers.flatten(h)\n",
    "    loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=h, labels=labels))\n",
    "    outputs.append(h)\n",
    "    losses.append(loss)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"model5/model5/conv1/Relu:0\", shape=(?, 32, 32, 19), dtype=float32)\n",
      "Tensor(\"model5/model5/conv2/Relu:0\", shape=(?, 16, 16, 30), dtype=float32)\n",
      "Tensor(\"model5/model5/conv3/Relu:0\", shape=(?, 8, 8, 50), dtype=float32)\n",
      "Tensor(\"model5/model5/conv4/Relu:0\", shape=(?, 4, 4, 100), dtype=float32)\n",
      "Tensor(\"model5/model5/pool4/MaxPool:0\", shape=(?, 2, 2, 100), dtype=float32)\n",
      "Tensor(\"model5/model5/conv5/Relu:0\", shape=(?, 1, 1, 6), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Step 4: Define multiple models in your ensemble. You should train an ensemble of 5 models.\n",
    "# Let's put all variables in a scope, this makes training ensembles easier. Make sure each model in your ensemble has it's own scope and produces an output and loss\n",
    "with tf.name_scope('model5'), tf.variable_scope('model5'):\n",
    "    # Step 2: define the compute graph of your CNN here (use your solution to HW4 here)\n",
    "    #   Add weight regularization (l2-loss)\n",
    "    \n",
    "    conv1 = tf.contrib.layers.conv2d(inputs=white_inputs,num_outputs=19,kernel_size=[5, 5],weights_regularizer=tf.nn.l2_loss,\n",
    "      stride=2,padding=\"same\",scope='conv1') # put in scope everywhere\n",
    "    print(conv1)\n",
    "    conv1_bn= tf.layers.batch_normalization(conv1,training=True)\n",
    "   \n",
    "    conv2 = tf.contrib.layers.conv2d(inputs=conv1_bn,num_outputs=30,kernel_size=[5, 5],weights_regularizer=tf.nn.l2_loss,\n",
    "      stride=2,padding=\"same\",scope='conv2') # put in scope everywhere\n",
    "    print(conv2)\n",
    "    conv2_bn= tf.layers.batch_normalization(conv2,training=True)\n",
    "    \n",
    "    \n",
    "    conv3 = tf.contrib.layers.conv2d(inputs=conv2_bn,num_outputs=50,kernel_size=[5,5],weights_regularizer=tf.nn.l2_loss,\n",
    "      stride=2,padding=\"same\",scope='conv3') # put in scope everywhere\n",
    "    print(conv3)\n",
    "    conv3_bn= tf.layers.batch_normalization(conv3,training=True)\n",
    "\n",
    "    \n",
    "    conv4 = tf.contrib.layers.conv2d(inputs=conv3_bn,num_outputs=100,kernel_size=[3, 3],weights_regularizer=tf.nn.l2_loss,\n",
    "      stride=2,padding=\"same\",scope='conv4') # put in scope everywhere\n",
    "    print(conv4)\n",
    "#     drop4= tf.contrib.layers.dropout(inputs=conv4,keep_prob=0.5)\n",
    "    conv4_bn= tf.layers.batch_normalization(conv4,training=True)\n",
    "    pool4 = tf.contrib.layers.max_pool2d(inputs=conv4_bn, kernel_size=[2,2], stride=2, scope='pool4')\n",
    "    print(pool4)\n",
    "    \n",
    "    conv5 = tf.contrib.layers.conv2d(inputs=pool4,num_outputs=6,kernel_size=[1,1],weights_regularizer=tf.nn.l2_loss,\n",
    "      stride=2,padding=\"same\",scope='conv5') # put in scope everywhere\n",
    "    print(conv5)\n",
    "    \n",
    "    \n",
    "    h= tf.identity(conv5, name='conv_out')\n",
    "        \n",
    "    #put it all together\n",
    "    h = tf.contrib.layers.flatten(h)\n",
    "    loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=h, labels=labels))\n",
    "    outputs.append(h)\n",
    "    losses.append(loss)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of variables used  496890 / 500000\n"
     ]
    }
   ],
   "source": [
    "output = tf.add_n(outputs, name='output')\n",
    "\n",
    "# Sum up all the losses\n",
    "loss = tf.add_n(losses)\n",
    "regularization_loss = tf.losses.get_regularization_loss()\n",
    "# Let's weight the regularization loss down, otherwise it will hurt the model performance\n",
    "# You can tune this weight if you wish\n",
    "total_loss = loss + 1e-6 * regularization_loss\n",
    "\n",
    "# create an optimizer: Adam might work slightly better (it's a bit faster for Tux)\n",
    "optimizer = tf.train.AdamOptimizer(lrate, 0.9, 0.999)\n",
    "\n",
    "# use that optimizer on your loss function\n",
    "opt = optimizer.minimize(total_loss)\n",
    "correct = tf.equal(tf.argmax(output, 1), labels)\n",
    "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "\n",
    "# You're allowed to use 500k variables this time, 100k per model in your ensemble.\n",
    "print( \"Total number of variables used \", np.sum([v.get_shape().num_elements() for v in tf.trainable_variables()]), '/', 500000 )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 2: Training\n",
    "\n",
    "Training might take up to 20 min depending on your architecture.  This time around you should get close to 100% training accuracy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[  0] Accuracy: 0.928  \t  Loss: 1.824  \t  validation accuracy: 0.914\n",
      "[  1] Accuracy: 0.971  \t  Loss: 0.737  \t  validation accuracy: 0.931\n",
      "[  2] Accuracy: 0.982  \t  Loss: 0.431  \t  validation accuracy: 0.946\n",
      "[  3] Accuracy: 0.987  \t  Loss: 0.328  \t  validation accuracy: 0.945\n",
      "[  4] Accuracy: 0.990  \t  Loss: 0.274  \t  validation accuracy: 0.945\n",
      "[  5] Accuracy: 0.992  \t  Loss: 0.234  \t  validation accuracy: 0.950\n",
      "[  6] Accuracy: 0.995  \t  Loss: 0.185  \t  validation accuracy: 0.957\n",
      "[  7] Accuracy: 0.996  \t  Loss: 0.157  \t  validation accuracy: 0.963\n",
      "[  8] Accuracy: 0.996  \t  Loss: 0.156  \t  validation accuracy: 0.948\n",
      "[  9] Accuracy: 0.997  \t  Loss: 0.135  \t  validation accuracy: 0.956\n",
      "[ 10] Accuracy: 0.997  \t  Loss: 0.128  \t  validation accuracy: 0.960\n",
      "[ 11] Accuracy: 0.998  \t  Loss: 0.108  \t  validation accuracy: 0.965\n",
      "[ 12] Accuracy: 0.998  \t  Loss: 0.088  \t  validation accuracy: 0.965\n",
      "[ 13] Accuracy: 0.998  \t  Loss: 0.107  \t  validation accuracy: 0.963\n",
      "[ 14] Accuracy: 0.998  \t  Loss: 0.095  \t  validation accuracy: 0.966\n",
      "[ 15] Accuracy: 0.999  \t  Loss: 0.091  \t  validation accuracy: 0.965\n",
      "[ 16] Accuracy: 0.998  \t  Loss: 0.082  \t  validation accuracy: 0.964\n",
      "[ 17] Accuracy: 0.999  \t  Loss: 0.071  \t  validation accuracy: 0.963\n",
      "[ 18] Accuracy: 0.999  \t  Loss: 0.063  \t  validation accuracy: 0.965\n",
      "[ 19] Accuracy: 0.999  \t  Loss: 0.078  \t  validation accuracy: 0.956\n",
      "[ 20] Accuracy: 0.999  \t  Loss: 0.068  \t  validation accuracy: 0.958\n",
      "[ 21] Accuracy: 0.999  \t  Loss: 0.058  \t  validation accuracy: 0.968\n",
      "[ 22] Accuracy: 0.999  \t  Loss: 0.051  \t  validation accuracy: 0.963\n",
      "[ 23] Accuracy: 0.999  \t  Loss: 0.049  \t  validation accuracy: 0.963\n",
      "[ 24] Accuracy: 0.999  \t  Loss: 0.075  \t  validation accuracy: 0.967\n"
     ]
    }
   ],
   "source": [
    "image_val, label_val = load('tux_val.dat')\n",
    "\n",
    "# Batch size\n",
    "BS = 32\n",
    "\n",
    "# Start a session\n",
    "sess = tf.Session()\n",
    "\n",
    "# Set up training\n",
    "sess.run(tf.global_variables_initializer())\n",
    "\n",
    "# Train convnet\n",
    "# Step 3: You should tune the number of epochs to maximize validation accuracy, you can either do this by hand or automate the process.\n",
    "# Adding an automated way to early stopping, we're also saving the model if it's the best valid accuracy.\n",
    "\n",
    "for epoch in range(25):\n",
    "    # Let's shuffle the data every epoch\n",
    "    np.random.seed(epoch)\n",
    "    np.random.shuffle(image_data)\n",
    "    np.random.seed(epoch)\n",
    "    np.random.shuffle(label_data)\n",
    "    # Go through the entire dataset once\n",
    "    accuracy_vals, loss_vals = [], []\n",
    "    for i in range(0, image_data.shape[0]-BS+1, BS):\n",
    "        # Train a single batch\n",
    "        batch_images, batch_labels = image_data[i:i+BS], label_data[i:i+BS]\n",
    "        accuracy_val, loss_val, _ = sess.run([accuracy, total_loss, opt], feed_dict={inputs: batch_images, labels: batch_labels,lrate:0.0005})\n",
    "        accuracy_vals.append(accuracy_val)\n",
    "        loss_vals.append(loss_val)\n",
    "\n",
    "    val_correct = []\n",
    "    for i in range(0, image_val.shape[0], BS):\n",
    "        batch_images, batch_labels = image_val[i:i+BS], label_val[i:i+BS]\n",
    "        val_correct.extend( sess.run(correct, feed_dict={eval_inputs: batch_images, labels: batch_labels}) )\n",
    "    print('[%3d] Accuracy: %0.3f  \\t  Loss: %0.3f  \\t  validation accuracy: %0.3f'%(epoch, np.mean(accuracy_vals), np.mean(loss_vals), np.mean(val_correct)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 3: Evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute the valiation accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input shape: (3912, 64, 64, 3)\n",
      "Labels shape: (3912,)\n",
      "ConvNet Validation Accuracy:  0.966768916155\n"
     ]
    }
   ],
   "source": [
    "image_val, label_val = load('tux_val.dat')\n",
    "\n",
    "print('Input shape: ' + str(image_val.shape))\n",
    "print('Labels shape: ' + str(label_val.shape))\n",
    "\n",
    "val_correct = []\n",
    "for i in range(0, image_val.shape[0], BS):\n",
    "    batch_images, batch_labels = image_val[i:i+BS], label_val[i:i+BS]\n",
    "    val_correct.extend( sess.run(correct, feed_dict={eval_inputs: batch_images, labels: batch_labels}) )\n",
    "print(\"ConvNet Validation Accuracy: \", np.mean(val_correct))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 4: Save Model\n",
    "Please note that we also want you to turn in your ipynb for this assignment.  Zip up the ipynb along with the tfg for your submission."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# util.save('assignment5_v3_9716.tfg', session=sess)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Part 5 (optional): See your model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub data rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_data_rate_limit`.\n"
     ]
    }
   ],
   "source": [
    "# Show the current graph\n",
    "util.show_graph(tf.get_default_graph().as_graph_def())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}