notebooks/3_NeuralNetworks/neural_network_eager_api.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neural Network with Eager API\n",
    "\n",
    "Build a 2-hidden layers fully connected neural network (a.k.a multilayer perceptron) with TensorFlow's Eager API.\n",
    "\n",
    "This example is using some of TensorFlow higher-level wrappers (tf.estimators, tf.layers, tf.metrics, ...), you can check 'neural_network_raw' example for a raw, and more detailed TensorFlow implementation.\n",
    "\n",
    "- Author: Aymeric Damien\n",
    "- Project: https://github.com/aymericdamien/TensorFlow-Examples/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Neural Network Overview\n",
    "\n",
    "<img src=\"http://cs231n.github.io/assets/nn1/neural_net2.jpeg\" alt=\"nn\" style=\"width: 400px;\"/>\n",
    "\n",
    "## MNIST Dataset Overview\n",
    "\n",
    "This example is using MNIST handwritten digits. The dataset contains 60,000 examples for training and 10,000 examples for testing. The digits have been size-normalized and centered in a fixed-size image (28x28 pixels) with values from 0 to 1. For simplicity, each image has been flattened and converted to a 1-D numpy array of 784 features (28*28).\n",
    "\n",
    "![MNIST Dataset](http://neuralnetworksanddeeplearning.com/images/mnist_100_digits.png)\n",
    "\n",
    "More info: http://yann.lecun.com/exdb/mnist/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Set Eager API\n",
    "tf.enable_eager_execution()\n",
    "tfe = tf.contrib.eager"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/data/train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# Import MNIST data\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"/tmp/data/\", one_hot=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Parameters\n",
    "learning_rate = 0.001\n",
    "num_steps = 1000\n",
    "batch_size = 128\n",
    "display_step = 100\n",
    "\n",
    "# Network Parameters\n",
    "n_hidden_1 = 256 # 1st layer number of neurons\n",
    "n_hidden_2 = 256 # 2nd layer number of neurons\n",
    "num_input = 784 # MNIST data input (img shape: 28*28)\n",
    "num_classes = 10 # MNIST total classes (0-9 digits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Using TF Dataset to split data into batches\n",
    "dataset = tf.data.Dataset.from_tensor_slices(\n",
    "    (mnist.train.images, mnist.train.labels))\n",
    "dataset = dataset.repeat().batch(batch_size).prefetch(batch_size)\n",
    "dataset_iter = tfe.Iterator(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Define the neural network. To use eager API and tf.layers API together,\n",
    "# we must instantiate a tfe.Network class as follow:\n",
    "class NeuralNet(tfe.Network):\n",
    "    def __init__(self):\n",
    "        # Define each layer\n",
    "        super(NeuralNet, self).__init__()\n",
    "        # Hidden fully connected layer with 256 neurons\n",
    "        self.layer1 = self.track_layer(\n",
    "            tf.layers.Dense(n_hidden_1, activation=tf.nn.relu))\n",
    "        # Hidden fully connected layer with 256 neurons\n",
    "        self.layer2 = self.track_layer(\n",
    "            tf.layers.Dense(n_hidden_2, activation=tf.nn.relu))\n",
    "        # Output fully connected layer with a neuron for each class\n",
    "        self.out_layer = self.track_layer(tf.layers.Dense(num_classes))\n",
    "\n",
    "    def call(self, x):\n",
    "        x = self.layer1(x)\n",
    "        x = self.layer2(x)\n",
    "        return self.out_layer(x)\n",
    "\n",
    "\n",
    "neural_net = NeuralNet()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Cross-Entropy loss function\n",
    "def loss_fn(inference_fn, inputs, labels):\n",
    "    # Using sparse_softmax cross entropy\n",
    "    return tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(\n",
    "        logits=inference_fn(inputs), labels=labels))\n",
    "\n",
    "\n",
    "# Calculate accuracy\n",
    "def accuracy_fn(inference_fn, inputs, labels):\n",
    "    prediction = tf.nn.softmax(inference_fn(inputs))\n",
    "    correct_pred = tf.equal(tf.argmax(prediction, 1), labels)\n",
    "    return tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "\n",
    "\n",
    "# SGD Optimizer\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "\n",
    "# Compute gradients\n",
    "grad = tfe.implicit_gradients(loss_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initial loss= 2.340397596\n",
      "Step: 0001  loss= 2.340397596  accuracy= 0.0703\n",
      "Step: 0100  loss= 0.586046159  accuracy= 0.8305\n",
      "Step: 0200  loss= 0.253318846  accuracy= 0.9282\n",
      "Step: 0300  loss= 0.214748293  accuracy= 0.9377\n",
      "Step: 0400  loss= 0.180644721  accuracy= 0.9466\n",
      "Step: 0500  loss= 0.137285724  accuracy= 0.9591\n",
      "Step: 0600  loss= 0.119845696  accuracy= 0.9636\n",
      "Step: 0700  loss= 0.113618039  accuracy= 0.9665\n",
      "Step: 0800  loss= 0.109642141  accuracy= 0.9676\n",
      "Step: 0900  loss= 0.085067607  accuracy= 0.9746\n",
      "Step: 1000  loss= 0.079819344  accuracy= 0.9754\n"
     ]
    }
   ],
   "source": [
    "# Training\n",
    "average_loss = 0.\n",
    "average_acc = 0.\n",
    "for step in range(num_steps):\n",
    "\n",
    "    # Iterate through the dataset\n",
    "    d = dataset_iter.next()\n",
    "    \n",
    "    # Images\n",
    "    x_batch = d[0]\n",
    "    # Labels\n",
    "    y_batch = tf.cast(d[1], dtype=tf.int64)\n",
    "\n",
    "    # Compute the batch loss\n",
    "    batch_loss = loss_fn(neural_net, x_batch, y_batch)\n",
    "    average_loss += batch_loss\n",
    "    # Compute the batch accuracy\n",
    "    batch_accuracy = accuracy_fn(neural_net, x_batch, y_batch)\n",
    "    average_acc += batch_accuracy\n",
    "\n",
    "    if step == 0:\n",
    "        # Display the initial cost, before optimizing\n",
    "        print(\"Initial loss= {:.9f}\".format(average_loss))\n",
    "\n",
    "    # Update the variables following gradients info\n",
    "    optimizer.apply_gradients(grad(neural_net, x_batch, y_batch))\n",
    "\n",
    "    # Display info\n",
    "    if (step + 1) % display_step == 0 or step == 0:\n",
    "        if step > 0:\n",
    "            average_loss /= display_step\n",
    "            average_acc /= display_step\n",
    "        print(\"Step:\", '%04d' % (step + 1), \" loss=\",\n",
    "              \"{:.9f}\".format(average_loss), \" accuracy=\",\n",
    "              \"{:.4f}\".format(average_acc))\n",
    "        average_loss = 0.\n",
    "        average_acc = 0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testset Accuracy: 0.9719\n"
     ]
    }
   ],
   "source": [
    "# Evaluate model on the test image set\n",
    "testX = mnist.test.images\n",
    "testY = mnist.test.labels\n",
    "\n",
    "test_acc = accuracy_fn(neural_net, testX, testY)\n",
    "print(\"Testset Accuracy: {:.4f}\".format(test_acc))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}