7_1.small_vgg_mnist.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#%env CUDA_DEVICE_ORDER=PCI_BUS_ID\n",
    "#%env CUDA_VISIBLE_DEVICES=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "from scipy.misc import imread, imresize\n",
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "import matplotlib.pyplot as plt\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "mnist = input_data.read_data_sets(\"../data/\", one_hot=True)\n",
    "# Check out https://www.tensorflow.org/get_started/mnist/beginners for\n",
    "# more information about the mnist dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# hyper parameters\n",
    "learning_rate = 0.001\n",
    "training_epochs = 70\n",
    "batch_size = 64\n",
    "n_class = 10\n",
    "seed = 777\n",
    "tf.set_random_seed(seed)\n",
    "cur_dir = os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def preproc(x):\n",
    "    # x = x*2 - 1.0\n",
    "    # per-example mean subtraction (http://ufldl.stanford.edu/wiki/index.php/Data_Preprocessing)\n",
    "    mean = tf.reduce_mean(x, axis=1, keep_dims=True)\n",
    "    return x - mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def conv_bn_activ_dropout(name, x, n_filters, kernel_size, strides, dropout_rate, training, seed, \n",
    "                          padding='SAME', activ_fn=tf.nn.relu):\n",
    "    with tf.variable_scope(name):\n",
    "        net = tf.layers.conv2d(x, n_filters, kernel_size, strides=strides, padding=padding, use_bias=False,\n",
    "                              kernel_initializer=tf.contrib.layers.xavier_initializer(seed=seed))\n",
    "        net = tf.layers.batch_normalization(net, training=training)\n",
    "        net = activ_fn(net)\n",
    "        if dropout_rate > 0.0:            \n",
    "            net = tf.layers.dropout(net, rate=dropout_rate, training=training, seed=seed)\n",
    "    return net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def build_vgg(X_img, dropout_rate, seed):\n",
    "    net = X_img\n",
    "    n_filters = 64\n",
    "    for i in range(3):\n",
    "        net = conv_bn_activ_dropout(name=\"3x3conv{}-{}\".format(i+1, 1), x=net,\n",
    "                                    n_filters=n_filters, kernel_size=[3,3], strides=1, \n",
    "                                    dropout_rate=dropout_rate, training=is_train, seed=seed)\n",
    "        net = conv_bn_activ_dropout(name=\"3x3conv{}-{}\".format(i+1, 2), x=net, \n",
    "                                    n_filters=n_filters, kernel_size=[3,3], strides=1, \n",
    "                                    dropout_rate=dropout_rate, training=is_train, seed=seed)\n",
    "        if i == 2:\n",
    "            net = conv_bn_activ_dropout(name=\"1x1conv\", x=net, \n",
    "                                        n_filters=n_filters, kernel_size=[1,1], strides=1, \n",
    "                                        dropout_rate=dropout_rate, training=is_train, seed=seed)\n",
    "        n_filters *= 2\n",
    "        net = conv_bn_activ_dropout(name=\"5x5stridepool{}\".format(i+1), x=net, n_filters=n_filters,\n",
    "                                   kernel_size=[5,5], strides=2, dropout_rate=dropout_rate,\n",
    "                                   training=is_train, seed=seed)\n",
    "    \n",
    "    net = tf.contrib.layers.flatten(net)\n",
    "    logits = tf.layers.dense(net, 10, kernel_initializer=tf.contrib.layers.xavier_initializer(seed=seed),\n",
    "                            name=\"logits\")\n",
    "    \n",
    "    return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = tf.placeholder(tf.float32, [None, 784], name=\"X\")\n",
    "Y = tf.placeholder(tf.float32, [None, n_class], name = \"Y\")\n",
    "is_train = tf.placeholder(tf.bool, name=\"is_train\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_pre = preproc(X)\n",
    "X_img = tf.reshape(X_pre, [-1, 28, 28, 1], name=\"X_img\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "logits = build_vgg(X_img, dropout_rate=0.3, seed=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=Y), name=\"loss\")\n",
    "global_step = tf.Variable(0, trainable=False)\n",
    "learningRate = tf.train.exponential_decay(learning_rate=learning_rate,\n",
    "                                          global_step= global_step,\n",
    "                                          decay_steps=5000,\n",
    "                                          decay_rate= 0.1,\n",
    "                                          staircase=True)\n",
    "update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n",
    "with tf.control_dependencies(update_ops):\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate=learningRate).minimize(\n",
    "        loss, global_step=global_step, name=\"optimizer\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred = tf.argmax(logits, axis=1, name=\"prediction\")\n",
    "prob = tf.nn.softmax(logits, name=\"softmax\")\n",
    "accuracy = tf.reduce_mean(tf.cast(tf.equal(pred, tf.argmax(Y, axis=1)), tf.float32), name=\"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_var = [X, Y, is_train, logits, pred, prob, accuracy]\n",
    "tf.add_to_collection('train_var', train_var[0])\n",
    "tf.add_to_collection('train_var', train_var[1])\n",
    "tf.add_to_collection('train_var', train_var[2])\n",
    "tf.add_to_collection('train_var', train_var[3])\n",
    "tf.add_to_collection('train_var', train_var[4])\n",
    "tf.add_to_collection('train_var', train_var[5])\n",
    "tf.add_to_collection('train_var', train_var[6])\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "savedir = 'checkpoints'\n",
    "if not os.path.exists(savedir):\n",
    "    os.makedirs(savedir)\n",
    "saver.export_meta_graph(os.path.join(cur_dir, savedir, 'mnist_vggsmall.meta'), collection_list=['train_var'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# initialize\n",
    "sess = tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth =True)))\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# train my model\n",
    "print('Learning started. It takes sometime.')\n",
    "max_test_acc = 0.\n",
    "for epoch in range(training_epochs):\n",
    "    avg_cost = 0.\n",
    "    avg_train_acc = 0.\n",
    "    avg_test_acc = 0.\n",
    "    \n",
    "    total_batch = int(mnist.train.num_examples / batch_size)\n",
    "    total_batch_test = int(mnist.test.num_examples / batch_size)\n",
    "\n",
    "    for i in range(total_batch):\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(batch_size)\n",
    "        #batch_xs = batch_xs.reshape(-1, time_steps, element_size)\n",
    "        feed_dict = {X: batch_xs, Y: batch_ys, is_train:True}\n",
    "        acc, c, _ = sess.run([accuracy, loss, optimizer], feed_dict=feed_dict)\n",
    "        avg_cost += c / total_batch\n",
    "        avg_train_acc += acc / total_batch\n",
    "        \n",
    "    for i in range(total_batch_test):\n",
    "        batch_xs, batch_ys = mnist.test.next_batch(batch_size)        \n",
    "        #batch_xs = batch_xs.reshape(-1, time_steps, element_size)\n",
    "        feed_dict = {X: batch_xs, Y: batch_ys, is_train:False}\n",
    "        acc = sess.run(accuracy, feed_dict=feed_dict)\n",
    "        avg_test_acc += acc / total_batch_test\n",
    "    if avg_test_acc > max_test_acc:\n",
    "        max_test_acc = avg_test_acc\n",
    "        print('saving a graph and weights => accuracy : {}'.format(max_test_acc))\n",
    "        saver.save(sess, os.path.join(cur_dir, savedir, 'mnist_vggsmall.ckpt'))\n",
    "\n",
    "    print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.9f}'.format(avg_cost), \n",
    "          'train accuracy = ', '{:.5f}'.format(avg_train_acc), \n",
    "          'test accuracy = ', '{:.5f}'.format(avg_test_acc))\n",
    "\n",
    "\n",
    "print('Learning Finished!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#saver.save(sess, os.path.join(cur_dir, 'checkpoints', 'mnist_save.ckpt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def evaluate(X_sample, y_sample, batch_size=100):\n",
    "    \"\"\"Run a minibatch accuracy op\"\"\"\n",
    "\n",
    "    N = X_sample.shape[0]\n",
    "    correct_sample = 0\n",
    "\n",
    "    for i in range(0, N, batch_size):\n",
    "        X_batch = X_sample[i: i + batch_size]\n",
    "        y_batch = y_sample[i: i + batch_size]\n",
    "        N_batch = X_batch.shape[0]\n",
    "\n",
    "        feed = {\n",
    "            X: X_batch,\n",
    "            Y: y_batch,\n",
    "            is_train: False\n",
    "        }\n",
    "\n",
    "        correct_sample += sess.run(accuracy, feed_dict=feed) * N_batch\n",
    "\n",
    "    return correct_sample / N\n",
    "\n",
    "print(\"\\nAccuracy Evaluates\")\n",
    "print(\"-------------------------------\")\n",
    "print('Train Accuracy:', '{:.5f}'.format(evaluate(mnist.train.images, mnist.train.labels)))\n",
    "print('Test Accuracy:', '{:.5f}'.format(evaluate(mnist.test.images, mnist.test.labels)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}