example/notebooks/simple_bind.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this example, we will show how to use ```simple_bind``` API. \n",
    "\n",
    "Note it is a low level API, by using low level API we are able to touch more details about MXNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import mxnet as mx\n",
    "import numpy as np\n",
    "import logging\n",
    "\n",
    "logger = logging.getLogger()\n",
    "logger.setLevel(logging.DEBUG)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will use a very simple 1 hidden layer BatchNorm fully connected MNIST network to demo how to use low level API.\n",
    "The network looks like:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.36.0 (20140111.2315)\n",
       " -->\n",
       "<!-- Title: plot Pages: 1 -->\n",
       "<svg width=\"144pt\" height=\"506pt\"\n",
       " viewBox=\"0.00 0.00 144.00 506.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 502)\">\n",
       "<title>plot</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-502 140,-502 140,4 -4,4\"/>\n",
       "<!-- fc1 -->\n",
       "<g id=\"node1\" class=\"node\"><title>fc1</title>\n",
       "<ellipse fill=\"#fb8072\" stroke=\"black\" cx=\"68\" cy=\"-29\" rx=\"68.2532\" ry=\"29\"/>\n",
       "<text text-anchor=\"middle\" x=\"68\" y=\"-32.8\" font-family=\"Times,serif\" font-size=\"14.00\">FullyConnected</text>\n",
       "<text text-anchor=\"middle\" x=\"68\" y=\"-17.8\" font-family=\"Times,serif\" font-size=\"14.00\">128</text>\n",
       "</g>\n",
       "<!-- bn1 -->\n",
       "<g id=\"node2\" class=\"node\"><title>bn1</title>\n",
       "<ellipse fill=\"#bebada\" stroke=\"black\" cx=\"68\" cy=\"-139\" rx=\"47\" ry=\"29\"/>\n",
       "<text text-anchor=\"middle\" x=\"68\" y=\"-135.3\" font-family=\"Times,serif\" font-size=\"14.00\">BatchNorm</text>\n",
       "</g>\n",
       "<!-- bn1&#45;&gt;fc1 -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>bn1&#45;&gt;fc1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M68,-99.8131C68,-86.1516 68,-71.0092 68,-58.3283\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"68,-109.906 63.5001,-99.9062 68,-104.906 68.0001,-99.9062 68.0001,-99.9062 68.0001,-99.9062 68,-104.906 72.5001,-99.9062 68,-109.906 68,-109.906\"/>\n",
       "<text text-anchor=\"middle\" x=\"78.5\" y=\"-80.3\" font-family=\"Times,serif\" font-size=\"14.00\">128</text>\n",
       "</g>\n",
       "<!-- act1 -->\n",
       "<g id=\"node3\" class=\"node\"><title>act1</title>\n",
       "<ellipse fill=\"#ffffb3\" stroke=\"black\" cx=\"68\" cy=\"-249\" rx=\"48.4635\" ry=\"29\"/>\n",
       "<text text-anchor=\"middle\" x=\"68\" y=\"-252.8\" font-family=\"Times,serif\" font-size=\"14.00\">Activation</text>\n",
       "<text text-anchor=\"middle\" x=\"68\" y=\"-237.8\" font-family=\"Times,serif\" font-size=\"14.00\">tanh</text>\n",
       "</g>\n",
       "<!-- act1&#45;&gt;bn1 -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>act1&#45;&gt;bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M68,-209.813C68,-196.152 68,-181.009 68,-168.328\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"68,-219.906 63.5001,-209.906 68,-214.906 68.0001,-209.906 68.0001,-209.906 68.0001,-209.906 68,-214.906 72.5001,-209.906 68,-219.906 68,-219.906\"/>\n",
       "<text text-anchor=\"middle\" x=\"78.5\" y=\"-190.3\" font-family=\"Times,serif\" font-size=\"14.00\">128</text>\n",
       "</g>\n",
       "<!-- fc2 -->\n",
       "<g id=\"node4\" class=\"node\"><title>fc2</title>\n",
       "<ellipse fill=\"#fb8072\" stroke=\"black\" cx=\"68\" cy=\"-359\" rx=\"68.2532\" ry=\"29\"/>\n",
       "<text text-anchor=\"middle\" x=\"68\" y=\"-362.8\" font-family=\"Times,serif\" font-size=\"14.00\">FullyConnected</text>\n",
       "<text text-anchor=\"middle\" x=\"68\" y=\"-347.8\" font-family=\"Times,serif\" font-size=\"14.00\">10</text>\n",
       "</g>\n",
       "<!-- fc2&#45;&gt;act1 -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>fc2&#45;&gt;act1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M68,-319.813C68,-306.152 68,-291.009 68,-278.328\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"68,-329.906 63.5001,-319.906 68,-324.906 68.0001,-319.906 68.0001,-319.906 68.0001,-319.906 68,-324.906 72.5001,-319.906 68,-329.906 68,-329.906\"/>\n",
       "<text text-anchor=\"middle\" x=\"78.5\" y=\"-300.3\" font-family=\"Times,serif\" font-size=\"14.00\">128</text>\n",
       "</g>\n",
       "<!-- softmax -->\n",
       "<g id=\"node5\" class=\"node\"><title>softmax</title>\n",
       "<ellipse fill=\"#fccde5\" stroke=\"black\" cx=\"68\" cy=\"-469\" rx=\"55.0152\" ry=\"29\"/>\n",
       "<text text-anchor=\"middle\" x=\"68\" y=\"-465.3\" font-family=\"Times,serif\" font-size=\"14.00\">SoftmaxOutput</text>\n",
       "</g>\n",
       "<!-- softmax&#45;&gt;fc2 -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>softmax&#45;&gt;fc2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M68,-429.813C68,-416.152 68,-401.009 68,-388.328\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"68,-439.906 63.5001,-429.906 68,-434.906 68.0001,-429.906 68.0001,-429.906 68.0001,-429.906 68,-434.906 72.5001,-429.906 68,-439.906 68,-439.906\"/>\n",
       "<text text-anchor=\"middle\" x=\"75\" y=\"-410.3\" font-family=\"Times,serif\" font-size=\"14.00\">10</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x7f4da2a90e10>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# we can use mx.sym in short of mx.symbol\n",
    "data = mx.sym.Variable(\"data\")\n",
    "fc1 = mx.sym.FullyConnected(data=data, num_hidden=128, name=\"fc1\")\n",
    "bn1 = mx.sym.BatchNorm(data=fc1, name=\"bn1\")\n",
    "act1 = mx.sym.Activation(data=bn1, name=\"act1\", act_type=\"tanh\")\n",
    "fc2 = mx.sym.FullyConnected(data=act1, name=\"fc2\", num_hidden=10)\n",
    "softmax = mx.sym.Softmax(data=fc2, name=\"softmax\")\n",
    "# visualize the network\n",
    "batch_size = 100\n",
    "data_shape = (batch_size, 784)\n",
    "mx.viz.plot_network(softmax, shape={\"data\":data_shape}, node_attrs={\"shape\":'oval',\"fixedsize\":'false'})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can use ```simple_bind``` api to generate executor from symbol."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# context different to ```mx.model```, \n",
    "# In mx.model, we wrapped parameter server, but for a single executor, the context is only able to be ONE device\n",
    "# run on cpu\n",
    "ctx = mx.cpu()\n",
    "# run on gpu\n",
    "# ctx = mx.gpu()\n",
    "# run on third gpu\n",
    "# ctx = mx.gpu(2)\n",
    "executor = softmax.simple_bind(ctx=ctx, data=data_shape, grad_req='write')\n",
    "# The default ctx is CPU, data's shape is required and ```simple_bind``` will try to infer all other required \n",
    "# For MLP, the ```grad_req``` is write to, and for RNN it is different"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After generating executor, we get get data from executor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# get argument arrays\n",
    "arg_arrays = executor.arg_arrays\n",
    "# get grad arrays\n",
    "grad_arrays = executor.grad_arrays\n",
    "# get aux_states arrays. Note: currently only BatchNorm symbol has auxiliary states, which is moving_mean and moving_var\n",
    "aux_arrays = executor.aux_arrays\n",
    "# get outputs from executor\n",
    "output_arrays = executor.outputs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The sequence of arrays is in same sequence of symbol arguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('args: ', {'bn1_beta': <mxnet.ndarray.NDArray object at 0x7f4da2aa92d0>, 'fc2_weight': <mxnet.ndarray.NDArray object at 0x7f4da2aa9310>, 'fc1_weight': <mxnet.ndarray.NDArray object at 0x7f4d98d4e690>, 'softmax_label': <mxnet.ndarray.NDArray object at 0x7f4da2aa9550>, 'bn1_gamma': <mxnet.ndarray.NDArray object at 0x7f4da2aa9250>, 'fc2_bias': <mxnet.ndarray.NDArray object at 0x7f4da2aa9290>, 'data': <mxnet.ndarray.NDArray object at 0x7f4d98d4eb50>, 'fc1_bias': <mxnet.ndarray.NDArray object at 0x7f4da2a90150>})\n",
      "--------------------\n",
      "('grads: ', {'bn1_beta': <mxnet.ndarray.NDArray object at 0x7f4da2aa93d0>, 'fc2_weight': <mxnet.ndarray.NDArray object at 0x7f4da2aa9510>, 'fc1_weight': <mxnet.ndarray.NDArray object at 0x7f4da2aa9590>, 'softmax_label': None, 'bn1_gamma': <mxnet.ndarray.NDArray object at 0x7f4da2aa9390>, 'fc2_bias': <mxnet.ndarray.NDArray object at 0x7f4da2aa94d0>, 'data': None, 'fc1_bias': <mxnet.ndarray.NDArray object at 0x7f4da2aa9350>})\n",
      "--------------------\n",
      "('aux_states: ', {'bn1_moving_mean': <mxnet.ndarray.NDArray object at 0x7f4da2aa9490>, 'bn1_moving_var': <mxnet.ndarray.NDArray object at 0x7f4da2aa9450>})\n",
      "--------------------\n",
      "('outputs: ', {'softmax_output': <mxnet.ndarray.NDArray object at 0x7f4da2aa9890>})\n"
     ]
    }
   ],
   "source": [
    "args = dict(zip(softmax.list_arguments(), arg_arrays))\n",
    "grads = dict(zip(softmax.list_arguments(), grad_arrays))\n",
    "outputs = dict(zip(softmax.list_outputs(), output_arrays))\n",
    "aux_states = dict(zip(softmax.list_auxiliary_states(), aux_arrays))\n",
    "# we can print the args we have\n",
    "print(\"args: \", args)\n",
    "print(\"-\" * 20)\n",
    "print(\"grads: \", grads)\n",
    "print(\"-\" * 20)\n",
    "print(\"aux_states: \", aux_states)\n",
    "print(\"-\" * 20)\n",
    "print(\"outputs: \", outputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The next step is intilize weight. We can set weight directly by using ```mx.random``` or numpy ndarray"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# helper function\n",
    "def Init(key, arr):\n",
    "    if \"weight\" in key:\n",
    "        arr[:] = mx.random.uniform(-0.07, 0.07, arr.shape)\n",
    "        # or\n",
    "        # arr[:] = np.random.uniform(-0.07, 0.07, arr.shape)\n",
    "    elif \"gamma\" in key:\n",
    "        # for batch norm slope\n",
    "        arr[:] = 1.0\n",
    "    elif \"bias\" in key:\n",
    "        arr[:] = 0\n",
    "    elif \"beta\" in key:\n",
    "        # for batch norm bias\n",
    "        arr[:] = 0\n",
    "\n",
    "# Init args\n",
    "for key, arr in args.items():\n",
    "    Init(key, arr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then we can customize our update rule. Here for demo purpose, we make a simple update rule to show mxnet feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def SGD(key, weight, grad, lr=0.1, grad_norm=batch_size):\n",
    "    # key is key for weight, we can customize update rule\n",
    "    # weight is weight array\n",
    "    # grad is grad array\n",
    "    # lr is learning rate\n",
    "    # grad_norm is scalar to norm gradient, usually it is batch_size\n",
    "    norm = 1.0 / grad_norm\n",
    "    # here we can bias' learning rate 2 times larger than weight\n",
    "    if \"weight\" in key or \"gamma\" in key:\n",
    "        weight[:] -= lr * (grad * norm)\n",
    "    elif \"bias\" in key or \"beta\" in key:\n",
    "        weight[:] -= 2.0 * lr * (grad * norm)\n",
    "    else:\n",
    "        pass\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then we will make a data iterator. We can either use build in iterator to load from binary file or build a numpy iterator.\n",
    "\n",
    "For special case, you are free to write your own iterator in python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# We use utils function in sklearn to get MNIST dataset in pickle\n",
    "from sklearn.datasets import fetch_mldata\n",
    "from sklearn.utils import shuffle\n",
    "mnist = fetch_mldata('MNIST original', data_home=\"./data\")\n",
    "# shuffle data\n",
    "X, y = shuffle(mnist.data, mnist.target)\n",
    "# split dataset\n",
    "train_data = X[:50000, :].astype('float32')\n",
    "train_label = y[:50000]\n",
    "val_data = X[50000: 60000, :].astype('float32')\n",
    "val_label = y[50000:60000]\n",
    "# Normalize data\n",
    "train_data[:] /= 256.0\n",
    "val_data[:] /= 256.0\n",
    "# Build iterator\n",
    "train_iter = mx.io.NDArrayIter(data=train_data, label=train_label, batch_size=batch_size, shuffle=True)\n",
    "val_iter = mx.io.NDArrayIter(data=val_data, label=val_label, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Also, we need to define a helper function to calculate accuracy of current training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Accuracy(label, pred_prob):\n",
    "    pred = np.argmax(pred_prob, axis=1)\n",
    "    return np.sum(label == pred) * 1.0 / label.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then we can train the network by using executor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Finish training iteration 0\n",
      "INFO:root:Train Acc: 0.9506\n",
      "INFO:root:Val Acc: 0.9480\n",
      "INFO:root:Finish training iteration 1\n",
      "INFO:root:Train Acc: 0.9579\n",
      "INFO:root:Val Acc: 0.9521\n",
      "INFO:root:Finish training iteration 2\n",
      "INFO:root:Train Acc: 0.9639\n",
      "INFO:root:Val Acc: 0.9551\n"
     ]
    }
   ],
   "source": [
    "num_round = 3\n",
    "keys = softmax.list_arguments()\n",
    "# we use extra ndarray to save output of net\n",
    "pred_prob = mx.nd.zeros(executor.outputs[0].shape)\n",
    "for i in range(num_round):\n",
    "    train_iter.reset()\n",
    "    val_iter.reset()\n",
    "    train_acc = 0.\n",
    "    val_acc = 0.\n",
    "    nbatch = 0.\n",
    "    # train\n",
    "    for dbatch in train_iter:\n",
    "        data = dbatch.data[0]\n",
    "        label = dbatch.label[0]\n",
    "        # copy data into args\n",
    "        args[\"data\"][:] = data # or we can ```data.copyto(args[\"data\"])```\n",
    "        args[\"softmax_label\"][:] = label\n",
    "        executor.forward(is_train=True)\n",
    "        pred_prob[:] = executor.outputs[0]\n",
    "        executor.backward()\n",
    "        for key in keys:\n",
    "            SGD(key, args[key], grads[key])\n",
    "        train_acc += Accuracy(label.asnumpy(), pred_prob.asnumpy())\n",
    "        nbatch += 1.\n",
    "    logging.info(\"Finish training iteration %d\" % i)\n",
    "    train_acc /= nbatch\n",
    "    nbatch = 0.\n",
    "    # eval\n",
    "    for dbatch in val_iter:\n",
    "        data = dbatch.data[0]\n",
    "        label = dbatch.label[0]\n",
    "        args[\"data\"][:] = data\n",
    "        executor.forward(is_train=False)\n",
    "        pred_prob[:] = executor.outputs[0]\n",
    "        val_acc += Accuracy(label.asnumpy(), pred_prob.asnumpy())\n",
    "        nbatch += 1.\n",
    "    val_acc /= nbatch\n",
    "    logging.info(\"Train Acc: %.4f\" % train_acc)\n",
    "    logging.info(\"Val Acc: %.4f\" % val_acc)\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "This is simple example gives a demo on how to directly use symbolic API to build a neural net from scratch."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}