pytorch_demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Basics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x = torch.Tensor(5, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "-6.0406e-13  4.5860e-41 -6.0406e-13\n",
       " 4.5860e-41  1.3593e-43  4.4842e-44\n",
       " 1.4013e-43  1.4714e-43  1.5975e-43\n",
       " 1.4153e-43  1.3873e-43  1.6255e-43\n",
       " 1.5554e-43  1.5975e-43  1.6956e-43\n",
       "[torch.FloatTensor of size 5x3]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([5, 3])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x, y = torch.rand(5, 3), torch.rand(5, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 0.9537  0.4231  0.7722\n",
       " 0.8902  0.7525  0.2126\n",
       " 0.3621  0.1128  0.1894\n",
       " 0.7023  0.0246  0.6558\n",
       " 0.5371  0.8103  0.6110\n",
       "[torch.FloatTensor of size 5x3]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 0.1505  0.8563  0.1885\n",
       " 0.6586  0.2491  0.3338\n",
       " 0.0681  0.6530  0.1600\n",
       " 0.9413  0.9524  0.2237\n",
       " 0.1020  0.4949  0.0200\n",
       "[torch.FloatTensor of size 5x3]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 1.1042  1.2793  0.9607\n",
       " 1.5488  1.0016  0.5464\n",
       " 0.4302  0.7658  0.3493\n",
       " 1.6435  0.9770  0.8795\n",
       " 0.6391  1.3052  0.6310\n",
       "[torch.FloatTensor of size 5x3]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x + y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 1.1042  1.2793  0.9607\n",
       " 1.5488  1.0016  0.5464\n",
       " 0.4302  0.7658  0.3493\n",
       " 1.6435  0.9770  0.8795\n",
       " 0.6391  1.3052  0.6310\n",
       "[torch.FloatTensor of size 5x3]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.add(x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 1.1042  1.2793  0.9607\n",
       " 1.5488  1.0016  0.5464\n",
       " 0.4302  0.7658  0.3493\n",
       " 1.6435  0.9770  0.8795\n",
       " 0.6391  1.3052  0.6310\n",
       "[torch.FloatTensor of size 5x3]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result = torch.Tensor(5, 3)\n",
    "torch.add(x, y, out=result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 1.1042  1.2793  0.9607\n",
       " 1.5488  1.0016  0.5464\n",
       " 0.4302  0.7658  0.3493\n",
       " 1.6435  0.9770  0.8795\n",
       " 0.6391  1.3052  0.6310\n",
       "[torch.FloatTensor of size 5x3]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 1.1042  1.2793  0.9607\n",
       " 1.5488  1.0016  0.5464\n",
       " 0.4302  0.7658  0.3493\n",
       " 1.6435  0.9770  0.8795\n",
       " 0.6391  1.3052  0.6310\n",
       "[torch.FloatTensor of size 5x3]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.add_(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 1.1042  1.2793  0.9607\n",
       " 1.5488  1.0016  0.5464\n",
       " 0.4302  0.7658  0.3493\n",
       " 1.6435  0.9770  0.8795\n",
       " 0.6391  1.3052  0.6310\n",
       "[torch.FloatTensor of size 5x3]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#x.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from torch.autograd import Variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = Variable(torch.randn(1, 10))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Numpy bridge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(\n",
       "  1\n",
       "  1\n",
       "  1\n",
       "  1\n",
       "  1\n",
       " [torch.FloatTensor of size 5],\n",
       " array([ 1.,  1.,  1.,  1.,  1.], dtype=float32))"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = torch.ones(5)\n",
    "b = a.numpy()\n",
    "a, b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(\n",
       "  2\n",
       "  2\n",
       "  2\n",
       "  2\n",
       "  2\n",
       " [torch.FloatTensor of size 5],\n",
       " array([ 2.,  2.,  2.,  2.,  2.], dtype=float32))"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.add_(1)\n",
    "a, b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 2.,  2.,  2.,  2.,  2.]), \n",
       "  2\n",
       "  2\n",
       "  2\n",
       "  2\n",
       "  2\n",
       " [torch.DoubleTensor of size 5])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "a = np.ones(5)\n",
    "b = torch.from_numpy(a)\n",
    "np.add(a, 1, out=a)\n",
    "a, b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 1\n",
       " 1\n",
       " 1\n",
       " 1\n",
       " 1\n",
       "[torch.FloatTensor of size 5]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c = torch.ones(5)\n",
    "c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 11\n",
       " 11\n",
       " 11\n",
       " 11\n",
       " 11\n",
       "[torch.FloatTensor of size 5]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c.add_(10)\n",
    "c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 21\n",
       " 21\n",
       " 21\n",
       " 21\n",
       " 21\n",
       "[torch.FloatTensor of size 5]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.add(c, 10, out=c)\n",
    "c"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Autograd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.autograd import Variable"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Case"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Variable containing:\n",
       "  3\n",
       "  3\n",
       " [torch.FloatTensor of size 2], None)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = Variable(torch.ones(2)*3, requires_grad=True)\n",
    "x, x.creator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Variable containing:\n",
       "  6\n",
       " [torch.FloatTensor of size 1],\n",
       " <torch.autograd._functions.basic_ops.Add at 0x7f66ffcc3908>)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = x[0] + x[1]\n",
    "y, y.creator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(\n",
       "  3\n",
       "  3\n",
       " [torch.FloatTensor of size 2], Variable containing:\n",
       "  1\n",
       "  1\n",
       " [torch.FloatTensor of size 2])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.data, x.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(\n",
       "  6\n",
       " [torch.FloatTensor of size 1], None)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.data, y.grad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Case"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Variable containing:\n",
       "  3\n",
       "  3\n",
       " [torch.FloatTensor of size 2], None)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = Variable(torch.ones(2)*3, requires_grad=True)\n",
    "x, x.creator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Variable containing:\n",
       "  15\n",
       " [torch.FloatTensor of size 1],\n",
       " <torch.autograd._functions.basic_ops.Add at 0x7f66ffcc39e8>)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = x[0]*2 + x[1]*3\n",
    "y, y.creator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(\n",
       "  3\n",
       "  3\n",
       " [torch.FloatTensor of size 2], Variable containing:\n",
       "  2\n",
       "  3\n",
       " [torch.FloatTensor of size 2])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.data, x.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(\n",
       "  15\n",
       " [torch.FloatTensor of size 1], None)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.data, y.grad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Case"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Variable containing:\n",
       "  3\n",
       "  3\n",
       " [torch.FloatTensor of size 2], None)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = Variable(torch.ones(2)*3, requires_grad=True)\n",
    "x, x.creator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Variable containing:\n",
       "  36\n",
       " [torch.FloatTensor of size 1],\n",
       " <torch.autograd._functions.basic_ops.Add at 0x7f66ffcc3f28>)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = x[0]**2 + x[1]**3\n",
    "y, y.creator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(\n",
       "  3\n",
       "  3\n",
       " [torch.FloatTensor of size 2], Variable containing:\n",
       "   6\n",
       "  27\n",
       " [torch.FloatTensor of size 2])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.data, x.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(\n",
       "  36\n",
       " [torch.FloatTensor of size 1], None)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.data, y.grad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Case - multiple outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Variable containing:\n",
       "  1\n",
       "  1\n",
       " [torch.FloatTensor of size 2], None)"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = Variable(torch.FloatTensor([1, 1]), requires_grad=True)\n",
    "x, x.creator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Variable containing:\n",
       "  5\n",
       "  5\n",
       " [torch.FloatTensor of size 2],\n",
       " <torch.autograd._functions.basic_ops.MulConstant at 0x7f66fc0c63c8>)"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = 5 * x **2\n",
    "y, y.creator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```\n",
    "y.backward()\n",
    "RuntimeError: backward should be called only on a scalar (i.e. 1-element tensor) or with gradient w.r.t. the variable\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "gradients = torch.FloatTensor([1.0, 2.0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "y.backward(gradients)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       " 10\n",
       " 20\n",
       "[torch.FloatTensor of size 2]"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "gradients = torch.FloatTensor([100.0, 200.0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "y.backward(gradients)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       " 1000\n",
       " 2000\n",
       "[torch.FloatTensor of size 2]"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.grad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Not sure why `gradients` is needed."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Case"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Variable containing:\n",
       "  1\n",
       "  1\n",
       " [torch.FloatTensor of size 2], None)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = Variable(torch.ones(2), requires_grad=True)\n",
    "x, x.creator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Variable containing:\n",
       "  2\n",
       "  2\n",
       " [torch.FloatTensor of size 2],\n",
       " <torch.autograd._functions.basic_ops.MulConstant at 0x7f66ffcc3c88>)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = 2 * x\n",
    "y, y.creator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "z = 3 * y**2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "out = z.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "out.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(\n",
       "  1\n",
       "  1\n",
       " [torch.FloatTensor of size 2], Variable containing:\n",
       "  12\n",
       "  12\n",
       " [torch.FloatTensor of size 2])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.data, x.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(\n",
       "  2\n",
       "  2\n",
       " [torch.FloatTensor of size 2], None)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.data, y.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(\n",
       "  12\n",
       "  12\n",
       " [torch.FloatTensor of size 2], None)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "z.data, z.grad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\n",
    "o = (z_0 + z_1) / 2 = (3y_0^2 + 3y_1^2) /2 = (3(2x_0)^2 + 3(2x_1)^2) / 2 = 6(x_0^2 + x_1^2)\n",
    "$$\n",
    "\n",
    "$$\n",
    "\\left.\n",
    "\\frac{\\partial o}{\\partial x_0}\n",
    "\\right|_{x_0=1} = \n",
    "\\left.\\frac{\\partial 6(x_0^2 + x_1^2)}{\\partial x_0}\\right|_{x_0=1}=\\left.12x_0\\right|_{x_0=1} = 12\n",
    "$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Crazy case"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x = Variable(torch.randn(3), requires_grad=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       "-0.1654\n",
       " 0.4354\n",
       "-0.0481\n",
       "[torch.FloatTensor of size 3]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = x*2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "while y.data.norm() < 1000:\n",
    "    y = y*2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       " -677.5988\n",
       " 1783.4092\n",
       " -196.9062\n",
       "[torch.FloatTensor of size 3]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "gradients = torch.FloatTensor([0.1, 1.0, 0.0001])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y.backward(gradients)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       "  409.6000\n",
       " 4096.0000\n",
       "    0.4096\n",
       "[torch.FloatTensor of size 3]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.grad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Neural networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### References\n",
    "- http://pytorch.org/tutorials/index.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  },
  "toc": {
   "nav_menu": {
    "height": "12px",
    "width": "252px"
   },
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}