Day #8 Beginner_tutorial_on_Pytorch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "view-in-github"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/skyprince999/100-Days-Of-ML/blob/master/Day_8_Beginner_tutorial_on_Pytorch.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "LZYiKI_CTEZk"
   },
   "source": [
    "Date: 30th Jan 2020\n",
    " \n",
    "This is the Pytorch tutorial by Jeremy Howard. Original Source: \n",
    "\n",
    "\n",
    "https://pytorch.org/tutorials/beginner/nn_tutorial.html\n",
    "\n",
    "-------------------------\n",
    "\n",
    "Loading directories using pathlib. Pathlib is now part of the Python 3 standard libraries \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Mm33a75-R3GF"
   },
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import requests\n",
    "\n",
    "DATA_PATH = Path(\"data\")\n",
    "PATH = DATA_PATH / \"mnist\"\n",
    "\n",
    "PATH.mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "URL = \"http://deeplearning.net/data/mnist/\"\n",
    "FILENAME = \"mnist.pkl.gz\"\n",
    "\n",
    "# The file is downloaded as a zipped pickle \n",
    "if not (PATH / FILENAME).exists():\n",
    "        content = requests.get(URL + FILENAME).content\n",
    "        (PATH / FILENAME).open(\"wb\").write(content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "IFyhU1_MYyT8"
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "import gzip\n",
    "\n",
    "with gzip.open((PATH / FILENAME).as_posix(), \"rb\") as f:\n",
    "        ((x_train, y_train), (x_valid, y_valid), _) = pickle.load(f, encoding=\"latin-1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 282
    },
    "colab_type": "code",
    "id": "w7OPgO73ZMnW",
    "outputId": "350c700a-8731-4314-af07-84ef99f28bec"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50000, 784)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAOZ0lEQVR4nO3dbYxc5XnG8euKbezamMQbB9chLjjg\nFAg0Jl0ZEBZQobgOqgSoCsSKIkJpnSY4Ca0rQWlV3IpWbpUQUUqRTHExFS+BBIQ/0CTUQpCowWWh\nBgwEDMY0NmaNWYENIX5Z3/2w42iBnWeXmTMv3vv/k1Yzc+45c24NXD5nznNmHkeEAIx/H+p0AwDa\ng7ADSRB2IAnCDiRB2IEkJrZzY4d5ckzRtHZuEkjlV3pbe2OPR6o1FXbbiyVdJ2mCpH+LiJWl50/R\nNJ3qc5rZJICC9bGubq3hw3jbEyTdIOnzkk6UtMT2iY2+HoDWauYz+wJJL0TE5ojYK+lOSedV0xaA\nqjUT9qMk/WLY4621Ze9ie6ntPtt9+7Snic0BaEbLz8ZHxKqI6I2I3kma3OrNAaijmbBvkzRn2ONP\n1JYB6ELNhP1RSfNsz7V9mKQvSlpbTVsAqtbw0FtE7Le9TNKPNDT0tjoinq6sMwCVamqcPSLul3R/\nRb0AaCEulwWSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB\n2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiCJpmZx\nRffzxPJ/4gkfm9nS7T/3F8fUrQ1OPVBc9+hjdxTrU7/uYv3Vaw+rW3u893vFdXcOvl2sn3r38mL9\nuD9/pFjvhKbCbnuLpN2SBiXtj4jeKpoCUL0q9uy/FxE7K3gdAC3EZ3YgiWbDHpJ+bPsx20tHeoLt\npbb7bPft054mNwegUc0exi+MiG22j5T0gO2fR8TDw58QEaskrZKkI9wTTW4PQIOa2rNHxLba7Q5J\n90paUEVTAKrXcNhtT7M9/eB9SYskbayqMQDVauYwfpake20ffJ3bI+KHlXQ1zkw4YV6xHpMnFeuv\nnPWRYv2d0+qPCfd8uDxe/JPPlMebO+k/fzm9WP/Hf1lcrK8/+fa6tZf2vVNcd2X/54r1j//k0PtE\n2nDYI2KzpM9U2AuAFmLoDUiCsANJEHYgCcIOJEHYgST4imsFBs/+bLF+7S03FOufmlT/q5jj2b4Y\nLNb/5vqvFOsT3y4Pf51+97K6tenb9hfXnbyzPDQ3tW99sd6N2LMDSRB2IAnCDiRB2IEkCDuQBGEH\nkiDsQBKMs1dg8nOvFOuP/WpOsf6pSf1VtlOp5dtPK9Y3v1X+Kepbjv1+3dqbB8rj5LP++b+L9VY6\n9L7AOjr27EAShB1IgrADSRB2IAnCDiRB2IEkCDuQhCPaN6J4hHviVJ/Ttu11i4FLTi/Wdy0u/9zz\nhCcPL9af+Pr1H7ing67Z+TvF+qNnlcfRB994s1iP0+v/APGWbxZX1dwlT5SfgPdZH+u0KwZGnMua\nPTuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJME4exeYMPOjxfrg6wPF+ku31x8rf/rM1cV1F/zDN4r1\nI2/o3HfK8cE1Nc5ue7XtHbY3DlvWY/sB25tqtzOqbBhA9cZyGH+LpPfOen+lpHURMU/SutpjAF1s\n1LBHxMOS3nsceZ6kNbX7aySdX3FfACrW6G/QzYqI7bX7r0qaVe+JtpdKWipJUzS1wc0BaFbTZ+Nj\n6Axf3bN8EbEqInojoneSJje7OQANajTs/bZnS1Ltdkd1LQFohUbDvlbSxbX7F0u6r5p2ALTKqJ/Z\nbd8h6WxJM21vlXS1pJWS7rJ9qaSXJV3YyibHu8Gdrze1/r5djc/v/ukvPVOsv3bjhPILHCjPsY7u\nMWrYI2JJnRJXxwCHEC6XBZIg7EAShB1IgrADSRB2IAmmbB4HTrji+bq1S04uD5r8+9HrivWzvnBZ\nsT79e48U6+ge7NmBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnG2ceB0rTJr3/thOK6/7f2nWL9ymtu\nLdb/8sILivX43w/Xrc35+58V11Ubf+Y8A/bsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEUzYnN/BH\npxfrt1397WJ97sQpDW/707cuK9bn3bS9WN+/eUvD2x6vmpqyGcD4QNiBJAg7kARhB5Ig7EAShB1I\ngrADSTDOjqI4Y36xfsTKrcX6HZ/8UcPbPv7BPy7Wf/tv63+PX5IGN21ueNuHqqbG2W2vtr3D9sZh\ny1bY3mZ7Q+3v3CobBlC9sRzG3yJp8QjLvxsR82t/91fbFoCqjRr2iHhY0kAbegHQQs2coFtm+8na\nYf6Mek+yvdR2n+2+fdrTxOYANKPRsN8o6VhJ8yVtl/Sdek+MiFUR0RsRvZM0ucHNAWhWQ2GPiP6I\nGIyIA5JukrSg2rYAVK2hsNuePezhBZI21nsugO4w6ji77TsknS1ppqR+SVfXHs+XFJK2SPpqRJS/\nfCzG2cejCbOOLNZfuei4urX1V1xXXPdDo+yLvvTSomL9zYWvF+vjUWmcfdRJIiJiyQiLb266KwBt\nxeWyQBKEHUiCsANJEHYgCcIOJMFXXNExd20tT9k81YcV67+MvcX6H3zj8vqvfe/64rqHKn5KGgBh\nB7Ig7EAShB1IgrADSRB2IAnCDiQx6rfekNuBheWfkn7xC+Upm0+av6VubbRx9NFcP3BKsT71vr6m\nXn+8Yc8OJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kwzj7OufekYv35b5bHum86Y02xfuaU8nfKm7En\n9hXrjwzMLb/AgVF/3TwV9uxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kATj7IeAiXOPLtZfvOTjdWsr\nLrqzuO4fHr6zoZ6qcFV/b7H+0HWnFesz1pR/dx7vNuqe3fYc2w/afsb207a/VVveY/sB25tqtzNa\n3y6ARo3lMH6/pOURcaKk0yRdZvtESVdKWhcR8yStqz0G0KVGDXtEbI+Ix2v3d0t6VtJRks6TdPBa\nyjWSzm9VkwCa94E+s9s+RtIpktZLmhURBy8+flXSrDrrLJW0VJKmaGqjfQJo0pjPxts+XNIPJF0e\nEbuG12JodsgRZ4iMiFUR0RsRvZM0ualmATRuTGG3PUlDQb8tIu6pLe63PbtWny1pR2taBFCFUQ/j\nbVvSzZKejYhrh5XWSrpY0sra7X0t6XAcmHjMbxXrb/7u7GL9or/7YbH+px+5p1hvpeXby8NjP/vX\n+sNrPbf8T3HdGQcYWqvSWD6znyHpy5Kesr2htuwqDYX8LtuXSnpZ0oWtaRFAFUYNe0T8VNKIk7tL\nOqfadgC0CpfLAkkQdiAJwg4kQdiBJAg7kARfcR2jibN/s25tYPW04rpfm/tQsb5ken9DPVVh2baF\nxfrjN5anbJ75/Y3Fes9uxsq7BXt2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUgizTj73t8v/2zx3j8b\nKNavOu7+urVFv/F2Qz1VpX/wnbq1M9cuL657/F//vFjveaM8Tn6gWEU3Yc8OJEHYgSQIO5AEYQeS\nIOxAEoQdSIKwA0mkGWffcn7537XnT767Zdu+4Y1ji/XrHlpUrHuw3o/7Djn+mpfq1ub1ry+uO1is\nYjxhzw4kQdiBJAg7kARhB5Ig7EAShB1IgrADSTgiyk+w50i6VdIsSSFpVURcZ3uFpD+R9FrtqVdF\nRP0vfUs6wj1xqpn4FWiV9bFOu2JgxAszxnJRzX5JyyPicdvTJT1m+4Fa7bsR8e2qGgXQOmOZn327\npO21+7ttPyvpqFY3BqBaH+gzu+1jJJ0i6eA1mMtsP2l7te0ZddZZarvPdt8+7WmqWQCNG3PYbR8u\n6QeSLo+IXZJulHSspPka2vN/Z6T1ImJVRPRGRO8kTa6gZQCNGFPYbU/SUNBvi4h7JCki+iNiMCIO\nSLpJ0oLWtQmgWaOG3bYl3Szp2Yi4dtjy2cOedoGk8nSeADpqLGfjz5D0ZUlP2d5QW3aVpCW252to\nOG6LpK+2pEMAlRjL2fifShpp3K44pg6gu3AFHZAEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7\nkARhB5Ig7EAShB1IgrADSRB2IIlRf0q60o3Zr0l6ediimZJ2tq2BD6Zbe+vWviR6a1SVvR0dER8b\nqdDWsL9v43ZfRPR2rIGCbu2tW/uS6K1R7eqNw3ggCcIOJNHpsK/q8PZLurW3bu1LordGtaW3jn5m\nB9A+nd6zA2gTwg4k0ZGw215s+znbL9i+shM91GN7i+2nbG+w3dfhXlbb3mF747BlPbYfsL2pdjvi\nHHsd6m2F7W21926D7XM71Nsc2w/afsb207a/VVve0feu0Fdb3re2f2a3PUHS85I+J2mrpEclLYmI\nZ9raSB22t0jqjYiOX4Bh+0xJb0m6NSJOqi37J0kDEbGy9g/ljIi4okt6WyHprU5P412brWj28GnG\nJZ0v6Svq4HtX6OtCteF968SefYGkFyJic0TslXSnpPM60EfXi4iHJQ28Z/F5ktbU7q/R0P8sbVen\nt64QEdsj4vHa/d2SDk4z3tH3rtBXW3Qi7EdJ+sWwx1vVXfO9h6Qf237M9tJONzOCWRGxvXb/VUmz\nOtnMCEadxrud3jPNeNe8d41Mf94sTtC938KI+Kykz0u6rHa42pVi6DNYN42djmka73YZYZrxX+vk\ne9fo9OfN6kTYt0maM+zxJ2rLukJEbKvd7pB0r7pvKur+gzPo1m53dLifX+umabxHmmZcXfDedXL6\n806E/VFJ82zPtX2YpC9KWtuBPt7H9rTaiRPZniZpkbpvKuq1ki6u3b9Y0n0d7OVdumUa73rTjKvD\n713Hpz+PiLb/STpXQ2fkX5T0V53ooU5fn5T0RO3v6U73JukODR3W7dPQuY1LJX1U0jpJmyT9l6Se\nLurtPyQ9JelJDQVrdod6W6ihQ/QnJW2o/Z3b6feu0Fdb3jculwWS4AQdkARhB5Ig7EAShB1IgrAD\nSRB2IAnCDiTx/65XcTNOWsh5AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Each image is a 28x28 numpy array which has been flattened. This will be reshaped to a gray scale image\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import numpy as np\n",
    "\n",
    "pyplot.imshow(x_train[0].reshape((28, 28))) #, cmap=\"gray\")\n",
    "print(x_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 187
    },
    "colab_type": "code",
    "id": "JX_WIkbAZdrG",
    "outputId": "84e752a0-8933-4d9a-e44e-54121c441ee3"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
       "         [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "         [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "         ...,\n",
       "         [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "         [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "         [0., 0., 0.,  ..., 0., 0., 0.]]),\n",
       " torch.Size([50000, 784]),\n",
       " tensor(0),\n",
       " tensor(9))"
      ]
     },
     "execution_count": 4,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Since pytorch uses torch.tensors we will convert these numpy arrays \n",
    "\n",
    "import torch\n",
    "\n",
    "x_train, y_train, x_valid, y_valid = map(\n",
    "    torch.tensor, (x_train, y_train, x_valid, y_valid)\n",
    ")\n",
    "n, c = x_train.shape\n",
    "x_train, x_train.shape, y_train.min(), y_train.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 170
    },
    "colab_type": "code",
    "id": "vJBaQCw8Z5B5",
    "outputId": "046a761c-e75b-4fb1-fc44-67a3428cb52e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        ...,\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]]) tensor([5, 0, 4,  ..., 8, 4, 8])\n",
      "torch.Size([50000, 784])\n",
      "tensor(0) tensor(9)\n"
     ]
    }
   ],
   "source": [
    "print(x_train, y_train)\n",
    "print(x_train.shape)\n",
    "print(y_train.min(), y_train.max())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_B4vZJDDZ78h"
   },
   "outputs": [],
   "source": [
    "# Initializing via Xavier initalization 1/sqrt(n)\n",
    "\n",
    "import math\n",
    "\n",
    "weights = torch.randn(784, 10) / math.sqrt(784)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 153
    },
    "colab_type": "code",
    "id": "1p_O_XPzcMSS",
    "outputId": "212d7237-4bb1-477f-f285-ac42f578e325"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0332, -0.0217,  0.0023,  ..., -0.0916,  0.0100,  0.0480],\n",
       "        [ 0.0305, -0.0128, -0.0231,  ..., -0.0303,  0.0481, -0.0223],\n",
       "        [-0.0390,  0.0131, -0.0079,  ..., -0.0130,  0.0402, -0.0251],\n",
       "        ...,\n",
       "        [-0.0017,  0.0172,  0.0227,  ..., -0.0235,  0.0238, -0.0360],\n",
       "        [ 0.0348,  0.0061, -0.0230,  ...,  0.0713, -0.0379, -0.0309],\n",
       "        [-0.0128,  0.0170,  0.0099,  ..., -0.0373, -0.0639, -0.0301]],\n",
       "       requires_grad=True)"
      ]
     },
     "execution_count": 7,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weights.requires_grad_()  # This cause PyTorch to record all the operations done on the tensor & claculate the gradient during back-propogation\n",
    "                          # The _ indicates that operation is done in-place \n",
    "                          # This is done after initalization so that the initialization step is not included during back-propogation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "y6pV-Y9KcqE9"
   },
   "outputs": [],
   "source": [
    "bias = torch.zeros(10, requires_grad=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Au4I2Sa9dAm1"
   },
   "outputs": [],
   "source": [
    "def log_softmax(x):\n",
    "    return x - x.exp().sum(-1).log().unsqueeze(-1)\n",
    "\n",
    "def model(xb):\n",
    "    return log_softmax(xb @ weights + bias)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "EndEPt_3eI7h",
    "outputId": "913fd465-6bfb-4bc7-f6a8-0bda0f51bf14"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([-2.6632, -2.5091, -2.1814, -2.9018, -2.5293, -2.1376, -2.3095, -1.9985,\n",
      "        -2.1772, -2.0005], grad_fn=<SelectBackward>) torch.Size([64, 10])\n"
     ]
    }
   ],
   "source": [
    "bs = 64  # batch size\n",
    "\n",
    "xb = x_train[0:bs]  # a mini-batch from x\n",
    "preds = model(xb)  # predictions\n",
    "#preds[0], preds.shape\n",
    "print(preds[0], preds.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "p6kdW7QjfBeF"
   },
   "source": [
    "The preds tensor contains the tensor values & the gradient function. This will then beused to calculate the back propagation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8KM8jsygeqGb"
   },
   "outputs": [],
   "source": [
    "# We define the loss function\n",
    "def nll(input, target):\n",
    "    return -input[range(target.shape[0]), target].mean()\n",
    "\n",
    "loss_func = nll"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "llMJroIyfmzj",
    "outputId": "20de3c72-b0a9-4107-caf3-64c3813d091a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(2.3307, grad_fn=<NegBackward>)\n"
     ]
    }
   ],
   "source": [
    "yb = y_train[0:bs]\n",
    "print(loss_func(preds, yb))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3dQWPLvZgDw2"
   },
   "outputs": [],
   "source": [
    "def accuracy(out, yb):\n",
    "    preds = torch.argmax(out, dim=1)\n",
    "    return (preds == yb).float().mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "ITfxxc6egSMy",
    "outputId": "b96b14c2-a519-46b7-fa10-50f5cdf9683b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1406)\n"
     ]
    }
   ],
   "source": [
    "print(accuracy(preds, yb))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bmaXnXyBCch5"
   },
   "outputs": [],
   "source": [
    "from IPython.core.debugger import set_trace\n",
    "\n",
    "lr = 0.5  # learning rate\n",
    "epochs = 2  # how many epochs to train for\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    for i in range((n - 1) // bs + 1):\n",
    "        #set_trace()\n",
    "        start_i = i * bs\n",
    "        end_i = start_i + bs\n",
    "        xb = x_train[start_i:end_i]\n",
    "        yb = y_train[start_i:end_i]\n",
    "        pred = model(xb)\n",
    "        loss = loss_func(pred, yb)\n",
    "\n",
    "        loss.backward()\n",
    "        with torch.no_grad():\n",
    "            weights -= weights.grad * lr\n",
    "            bias -= bias.grad * lr\n",
    "            weights.grad.zero_()\n",
    "            bias.grad.zero_()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "3XOxLzOfD8-H",
    "outputId": "eb4bfccc-540b-4324-a983-e641e35a13c8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.0801, grad_fn=<NegBackward>) tensor(1.)\n"
     ]
    }
   ],
   "source": [
    "print(loss_func(model(xb), yb), accuracy(model(xb), yb))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rWQt_j_wE3Iq"
   },
   "source": [
    "** Now by using Pytorch nn class, we redefine the loss function, using the inbuilt F.cross_entropy**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "G9_C7wgMEl9Q"
   },
   "outputs": [],
   "source": [
    "import torch.nn.functional as F\n",
    "\n",
    "loss_func = F.cross_entropy # The activation function is replaced by inbuilt torch.nn.functional.cross_entropy\n",
    "\n",
    "def model(xb):\n",
    "    return xb @ weights + bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "Sgl8ZUuNFBLf",
    "outputId": "c3217fbc-6f22-4ed3-aee1-69796f608534"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.0801, grad_fn=<NllLossBackward>) tensor(1.)\n"
     ]
    }
   ],
   "source": [
    "print(loss_func(model(xb), yb), accuracy(model(xb), yb))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "vTVr8W8pFgxU"
   },
   "outputs": [],
   "source": [
    "# Now we will recreate the training loop in pytorch \n",
    "\n",
    "from torch import nn\n",
    "\n",
    "class Mnist_Logistic(nn.Module): # we are sub-classing nn.Module which is a base class for ll neural network modules \n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.weights = nn.Parameter(torch.randn(784, 10) / math.sqrt(784))\n",
    "        self.bias = nn.Parameter(torch.zeros(10))\n",
    "\n",
    "    def forward(self, xb):\n",
    "        return xb @ self.weights + self.bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Q_UFCR0YF3ms"
   },
   "outputs": [],
   "source": [
    "model = Mnist_Logistic() # First instantiating the object. This is different from the previous implementation where we had created a function'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "xytWX0cqGRa6",
    "outputId": "45122bfa-bc3a-49d0-f0f2-8e2e137e41b6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(2.4794, grad_fn=<NllLossBackward>)\n"
     ]
    }
   ],
   "source": [
    "print(loss_func(model(xb), yb))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tfbee9hhGYtq"
   },
   "outputs": [],
   "source": [
    "def fit():\n",
    "    for epoch in range(epochs):\n",
    "        for i in range((n - 1) // bs + 1):\n",
    "            start_i = i * bs\n",
    "            end_i = start_i + bs\n",
    "            xb = x_train[start_i:end_i]\n",
    "            yb = y_train[start_i:end_i]\n",
    "            pred = model(xb)\n",
    "            loss = loss_func(pred, yb)\n",
    "\n",
    "            loss.backward()\n",
    "            with torch.no_grad():\n",
    "                for p in model.parameters():\n",
    "                    p -= p.grad * lr\n",
    "                model.zero_grad()\n",
    "\n",
    "fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "7Yk-v4boIOgT",
    "outputId": "83c0664f-e756-4e10-d934-ca0d785a24f3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.0822, grad_fn=<NllLossBackward>)\n"
     ]
    }
   ],
   "source": [
    "print(loss_func(model(xb), yb))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0Cyp4AiQIgH_"
   },
   "outputs": [],
   "source": [
    "class Mnist_Logistic(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.lin = nn.Linear(784, 10)\n",
    "\n",
    "    def forward(self, xb):\n",
    "        return self.lin(xb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "jSainxOnIsKF",
    "outputId": "64e447a0-760b-4efb-993a-428323117e8d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(2.2590, grad_fn=<NllLossBackward>)\n"
     ]
    }
   ],
   "source": [
    "model = Mnist_Logistic()\n",
    "print(loss_func(model(xb), yb))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "cTy7w1wuIvLp",
    "outputId": "d21064d7-4d94-4d24-98b3-a20275cbbdb3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.0817, grad_fn=<NllLossBackward>)\n"
     ]
    }
   ],
   "source": [
    "fit()\n",
    "\n",
    "print(loss_func(model(xb), yb))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "DCfo-D-oI0Cd",
    "outputId": "5b291d1f-cfdf-4426-926e-7e7a62590efc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(2.1780, grad_fn=<NllLossBackward>)\n"
     ]
    }
   ],
   "source": [
    "from torch import optim\n",
    "\n",
    "def get_model():\n",
    "    model = Mnist_Logistic()\n",
    "    return model, optim.SGD(model.parameters(), lr=lr)\n",
    "\n",
    "model, opt = get_model()\n",
    "print(loss_func(model(xb), yb))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "6FlbU73RI6_m",
    "outputId": "89a89020-7df2-4aa9-eba7-2b916b67bed2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.0814, grad_fn=<NllLossBackward>)\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(epochs):\n",
    "    for i in range((n - 1) // bs + 1):\n",
    "        start_i = i * bs\n",
    "        end_i = start_i + bs\n",
    "        xb = x_train[start_i:end_i]\n",
    "        yb = y_train[start_i:end_i]\n",
    "        pred = model(xb)\n",
    "        loss = loss_func(pred, yb)\n",
    "\n",
    "        loss.backward()\n",
    "        opt.step()\n",
    "        opt.zero_grad()\n",
    "\n",
    "print(loss_func(model(xb), yb))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "6UDsBrufKcwl"
   },
   "source": [
    "Converting  train  to a tensordataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UQ-9cEAqJWIm"
   },
   "outputs": [],
   "source": [
    "from torch.utils.data import TensorDataset\n",
    "\n",
    "train_ds = TensorDataset(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "E0znu_IXKk7w",
    "outputId": "4ac77ce8-256a-47f2-a55b-296eed52ee4d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.0823, grad_fn=<NllLossBackward>)\n"
     ]
    }
   ],
   "source": [
    "model, opt = get_model()\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    for i in range((n - 1) // bs + 1):\n",
    "        xb, yb = train_ds[i * bs: i * bs + bs]\n",
    "        pred = model(xb)\n",
    "        loss = loss_func(pred, yb)\n",
    "\n",
    "        loss.backward()\n",
    "        opt.step()\n",
    "        opt.zero_grad()\n",
    "\n",
    "print(loss_func(model(xb), yb))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "gTnT_IfBK9ZO"
   },
   "source": [
    "Now we refactor the code to a Dataset loader. This class created mini-batches on its own. \n",
    "\n",
    "```train_ds[i * bs: i * bs + bs]``` << - This is not required "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5g49O-b-K1br"
   },
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "\n",
    "train_ds = TensorDataset(x_train, y_train) # Converting to tensordataset\n",
    "train_dl = DataLoader(train_ds, batch_size=bs) # and then initializing a dataloader class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "8VHD3NH7LU3l",
    "outputId": "276250c5-5348-485b-d36b-a7d801de3368"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.0797, grad_fn=<NllLossBackward>)\n"
     ]
    }
   ],
   "source": [
    "model, opt = get_model()\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    for xb, yb in train_dl:\n",
    "        pred = model(xb)\n",
    "        loss = loss_func(pred, yb)\n",
    "\n",
    "        loss.backward()\n",
    "        opt.step()\n",
    "        opt.zero_grad()\n",
    "\n",
    "print(loss_func(model(xb), yb))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "pAgDcgbCLfKK"
   },
   "outputs": [],
   "source": [
    "train_ds = TensorDataset(x_train, y_train)\n",
    "train_dl = DataLoader(train_ds, batch_size=bs, shuffle=True)\n",
    "\n",
    "valid_ds = TensorDataset(x_valid, y_valid)\n",
    "valid_dl = DataLoader(valid_ds, batch_size=bs * 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "MJJqChxZN4A0",
    "outputId": "2d549a1a-4912-4bb3-99c3-51b54d38636d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 tensor(0.3383)\n",
      "1 tensor(0.2905)\n"
     ]
    }
   ],
   "source": [
    "model, opt = get_model()\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    model.train()\n",
    "    for xb, yb in train_dl:\n",
    "        pred = model(xb)\n",
    "        loss = loss_func(pred, yb)\n",
    "\n",
    "        loss.backward()\n",
    "        opt.step()\n",
    "        opt.zero_grad()\n",
    "\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        valid_loss = sum(loss_func(model(xb), yb) for xb, yb in valid_dl)\n",
    "\n",
    "    print(epoch, valid_loss / len(valid_dl))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PWl27mN1OGvo"
   },
   "outputs": [],
   "source": [
    "def loss_batch(model, loss_func, xb, yb, opt=None):\n",
    "    loss = loss_func(model(xb), yb)\n",
    "\n",
    "    if opt is not None:\n",
    "        loss.backward()\n",
    "        opt.step()\n",
    "        opt.zero_grad()\n",
    "\n",
    "    return loss.item(), len(xb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "XpSIJYM6OYhn"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "def fit(epochs, model, loss_func, opt, train_dl, valid_dl):\n",
    "    for epoch in range(epochs):\n",
    "        model.train()\n",
    "        for xb, yb in train_dl:\n",
    "            loss_batch(model, loss_func, xb, yb, opt)\n",
    "\n",
    "        model.eval()\n",
    "        with torch.no_grad():\n",
    "            losses, nums = zip(\n",
    "                *[loss_batch(model, loss_func, xb, yb) for xb, yb in valid_dl]\n",
    "            )\n",
    "        val_loss = np.sum(np.multiply(losses, nums)) / np.sum(nums)\n",
    "\n",
    "        print(epoch, val_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "lf5CnXFFO4C5"
   },
   "outputs": [],
   "source": [
    "def get_data(train_ds, valid_ds, bs):\n",
    "    return (\n",
    "        DataLoader(train_ds, batch_size=bs, shuffle=True),\n",
    "        DataLoader(valid_ds, batch_size=bs * 2),\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Mpg5kokdPFFe"
   },
   "source": [
    "**After defining the above functions. We get the *famous* 3 LINES OF CODE**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "prAE-Lx_O8ki",
    "outputId": "8753907d-4f33-41d6-971f-c1587b912cad"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 0.304442338180542\n",
      "1 0.29266126551628113\n"
     ]
    }
   ],
   "source": [
    "train_dl, valid_dl = get_data(train_ds, valid_ds, bs)\n",
    "model, opt = get_model()\n",
    "fit(epochs, model, loss_func, opt, train_dl, valid_dl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "6QD_IcMUPOf0"
   },
   "outputs": [],
   "source": [
    "class Mnist_CNN(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.conv1 = nn.Conv2d(1, 16, kernel_size=3, stride=2, padding=1)\n",
    "        self.conv2 = nn.Conv2d(16, 16, kernel_size=3, stride=2, padding=1)\n",
    "        self.conv3 = nn.Conv2d(16, 10, kernel_size=3, stride=2, padding=1)\n",
    "\n",
    "    def forward(self, xb):\n",
    "        xb = xb.view(-1, 1, 28, 28)\n",
    "        xb = F.relu(self.conv1(xb))\n",
    "        xb = F.relu(self.conv2(xb))\n",
    "        xb = F.relu(self.conv3(xb))\n",
    "        xb = F.avg_pool2d(xb, 4)\n",
    "        return xb.view(-1, xb.size(1))\n",
    "\n",
    "lr = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "y7HwCIqTP5VF",
    "outputId": "ea60dcc1-7072-4710-87f6-ce0ecb667474"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 0.28648542103767394\n",
      "1 0.2212788229227066\n"
     ]
    }
   ],
   "source": [
    "model = Mnist_CNN()\n",
    "opt = optim.SGD(model.parameters(), lr=lr, momentum=0.9)\n",
    "\n",
    "fit(epochs, model, loss_func, opt, train_dl, valid_dl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "mkGn3UuUQN1P"
   },
   "outputs": [],
   "source": [
    "class Lambda(nn.Module):\n",
    "    def __init__(self, func):\n",
    "        super().__init__()\n",
    "        self.func = func\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.func(x)\n",
    "\n",
    "\n",
    "def preprocess(x):\n",
    "    return x.view(-1, 1, 28, 28)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "iXclSUwYQndu",
    "outputId": "099fa372-f96b-4ded-aa2d-365eb3ba423f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 0.4236139154434204\n",
      "1 0.28290433728694914\n"
     ]
    }
   ],
   "source": [
    "model = nn.Sequential(\n",
    "    Lambda(preprocess),\n",
    "    nn.Conv2d(1, 16, kernel_size=3, stride=2, padding=1),\n",
    "    nn.ReLU(),\n",
    "    nn.Conv2d(16, 16, kernel_size=3, stride=2, padding=1),\n",
    "    nn.ReLU(),\n",
    "    nn.Conv2d(16, 10, kernel_size=3, stride=2, padding=1),\n",
    "    nn.ReLU(),\n",
    "    nn.AvgPool2d(4),\n",
    "    Lambda(lambda x: x.view(x.size(0), -1)),\n",
    ")\n",
    "\n",
    "opt = optim.SGD(model.parameters(), lr=lr, momentum=0.9)\n",
    "\n",
    "fit(epochs, model, loss_func, opt, train_dl, valid_dl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Y29tLf2WQ8hx"
   },
   "outputs": [],
   "source": [
    "def preprocess(x, y):\n",
    "    return x.view(-1, 1, 28, 28), y\n",
    "\n",
    "\n",
    "class WrappedDataLoader:\n",
    "    def __init__(self, dl, func):\n",
    "        self.dl = dl\n",
    "        self.func = func\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.dl)\n",
    "\n",
    "    def __iter__(self):\n",
    "        batches = iter(self.dl)\n",
    "        for b in batches:\n",
    "            yield (self.func(*b))\n",
    "\n",
    "train_dl, valid_dl = get_data(train_ds, valid_ds, bs)\n",
    "train_dl = WrappedDataLoader(train_dl, preprocess)    # The Wrapped dataloader class , pre-processes the image to a 28x28 vector \n",
    "valid_dl = WrappedDataLoader(valid_dl, preprocess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "XxGpQ5LRRdyJ"
   },
   "outputs": [],
   "source": [
    "model = nn.Sequential(\n",
    "    nn.Conv2d(1, 16, kernel_size=3, stride=2, padding=1),\n",
    "    nn.ReLU(),\n",
    "    nn.Conv2d(16, 16, kernel_size=3, stride=2, padding=1),\n",
    "    nn.ReLU(),\n",
    "    nn.Conv2d(16, 10, kernel_size=3, stride=2, padding=1),\n",
    "    nn.ReLU(),\n",
    "    nn.AdaptiveAvgPool2d(1), # Average adaptive pooling allows us to define the output tensor of our choice \n",
    "                             # so we can use any input tensor of choice\n",
    "    Lambda(lambda x: x.view(x.size(0), -1)),\n",
    ")\n",
    "\n",
    "opt = optim.SGD(model.parameters(), lr=lr, momentum=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "-R4q2eifRjpx",
    "outputId": "a7effa9e-e050-4518-f219-3dddb511fa3b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 0.31927940299510954\n",
      "1 0.24793808761835098\n"
     ]
    }
   ],
   "source": [
    "fit(epochs, model, loss_func, opt, train_dl, valid_dl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "SZZJ1YirRnAd",
    "outputId": "71863e6c-f480-4222-a2af-876c046a57ad"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False\n"
     ]
    }
   ],
   "source": [
    "print(torch.cuda.is_available())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "YBVsWnKJRr50"
   },
   "outputs": [],
   "source": [
    "dev = torch.device(\n",
    "    \"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\") #  Creating a device objct which points to a GPU, if it exists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "fyhPpbVmSPkl"
   },
   "outputs": [],
   "source": [
    "# The preprocessing is updated to move the tensors to a GPU (if available)\n",
    "\n",
    "def preprocess(x, y):\n",
    "    return x.view(-1, 1, 28, 28).to(dev), y.to(dev)\n",
    "\n",
    "\n",
    "train_dl, valid_dl = get_data(train_ds, valid_ds, bs)\n",
    "train_dl = WrappedDataLoader(train_dl, preprocess)\n",
    "valid_dl = WrappedDataLoader(valid_dl, preprocess)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5WGHoF2HSZK2"
   },
   "outputs": [],
   "source": [
    "model.to(dev)\n",
    "opt = optim.SGD(model.parameters(), lr=lr, momentum=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "Zb_DMrH6Sc-C",
    "outputId": "ada5e6f5-9fa4-4f35-cb57-df651d7df756"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 0.20839096862077713\n",
      "1 0.1982756784558296\n"
     ]
    }
   ],
   "source": [
    "fit(epochs, model, loss_func, opt, train_dl, valid_dl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Hk7HUin6SfF3"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "authorship_tag": "ABX9TyNQojENF7V1NG4XtnrmIwpF",
   "collapsed_sections": [],
   "include_colab_link": true,
   "name": "Day 8# Beginner tutorial on Pytorch.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}