MNIST_pytorch_batchnorm.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "MNIST_pytorch_batchnorm.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "THJIMOj2ZkVR"
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torchvision\n",
        "import torchvision.transforms as transforms\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "\n",
        "input_size = 784\n",
        "hidden_size = 100\n",
        "num_classes = 10\n",
        "num_epochs = 2\n",
        "batch_size = 100\n",
        "learning_rate = 0.001\n",
        "\n",
        "\n",
        "train_dataset = torchvision.datasets.MNIST(root='./data', train=True, transform=transforms.ToTensor(), download=True)\n",
        "test_dataset = torchvision.datasets.MNIST(root='./data', train=False, transform=transforms.ToTensor())"
      ],
      "execution_count": 60,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PIwdDaYsaEM3"
      },
      "source": [
        "train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)\n",
        "test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False)"
      ],
      "execution_count": 61,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "R7Rx0mlkzxII",
        "outputId": "c3fb9f6c-fcc1-47fa-9690-e8fdfb32d438"
      },
      "source": [
        "examples = iter(train_loader)\n",
        "samples, labels = examples.next()\n",
        "print(samples.shape, labels.shape)\n",
        "\n"
      ],
      "execution_count": 62,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([100, 1, 28, 28]) torch.Size([100])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 267
        },
        "id": "UblOynXf0T1G",
        "outputId": "5e2b8e7e-3a7b-4c48-e269-1b33d69bc66f"
      },
      "source": [
        "for i in range(6):\n",
        "  plt.subplot(2,3,i+1)\n",
        "  plt.imshow(samples[i][0], cmap='gray')\n",
        "plt.show()  "
      ],
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD6CAYAAAC4RRw1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAcNUlEQVR4nO3de5RVVR0H8O8PFA0wecqacBR1GdOImviCpFQIFlgGSZIuhHGFUb4NWjKGki3FptUKc5UvEnJ0mWlKgZYyREqBRDzk4SDIY4WoA+PgA0xR0N0fc9jsvZl7586955x79rnfz1qz+O2779zzgx+z59x99zlblFIgIiL/tCt2AkRElB8O4EREnuIATkTkKQ7gRESe4gBOROQpDuBERJ4qaAAXkeEislFENotIdVhJUXGxrunF2qaL5LsOXETaA3gNwFAAbwBYDuBypdT68NKjuLGu6cXaps9hBXzvOQA2K6W2AoCI/BHASAAZ/zOICK8aSgillGToYl09lqWuQBtry7omSpNSqqf7YCFTKL0BbDfabwSPWURkooisEJEVBRyL4sO6plertWVdE2tbSw8WcgaeE6XUTAAzAf5GTxPWNZ1YV78Ucgb+JoByo31s8Bj5jXVNL9Y2ZQoZwJcDOFlEThCRDgAuAzAvnLSoiFjX9GJtUybvKRSl1H4RuQ7AfADtAcxWStWHlhkVBeuaXqxt+uS9jDCvg3FOLTFaWa3QJqxrcrCuqbVSKXWW+yCvxCQi8hQHcCIiT3EAJyLyVOTrwImIiuXWW2+12lVVVVb75ptv1vH+/futvmeeeSa6xELCM3AiIk9xACci8hSXEYasXbuDvxM7depk9X33u9/V8dChQ62+MWPG6PiBBx6w+qZOnarjd955J5Q8udwsnVhX4Atf+IKOly5davWVl5e7T9fuvvtuqz158uRwEysMlxESEaUJB3AiIk9xACci8hSXEbaROccNAOPGjbPaQ4YM0fEVV1yR8+uan0X84Ac/sPoGDhyo48GDB1t9Yc2Jp1VFRYWOhw8fbvV9+ctf1rGIZOwDgFNPPTXjMUaNGqXjefN4b6g4mD+HZ51lTw2fcMIJOs425w0ATU1NOt6+fXuWZyYTz8CJiDzFAZyIyFOcQsnBSSedpOPbb7/d6hs7dqzV3rdvn443b95s9T3xxBM63rJlS8bj1dTUWO3TTjtNx+7SJnOJYan68Y9/rOMbbrjB6uvSpYuOO3funPE1PvroI6vtvp1+7bXXdOwuD+3Z85CtCilihx12cOj697//nfF5b75p71fR2NhotW+88UYdL168OKTs4sMzcCIiT3EAJyLyFAdwIiJPcQ480L17dx1fcsklVt99992nY3PuDQA2bNhgtX/0ox/p+Pnnn88rF3de+5hjjsnrdUrFd77zHR2789FLlizR8csvv2z1Pf300zrevXu31bd+/fqMx+vatavV3rNnT+7JUuTMOXF3GWFdXZ3V9nHe28QzcCIiT3EAJyLyVElNoXTr1k3HlZWVVt/06dN1/NWvfjXja9x5551W+6GHHrLa27ZtKyTFVvl4tVjYZs2aZbXHjx+vY3c6o6GhIfTjv/vuu6G/JrVN//79M/Y9+eSTOnY3afjd734XWU7FwDNwIiJPcQAnIvIUB3AiIk+lekcec84bAK699lod/+xnP8v4febyMgC45557dBzVsqOysjIdr1ixwur75JNPdPylL33J6tu7d29ex/Nt55ZTTjlFx8uXL7f6zLs1rlmzJupUEs23uubL3HD4G9/4htVn/h84++yzrT53Ttwj3JGHiChNWh3ARWS2iDSKyCvGY91EZIGIbAr+7JrtNSh5WNf0Ym1LRy7LCB8G8FsAjxiPVQNYqJSqEZHqoD0l/PTabsCAATp27xw4bNiwjN83cuRIHa9atcrqc+9oFoXvf//7OjanUwDgN7/5jY7znTJpwcNIcF3dDRbMOw66SzXNOwUSgITXNh/unSTr6+t1PGLECKvv5z//uY49njLJSatn4EqpfwJwt30ZCaA2iGsBjAJ5hXVNL9a2dOQ7B95LKXXgCokdAHqFlA8VF+uaXqxtChV8JaZSSmX7tFpEJgKYWOhxKF6sa3plqy3r6peclhGKSB8Azyql+gXtjQAuUEo1iEgZgBeVUn1zeJ3QlyX16dPHaq9bt07H7s4p5lyyGQP2vKq5q05Uzj//fKs9f/58HXfo0MHqMy/7d+9+mK/mn2Hpg4TWtWPHjlb7gw8+0PGMGTOsPnN+PBt3rtTcuPgrX/lK1u/duHGjjpctW2b1mRsZf/zxxznlEpUDywjDqG2SlhG6dxH83Oc+p2Nz1yUg+wbUHgt1GeE8AFVBXAVgbr5ZUaKwrunF2qZQLssIHwewFEBfEXlDRCYAqAEwVEQ2Afh60CaPsK7pxdqWjlbnwJVSl2foGhJyLnn53//+Z7Vnz56tY3MTYcC+wtG8urEY3CkUc9rE3bh406ZNoR8/6XXNpn379lb7ggsu0PGDDz5o9R177LE6PuKII6y+du1yfwPqXu1nMpeZXnzxxVbf6tWrcz5GWHyurcmc4jr99NOtPnNTjYqKiryPceaZZ+rYnJYB7GnVpN4FlFdiEhF5igM4EZGnOIATEXkq1XcjTJrjjjtOx4sWLbL6jj/+eB1/8YtftPo2b94cei5Jv2tdtmWEbWHuntPY2Gj1PfXUUzreunWr1XfiiSdmfE338wtzCaI7V2ouAf3oo49yyLgwSa9rW5hz4O4G4X/96191PGHChIyvMXr0aKt90003WW1zZx93DnzLli06Hjp0qNX33//+N+MxI8K7ERIRpQkHcCIiT5XUpsZxc5e7mZvxlpeXW30rV67U8a5du6JNzAPuFY1XXnmlji+88MKM3/ePf/wjYzuqu0qaV2a6GwgMHjxYx+bbfjpU7969rfavf/1rHbubs5hXLrvMuxGam7gAh24GsmDBAh1/61vfsvpOOukkHS9dujRjbr/4xS8y5hI1noETEXmKAzgRkac4gBMReYpz4CE77LCD/6S33nqr1TdkyMErmZ977jmrL9ul2qXo008/tdqPPPJIi3ESjB07Vsfubk7nnXeejjkHnl1TU5PVNjcenzvXvvdWtn9Lc0er999/3+r7wx/+kLH9+c9/3uqbMuXghkXu8kM312LhGTgRkac4gBMReYoDOBGRp1J3Kb15y1BzPtplzkcDwKhRue3xOn36dKu9e/duq3311Vfr+Pbbb7f6zPXC06ZNs/rM9ahxSNMl10ni3iKhR48eOjYvDQei2fkpTXU1b/9sXvIOAEcddZSO3VtKZ/u5b8su9d/85jd1bO66BAD333+/jt215hHhpfRERGnCAZyIyFPeLyN035b+8pe/1LH7dum2227TsbvDh/k6/fr1s/rMt2Tjxo2z+ty7kpmX37p3nzN32nnppZdA6fOXv/zFav/qV7/SsfvWPo7Ns32WbSokm7ZMk2Rz7rnn6ljEnpnK9+6YYeMZOBGRpziAExF5igM4EZGnvJwDN+evn3nmGavP3MHdvZTdXJZkxgBwxx136Pjmm2+2+u666y4du/Ny5py3y52L69WrV8bnUjq4u/VQ/mpra3VcU1Nj9Zm3fn388ccjOf4xxxyjY3e59erVqyM5ZlvxDJyIyFMcwImIPOXFFIq7we3f/vY3Hbt3rRszZoyO871jmLtkKF/m1WIA8Pvf/17HkyZNyvh9l112mdWOaieZtLj++ut1bL7tBg69UjZq5ubUAPDee+/p+LPPPos1F9+Z0xTuv91Pf/pTHYc1hfLtb3/bal911VWhvG6UeAZOROQpDuBERJ5qdQAXkXIReUFE1otIvYjcGDzeTUQWiMim4M+u0adLYWFd04l1LS25zIHvBzBZKbVKRI4CsFJEFgC4EsBCpVSNiFQDqAYwJcvr5G3GjBlW29xxw11GmOu897Bhw6x2dXW1jgcNGmT1tWt38PfcNddcY/W5u1Wb3GWD3/ve9zI+d8OGDTqOabePotc1LOa/q3tXSXO5mXvXurCUl5fr2Lz7IADcc889Ov74448jOb4jNXVdu3atjv/zn/9YfebPqLkLPQCsXLlSx+bPFQBUVFRY7QkTJui4srLS6jPn3d1doOrq6rLmHpdWz8CVUg1KqVVBvAfAqwB6AxgJ4MAnRrUAcrsfKyUC65pOrGtpadMqFBHpA+AMAMsA9FJKNQRdOwC0eJWKiEwEMDH/FClqrGs6sa7pl/OGDiLSGcAiANOVUnNE5D2lVBej/12lVNZ5tXxvEL9mzRqrfeqpp+rY3Wx0z549OnaXdJmbz5pvewGgQ4cOOq6vr7f6Lr30Uh27b8l8deDG/8Wsa1jMOwCaUyYAsG7dOh0/9dRTVp959W0hzOVup512mtV38cUX6ziOTY3TVFfT0UcfbbXNTcEHDBiQ8fvMZZwA0KVLlwzPPFRVVZWOH3300Zy/LyL5b+ggIocDeBrAY0qpOcHDO0WkLOgvA9AYVqYUD9Y1nVjX0pHLKhQBMAvAq0op89PEeQAO/IqqAjA3/PQoKqxrOrGupSWXOfDzAIwDsE5EDrxX/AmAGgBPisgEANsAjMnw/ZRMrGs6sa4lxItNjc27AQL2kj93aVinTp1yes3NmzdbbXPu9K233rL64r4cOw5p2vzWrPncufaJ5eDBg3Xs/l83P+tYsmSJ1ecuEzOXnbp3HOzbt6+OzUu8AXuJm3vbhyikqa7ZmDV37zpqbkbsfmY1f/58q92168GPAbZu3Wr1zZkzR8dxjpMZcFNjIqI04QBOROQpL6ZQOnfubLXNqy/Nt0CAvYzriSeesPrMt8XuHcz27t2bT2reSutbbfcOkH/60590fMopp1h9vXv31rF7B8q2/FyYd5l0r9SN6epLLa11JU6hEBGlCgdwIiJPcQAnIvKUF3PgFL5SnCs98sgjrfbAgQN1PGWKfWO+999/32qbt2h47LHHrL7FixfreN++fQXnWYhSrGuJ4Bw4EVGacAAnIvIUp1BKFN9qpxPrmlqcQiEiShMO4EREnuIATkTkKQ7gRESe4gBOROQpDuBERJ7iAE5E5CkO4EREnuIATkTkKQ7gRESeymVX+jA1oXlH7B5BnASlmMvxIb8e65od6xqeUs2lxdrGei8UfVCRFS1d118MzCU8ScqfuYQnSfkzFxunUIiIPMUBnIjIU8UawGcW6bgtYS7hSVL+zCU8ScqfuRiKMgdORESF4xQKEZGnOIATEXkq1gFcRIaLyEYR2Swi1XEeOzj+bBFpFJFXjMe6icgCEdkU/Nk1hjzKReQFEVkvIvUicmOxcgkD62rlkprasq5WLomsa2wDuIi0B3AvgBEAKgFcLiKVcR0/8DCA4c5j1QAWKqVOBrAwaEdtP4DJSqlKAAMAXBv8WxQjl4KwrodIRW1Z10Mks65KqVi+AAwEMN9o3wLglriObxy3D4BXjPZGAGVBXAZgYxFymgtgaBJyYV1ZW9bVn7rGOYXSG8B2o/1G8Fix9VJKNQTxDgC94jy4iPQBcAaAZcXOJU+sawae15Z1zSBJdeWHmAbV/Gs0tnWVItIZwNMAblJK7S5mLmlWjH9L1jZ6rGu8A/ibAMqN9rHBY8W2U0TKACD4szGOg4rI4Wj+j/CYUmpOMXMpEOvqSEltWVdHEusa5wC+HMDJInKCiHQAcBmAeTEeP5N5AKqCuArNc1uREhEBMAvAq0qpGcXMJQSsqyFFtWVdDYmta8wT/xcBeA3AFgBTi/DBw+MAGgDsQ/Oc3gQA3dH86fEmAH8H0C2GPAah+a3WWgCrg6+LipEL68rasq7+1pWX0hMReYofYhIReYoDOBGRpwoawIt9qS1Fg3VNL9Y2ZQqY1G+P5g83TgTQAcAaAJWtfI/iVzK+WNd0foX5M1vsvwu/rK+3W6pRIWfg5wDYrJTaqpT6BMAfAYws4PUoGVjX9GJt/bWtpQcLGcBzutRWRCaKyAoRWVHAsSg+rGt6tVpb1tUvh0V9AKXUTARbD4mIivp4FA/WNZ1YV78Ucgae1EttqTCsa3qxtilTyACe1EttqTCsa3qxtimT9xSKUmq/iFwHYD6aP92erZSqDy0zKgrWNb1Y2/SJ9VJ6zqklh1JKwnot1jU5WNfUWqmUOst9kFdiEhF5igM4EZGnOIATEXmKAzgRkac4gBMReYoDOBGRpyK/lJ6IKE7l5QcvNn399detvrvvvttqT5o0KZacosIzcCIiT3EAJyLyFAdwIiJPcQ68SB599FGrXVFRoeOzzz477nTIcfXVV1vte++9N+Nz58+fr+NLL73U6vvggw/CTYza5LPPPrPaV111ldWuq6vT8fPPPx9LTmHiGTgRkac4gBMReYpTKDEyp0lGjRpl9W3YsCHudKgNst21c9iwYTru2rWr1ccplGTp3Lmz1Z48ebKOlyxZYvXt2bMnlpwKwTNwIiJPcQAnIvIUB3AiIk9xDjxGnTp10nHHjh2tvqamprjToSw++eQTq/3pp5/quH379nGnQxE58sgjddyhQ4ciZpIfnoETEXmKAzgRkadKdgrla1/7mtVev369jqOazjCXDrrL0qZPnx7JMSk/s2bNsto33HCDjvv16xd3OhSR++67T8e7du0qYib54Rk4EZGnOIATEXmKAzgRkadKag586tSpOr7jjjusvgcffFDH7p3owjJ69Ggdu/NtixcvjuSYlJ/x48db7b59+xYpEypEu3b2OeqqVaus9nPPPRdnOqHjGTgRkadaHcBFZLaINIrIK8Zj3URkgYhsCv7smu01KHlY1/RibUtHLlMoDwP4LYBHjMeqASxUStWISHXQnhJ+euHKtowvCubdBwH7bfjbb78d+fFb8TBSUtcomBvjAsDhhx+e8bm7d+/WsXnFZhE9DNYWwKEbOrz44otW+7333osxm/C1egaulPongHech0cCqA3iWgCjQF5hXdOLtS0d+X6I2Usp1RDEOwD0yvREEZkIYGKex6F4sa7plVNtWVe/FLwKRSmlRCTjfIRSaiaAmQCQ7XmULKxremWrLevql3wH8J0iUqaUahCRMgCNYSYVluHDh1vt/v3761hEIj++e7m+ecw///nPkR8/D17UNQruDknuMlPzM5O9e/dm/N633norguxCUTK1NW97kHb5LiOcB6AqiKsAzA0nHSoy1jW9WNsUymUZ4eMAlgLoKyJviMgEADUAhorIJgBfD9rkEdY1vVjb0tHqFIpS6vIMXUNCziV0tbW1Vjvb0sEopjTcZYRxLF3Mlc91DUuXLl10fM0111h97hSbWbvrrrvO6lu0aFEE2eWv1Gp7xBFHWG335y7NeCUmEZGnOIATEXmKAzgRkadSdzfCnj17thgD9jzmtGnTrL66urpQjn/mmWfqeOzYsVbfhx9+GPrxKH8DBgzQ8ZAh9vSw+3nF2rVrdfzss89Gmxi1yRlnnGG1L7rooiJlEj+egRMReYoDOBGRp1I3hfLIIwdvwOa+DZ4zZ46O27KJsLksyb260mVewde9e3er7+WXX9ZxQq/ETLVzzjnHarsbF2czc+ZMHSfgTpKUhbkE1F0OGscV2HHiGTgRkac4gBMReYoDOBGRp7yfA3eXCprz1e581yWXXKJjd6cOc74822XU27dvt/rcOx6a+STp0nkCFixYYLWPOuqojM994IEHrPb9998fSU4UPvPnzv0ZTNvPJM/AiYg8xQGciMhTHMCJiDzl/Ry4uyb3X//6l46PO+44qy/b/JfZt2HDBqvPnC+/8MILrb6mpqaMr5O2+TYf9enTR8ft2tnnK9nqc+edd0aVElFoeAZOROQpDuBERJ7yfgrFNX78eB1v3LjR6jOnQnbt2mX1rV+/XseLFy/O+XhTp0612tku1b3rrrtyfl0Kx8SJE3XcsWPHjM+bPXu21U7w5sTkqKqqav1JAfNWG2nAM3AiIk9xACci8hQHcCIiT6VuDtzUllvG5mvUqFFW21yaZs6rA7yFbByOPvpoqz1w4MCMzzWXoD700EOR5UTROv3003N+7po1ayLMJH48Ayci8hQHcCIiT6V6CiUK5qbFANC/f3+rbS4jrKmpiSUnOmjQoEFWO9sOSjt27NDxsmXLIsuJojVp0iSr/dJLL+k4bTvwuHgGTkTkqVYHcBEpF5EXRGS9iNSLyI3B491EZIGIbAr+7Bp9uhQW1jWdWNfSkssZ+H4Ak5VSlQAGALhWRCoBVANYqJQ6GcDCoE3+YF3TiXUtIa3OgSulGgA0BPEeEXkVQG8AIwFcEDytFsCLAKZEkmWCZFs2CNh3JzTvjJg0aa3rD3/4w5yf+84770SYSXGkta5tUUp3BG3Th5gi0gfAGQCWAegV/GcBgB0AemX4nokAJrbUR8nAuqYT65p+OX+IKSKdATwN4Cal1G6zTzX/mmvxV51SaqZS6iyl1FkFZUqRYF3TiXUtDTmdgYvI4Wj+z/CYUmpO8PBOESlTSjWISBmAxqiSTJLRo0dbbXeZ0rRp03T8+uuvx5JTvkq9rrfddluxU4hEqdc1m6VLl1rt66+/XscrVqyIO52C5bIKRQDMAvCqUmqG0TUPwIH7OFYBmBt+ehQV1jWdWNfSkssZ+HkAxgFYJyKrg8d+AqAGwJMiMgHANgBjokmRIsK6phPrWkJyWYWyGECmy5mGhJsOxYV1TSfWtbTwUvocVFRU6Lhv375Wn7tMqb6+PpaciKjZvn37rPaHH36oY3cXpnPPPddq9+zZM7rEYsBL6YmIPMUBnIjIU5xCycH555+v43bt7N95ixYtstpt2RCZisu8i92SJUuKmAkVYuXKlVbbvBrX3cR48uTJVjvJV0vngmfgRESe4gBOROQpDuBERJ6SOO/WJSJe3hqsR48eOt65c6fVN2LECKtdV1cXS06FUkqFtlVJkup6xRVXWO3a2tqMz506daqO07J7UlrrSljZ0v1peAZOROQpDuBERJ7iFEqJ4lvtdGJdU4tTKEREacIBnIjIUxzAiYg8xQGciMhTHMCJiDzFAZyIyFMcwImIPMUBnIjIUxzAiYg8xQGciMhTce/I0wRgG4AeQZwEpZjL8SG/HuuaHesanlLNpcXaxnovFH1QkRUtXddfDMwlPEnKn7mEJ0n5Mxcbp1CIiDzFAZyIyFPFGsBnFum4LWEu4UlS/swlPEnKn7kYijIHTkREheMUChGRpziAExF5KtYBXESGi8hGEdksItVxHjs4/mwRaRSRV4zHuonIAhHZFPzZNYY8ykXkBRFZLyL1InJjsXIJA+tq5ZKa2rKuVi6JrGtsA7iItAdwL4ARACoBXC4ilXEdP/AwgOHOY9UAFiqlTgawMGhHbT+AyUqpSgADAFwb/FsUI5eCsK6HSEVtWddDJLOuSqlYvgAMBDDfaN8C4Ja4jm8ctw+AV4z2RgBlQVwGYGMRcpoLYGgScmFdWVvW1Z+6xjmF0hvAdqP9RvBYsfVSSjUE8Q4AveI8uIj0AXAGgGXFziVPrGsGnteWdc0gSXXlh5gG1fxrNLZ1lSLSGcDTAG5SSu0uZi5pVox/S9Y2eqxrvAP4mwDKjfaxwWPFtlNEygAg+LMxjoOKyOFo/o/wmFJqTjFzKRDr6khJbVlXRxLrGucAvhzAySJygoh0AHAZgHkxHj+TeQCqgrgKzXNbkRIRATALwKtKqRnFzCUErKshRbVlXQ2JrWvME/8XAXgNwBYAU4vwwcPjABoA7EPznN4EAN3R/OnxJgB/B9AthjwGofmt1loAq4Ovi4qRC+vK2rKu/taVl9ITEXmKH2ISEXmKAzgRkac4gBMReYoDOBGRpziAExF5igM4EZGnOIATEXnq/45baCNTtMXQAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 6 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "enO5lqni1CH2"
      },
      "source": [
        "import torch.nn.functional as F\n",
        "class NeuralNet(nn.Module):\n",
        "  def __init__(self, input_size, hidden_size, num_classes, use_batch_norm=True):\n",
        "    super(NeuralNet, self).__init__()\n",
        "    self.use_batch_norm=True\n",
        "    self.relu = nn.functional.relu\n",
        "    self.batch_norm1 = nn.BatchNorm1d(hidden_size)\n",
        "    if use_batch_norm:\n",
        "      self.fc1 = nn.Linear(input_size, hidden_size, bias=False)\n",
        "      self.batch_norm1 = nn.BatchNorm1d(hidden_size)\n",
        "    else:\n",
        "      self.fc1 = nn.Linear(input_size, hidden_size) \n",
        "    \n",
        "   \n",
        "\n",
        "\n",
        "    self.fc2 = nn.Linear(hidden_size, num_classes)   \n",
        "    \n",
        "    \n",
        "\n",
        "  def forward(self, x):\n",
        "    x = self.fc1(x)\n",
        "    if self.use_batch_norm:\n",
        "      x = self.batch_norm1(x)\n",
        "    x = self.relu(x)  \n",
        "    \n",
        "    out = self.fc2(x)\n",
        "    return out\n",
        "\n",
        "# model_batchnorm = NeuralNet(input_size, hidden_size, num_classes, use_batch_norm=True)    \n",
        "# model_no_batchnorm = NeuralNet(input_size, hidden_size, num_classes, use_batch_norm=False)\n",
        "model_batchnorm = NeuralNet(input_size, hidden_size, num_classes,use_batch_norm=False )    \n",
        "\n",
        "# print(model_batchnorm)\n",
        "# print()\n",
        "# print(model_no_batchnorm)\n"
      ],
      "execution_count": 64,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TCJuxHlk7f6u"
      },
      "source": [
        "criterion = nn.CrossEntropyLoss()\n",
        "optimizer = torch.optim.Adam(model_batchnorm.parameters(), lr=learning_rate)"
      ],
      "execution_count": 65,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "C3SuMqR0ZSHz",
        "outputId": "93e91634-bab8-4264-f796-7060c439d0cd"
      },
      "source": [
        "n_total_steps = len(train_loader)\n",
        "for epoch in range(num_epochs):\n",
        "  for i,(images, labels) in enumerate(train_loader):\n",
        "    images = images.reshape(-1, 28*28).to(device)\n",
        "    labels = labels.to(device)\n",
        "\n",
        "    # forward pass\n",
        "    outputs = model_batchnorm(images)\n",
        "    loss = criterion(outputs, labels)\n",
        "\n",
        "    # backward pass\n",
        "    optimizer.zero_grad()\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "    print (f'Epoch [{epoch+1}/{num_epochs}], Step [{i+1}/{n_total_steps}], Loss: {loss.item():.4f}')"
      ],
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch [1/2], Step [1/600], Loss: 2.3990\n",
            "Epoch [1/2], Step [2/600], Loss: 2.0961\n",
            "Epoch [1/2], Step [3/600], Loss: 1.8519\n",
            "Epoch [1/2], Step [4/600], Loss: 1.8243\n",
            "Epoch [1/2], Step [5/600], Loss: 1.6016\n",
            "Epoch [1/2], Step [6/600], Loss: 1.5333\n",
            "Epoch [1/2], Step [7/600], Loss: 1.3850\n",
            "Epoch [1/2], Step [8/600], Loss: 1.3023\n",
            "Epoch [1/2], Step [9/600], Loss: 1.3230\n",
            "Epoch [1/2], Step [10/600], Loss: 1.1806\n",
            "Epoch [1/2], Step [11/600], Loss: 1.1046\n",
            "Epoch [1/2], Step [12/600], Loss: 1.2111\n",
            "Epoch [1/2], Step [13/600], Loss: 1.1467\n",
            "Epoch [1/2], Step [14/600], Loss: 1.0520\n",
            "Epoch [1/2], Step [15/600], Loss: 0.9555\n",
            "Epoch [1/2], Step [16/600], Loss: 1.0581\n",
            "Epoch [1/2], Step [17/600], Loss: 1.0007\n",
            "Epoch [1/2], Step [18/600], Loss: 0.9654\n",
            "Epoch [1/2], Step [19/600], Loss: 0.9903\n",
            "Epoch [1/2], Step [20/600], Loss: 0.9334\n",
            "Epoch [1/2], Step [21/600], Loss: 0.9433\n",
            "Epoch [1/2], Step [22/600], Loss: 0.8684\n",
            "Epoch [1/2], Step [23/600], Loss: 0.9374\n",
            "Epoch [1/2], Step [24/600], Loss: 0.8602\n",
            "Epoch [1/2], Step [25/600], Loss: 0.8686\n",
            "Epoch [1/2], Step [26/600], Loss: 0.8011\n",
            "Epoch [1/2], Step [27/600], Loss: 0.8443\n",
            "Epoch [1/2], Step [28/600], Loss: 0.7751\n",
            "Epoch [1/2], Step [29/600], Loss: 0.8450\n",
            "Epoch [1/2], Step [30/600], Loss: 0.7363\n",
            "Epoch [1/2], Step [31/600], Loss: 0.7759\n",
            "Epoch [1/2], Step [32/600], Loss: 0.6978\n",
            "Epoch [1/2], Step [33/600], Loss: 0.6668\n",
            "Epoch [1/2], Step [34/600], Loss: 0.6437\n",
            "Epoch [1/2], Step [35/600], Loss: 0.7713\n",
            "Epoch [1/2], Step [36/600], Loss: 0.6432\n",
            "Epoch [1/2], Step [37/600], Loss: 0.6775\n",
            "Epoch [1/2], Step [38/600], Loss: 0.6399\n",
            "Epoch [1/2], Step [39/600], Loss: 0.7728\n",
            "Epoch [1/2], Step [40/600], Loss: 0.7129\n",
            "Epoch [1/2], Step [41/600], Loss: 0.5497\n",
            "Epoch [1/2], Step [42/600], Loss: 0.6665\n",
            "Epoch [1/2], Step [43/600], Loss: 0.6792\n",
            "Epoch [1/2], Step [44/600], Loss: 0.5798\n",
            "Epoch [1/2], Step [45/600], Loss: 0.6094\n",
            "Epoch [1/2], Step [46/600], Loss: 0.5911\n",
            "Epoch [1/2], Step [47/600], Loss: 0.5916\n",
            "Epoch [1/2], Step [48/600], Loss: 0.5970\n",
            "Epoch [1/2], Step [49/600], Loss: 0.6018\n",
            "Epoch [1/2], Step [50/600], Loss: 0.5161\n",
            "Epoch [1/2], Step [51/600], Loss: 0.5933\n",
            "Epoch [1/2], Step [52/600], Loss: 0.5822\n",
            "Epoch [1/2], Step [53/600], Loss: 0.5705\n",
            "Epoch [1/2], Step [54/600], Loss: 0.6029\n",
            "Epoch [1/2], Step [55/600], Loss: 0.5690\n",
            "Epoch [1/2], Step [56/600], Loss: 0.4834\n",
            "Epoch [1/2], Step [57/600], Loss: 0.4557\n",
            "Epoch [1/2], Step [58/600], Loss: 0.5212\n",
            "Epoch [1/2], Step [59/600], Loss: 0.6786\n",
            "Epoch [1/2], Step [60/600], Loss: 0.6168\n",
            "Epoch [1/2], Step [61/600], Loss: 0.4469\n",
            "Epoch [1/2], Step [62/600], Loss: 0.4561\n",
            "Epoch [1/2], Step [63/600], Loss: 0.5265\n",
            "Epoch [1/2], Step [64/600], Loss: 0.4750\n",
            "Epoch [1/2], Step [65/600], Loss: 0.4029\n",
            "Epoch [1/2], Step [66/600], Loss: 0.4866\n",
            "Epoch [1/2], Step [67/600], Loss: 0.4362\n",
            "Epoch [1/2], Step [68/600], Loss: 0.4855\n",
            "Epoch [1/2], Step [69/600], Loss: 0.3986\n",
            "Epoch [1/2], Step [70/600], Loss: 0.4943\n",
            "Epoch [1/2], Step [71/600], Loss: 0.4511\n",
            "Epoch [1/2], Step [72/600], Loss: 0.5239\n",
            "Epoch [1/2], Step [73/600], Loss: 0.4287\n",
            "Epoch [1/2], Step [74/600], Loss: 0.4778\n",
            "Epoch [1/2], Step [75/600], Loss: 0.3461\n",
            "Epoch [1/2], Step [76/600], Loss: 0.5121\n",
            "Epoch [1/2], Step [77/600], Loss: 0.4963\n",
            "Epoch [1/2], Step [78/600], Loss: 0.5517\n",
            "Epoch [1/2], Step [79/600], Loss: 0.5293\n",
            "Epoch [1/2], Step [80/600], Loss: 0.5412\n",
            "Epoch [1/2], Step [81/600], Loss: 0.5648\n",
            "Epoch [1/2], Step [82/600], Loss: 0.3523\n",
            "Epoch [1/2], Step [83/600], Loss: 0.4090\n",
            "Epoch [1/2], Step [84/600], Loss: 0.4315\n",
            "Epoch [1/2], Step [85/600], Loss: 0.4378\n",
            "Epoch [1/2], Step [86/600], Loss: 0.4915\n",
            "Epoch [1/2], Step [87/600], Loss: 0.5719\n",
            "Epoch [1/2], Step [88/600], Loss: 0.5355\n",
            "Epoch [1/2], Step [89/600], Loss: 0.3529\n",
            "Epoch [1/2], Step [90/600], Loss: 0.5149\n",
            "Epoch [1/2], Step [91/600], Loss: 0.4258\n",
            "Epoch [1/2], Step [92/600], Loss: 0.4359\n",
            "Epoch [1/2], Step [93/600], Loss: 0.2846\n",
            "Epoch [1/2], Step [94/600], Loss: 0.4903\n",
            "Epoch [1/2], Step [95/600], Loss: 0.4542\n",
            "Epoch [1/2], Step [96/600], Loss: 0.4809\n",
            "Epoch [1/2], Step [97/600], Loss: 0.4713\n",
            "Epoch [1/2], Step [98/600], Loss: 0.3327\n",
            "Epoch [1/2], Step [99/600], Loss: 0.3066\n",
            "Epoch [1/2], Step [100/600], Loss: 0.4643\n",
            "Epoch [1/2], Step [101/600], Loss: 0.4937\n",
            "Epoch [1/2], Step [102/600], Loss: 0.4735\n",
            "Epoch [1/2], Step [103/600], Loss: 0.4192\n",
            "Epoch [1/2], Step [104/600], Loss: 0.3470\n",
            "Epoch [1/2], Step [105/600], Loss: 0.2585\n",
            "Epoch [1/2], Step [106/600], Loss: 0.3586\n",
            "Epoch [1/2], Step [107/600], Loss: 0.2854\n",
            "Epoch [1/2], Step [108/600], Loss: 0.3460\n",
            "Epoch [1/2], Step [109/600], Loss: 0.5148\n",
            "Epoch [1/2], Step [110/600], Loss: 0.3696\n",
            "Epoch [1/2], Step [111/600], Loss: 0.4795\n",
            "Epoch [1/2], Step [112/600], Loss: 0.4211\n",
            "Epoch [1/2], Step [113/600], Loss: 0.4231\n",
            "Epoch [1/2], Step [114/600], Loss: 0.2784\n",
            "Epoch [1/2], Step [115/600], Loss: 0.4161\n",
            "Epoch [1/2], Step [116/600], Loss: 0.3624\n",
            "Epoch [1/2], Step [117/600], Loss: 0.4225\n",
            "Epoch [1/2], Step [118/600], Loss: 0.3571\n",
            "Epoch [1/2], Step [119/600], Loss: 0.2968\n",
            "Epoch [1/2], Step [120/600], Loss: 0.4356\n",
            "Epoch [1/2], Step [121/600], Loss: 0.3677\n",
            "Epoch [1/2], Step [122/600], Loss: 0.5070\n",
            "Epoch [1/2], Step [123/600], Loss: 0.3754\n",
            "Epoch [1/2], Step [124/600], Loss: 0.5236\n",
            "Epoch [1/2], Step [125/600], Loss: 0.3433\n",
            "Epoch [1/2], Step [126/600], Loss: 0.4152\n",
            "Epoch [1/2], Step [127/600], Loss: 0.2677\n",
            "Epoch [1/2], Step [128/600], Loss: 0.3080\n",
            "Epoch [1/2], Step [129/600], Loss: 0.4539\n",
            "Epoch [1/2], Step [130/600], Loss: 0.3114\n",
            "Epoch [1/2], Step [131/600], Loss: 0.2452\n",
            "Epoch [1/2], Step [132/600], Loss: 0.3901\n",
            "Epoch [1/2], Step [133/600], Loss: 0.3868\n",
            "Epoch [1/2], Step [134/600], Loss: 0.3002\n",
            "Epoch [1/2], Step [135/600], Loss: 0.4375\n",
            "Epoch [1/2], Step [136/600], Loss: 0.2965\n",
            "Epoch [1/2], Step [137/600], Loss: 0.3627\n",
            "Epoch [1/2], Step [138/600], Loss: 0.3105\n",
            "Epoch [1/2], Step [139/600], Loss: 0.2895\n",
            "Epoch [1/2], Step [140/600], Loss: 0.3001\n",
            "Epoch [1/2], Step [141/600], Loss: 0.3767\n",
            "Epoch [1/2], Step [142/600], Loss: 0.3070\n",
            "Epoch [1/2], Step [143/600], Loss: 0.2435\n",
            "Epoch [1/2], Step [144/600], Loss: 0.2847\n",
            "Epoch [1/2], Step [145/600], Loss: 0.3739\n",
            "Epoch [1/2], Step [146/600], Loss: 0.3163\n",
            "Epoch [1/2], Step [147/600], Loss: 0.3493\n",
            "Epoch [1/2], Step [148/600], Loss: 0.4011\n",
            "Epoch [1/2], Step [149/600], Loss: 0.3592\n",
            "Epoch [1/2], Step [150/600], Loss: 0.2841\n",
            "Epoch [1/2], Step [151/600], Loss: 0.3293\n",
            "Epoch [1/2], Step [152/600], Loss: 0.2525\n",
            "Epoch [1/2], Step [153/600], Loss: 0.3843\n",
            "Epoch [1/2], Step [154/600], Loss: 0.3118\n",
            "Epoch [1/2], Step [155/600], Loss: 0.2645\n",
            "Epoch [1/2], Step [156/600], Loss: 0.2980\n",
            "Epoch [1/2], Step [157/600], Loss: 0.3710\n",
            "Epoch [1/2], Step [158/600], Loss: 0.5590\n",
            "Epoch [1/2], Step [159/600], Loss: 0.4172\n",
            "Epoch [1/2], Step [160/600], Loss: 0.2766\n",
            "Epoch [1/2], Step [161/600], Loss: 0.2386\n",
            "Epoch [1/2], Step [162/600], Loss: 0.3750\n",
            "Epoch [1/2], Step [163/600], Loss: 0.2642\n",
            "Epoch [1/2], Step [164/600], Loss: 0.2985\n",
            "Epoch [1/2], Step [165/600], Loss: 0.2743\n",
            "Epoch [1/2], Step [166/600], Loss: 0.2364\n",
            "Epoch [1/2], Step [167/600], Loss: 0.2128\n",
            "Epoch [1/2], Step [168/600], Loss: 0.3597\n",
            "Epoch [1/2], Step [169/600], Loss: 0.2706\n",
            "Epoch [1/2], Step [170/600], Loss: 0.2761\n",
            "Epoch [1/2], Step [171/600], Loss: 0.3254\n",
            "Epoch [1/2], Step [172/600], Loss: 0.4028\n",
            "Epoch [1/2], Step [173/600], Loss: 0.3599\n",
            "Epoch [1/2], Step [174/600], Loss: 0.3537\n",
            "Epoch [1/2], Step [175/600], Loss: 0.2438\n",
            "Epoch [1/2], Step [176/600], Loss: 0.3852\n",
            "Epoch [1/2], Step [177/600], Loss: 0.2285\n",
            "Epoch [1/2], Step [178/600], Loss: 0.2165\n",
            "Epoch [1/2], Step [179/600], Loss: 0.3471\n",
            "Epoch [1/2], Step [180/600], Loss: 0.2075\n",
            "Epoch [1/2], Step [181/600], Loss: 0.2992\n",
            "Epoch [1/2], Step [182/600], Loss: 0.2394\n",
            "Epoch [1/2], Step [183/600], Loss: 0.3048\n",
            "Epoch [1/2], Step [184/600], Loss: 0.4062\n",
            "Epoch [1/2], Step [185/600], Loss: 0.2931\n",
            "Epoch [1/2], Step [186/600], Loss: 0.4409\n",
            "Epoch [1/2], Step [187/600], Loss: 0.2614\n",
            "Epoch [1/2], Step [188/600], Loss: 0.2786\n",
            "Epoch [1/2], Step [189/600], Loss: 0.3112\n",
            "Epoch [1/2], Step [190/600], Loss: 0.3052\n",
            "Epoch [1/2], Step [191/600], Loss: 0.2563\n",
            "Epoch [1/2], Step [192/600], Loss: 0.3742\n",
            "Epoch [1/2], Step [193/600], Loss: 0.2625\n",
            "Epoch [1/2], Step [194/600], Loss: 0.3159\n",
            "Epoch [1/2], Step [195/600], Loss: 0.1993\n",
            "Epoch [1/2], Step [196/600], Loss: 0.2641\n",
            "Epoch [1/2], Step [197/600], Loss: 0.2834\n",
            "Epoch [1/2], Step [198/600], Loss: 0.2945\n",
            "Epoch [1/2], Step [199/600], Loss: 0.2449\n",
            "Epoch [1/2], Step [200/600], Loss: 0.3341\n",
            "Epoch [1/2], Step [201/600], Loss: 0.4020\n",
            "Epoch [1/2], Step [202/600], Loss: 0.2289\n",
            "Epoch [1/2], Step [203/600], Loss: 0.4104\n",
            "Epoch [1/2], Step [204/600], Loss: 0.1818\n",
            "Epoch [1/2], Step [205/600], Loss: 0.2356\n",
            "Epoch [1/2], Step [206/600], Loss: 0.2723\n",
            "Epoch [1/2], Step [207/600], Loss: 0.2909\n",
            "Epoch [1/2], Step [208/600], Loss: 0.4741\n",
            "Epoch [1/2], Step [209/600], Loss: 0.3532\n",
            "Epoch [1/2], Step [210/600], Loss: 0.2398\n",
            "Epoch [1/2], Step [211/600], Loss: 0.2830\n",
            "Epoch [1/2], Step [212/600], Loss: 0.2384\n",
            "Epoch [1/2], Step [213/600], Loss: 0.3297\n",
            "Epoch [1/2], Step [214/600], Loss: 0.2429\n",
            "Epoch [1/2], Step [215/600], Loss: 0.2893\n",
            "Epoch [1/2], Step [216/600], Loss: 0.2672\n",
            "Epoch [1/2], Step [217/600], Loss: 0.2379\n",
            "Epoch [1/2], Step [218/600], Loss: 0.2731\n",
            "Epoch [1/2], Step [219/600], Loss: 0.2956\n",
            "Epoch [1/2], Step [220/600], Loss: 0.2908\n",
            "Epoch [1/2], Step [221/600], Loss: 0.2928\n",
            "Epoch [1/2], Step [222/600], Loss: 0.2592\n",
            "Epoch [1/2], Step [223/600], Loss: 0.3326\n",
            "Epoch [1/2], Step [224/600], Loss: 0.3249\n",
            "Epoch [1/2], Step [225/600], Loss: 0.1907\n",
            "Epoch [1/2], Step [226/600], Loss: 0.3866\n",
            "Epoch [1/2], Step [227/600], Loss: 0.2008\n",
            "Epoch [1/2], Step [228/600], Loss: 0.2678\n",
            "Epoch [1/2], Step [229/600], Loss: 0.2912\n",
            "Epoch [1/2], Step [230/600], Loss: 0.3336\n",
            "Epoch [1/2], Step [231/600], Loss: 0.3573\n",
            "Epoch [1/2], Step [232/600], Loss: 0.2366\n",
            "Epoch [1/2], Step [233/600], Loss: 0.3574\n",
            "Epoch [1/2], Step [234/600], Loss: 0.2713\n",
            "Epoch [1/2], Step [235/600], Loss: 0.2473\n",
            "Epoch [1/2], Step [236/600], Loss: 0.2676\n",
            "Epoch [1/2], Step [237/600], Loss: 0.2748\n",
            "Epoch [1/2], Step [238/600], Loss: 0.3293\n",
            "Epoch [1/2], Step [239/600], Loss: 0.2450\n",
            "Epoch [1/2], Step [240/600], Loss: 0.2629\n",
            "Epoch [1/2], Step [241/600], Loss: 0.2118\n",
            "Epoch [1/2], Step [242/600], Loss: 0.3047\n",
            "Epoch [1/2], Step [243/600], Loss: 0.2124\n",
            "Epoch [1/2], Step [244/600], Loss: 0.2511\n",
            "Epoch [1/2], Step [245/600], Loss: 0.2426\n",
            "Epoch [1/2], Step [246/600], Loss: 0.2338\n",
            "Epoch [1/2], Step [247/600], Loss: 0.3373\n",
            "Epoch [1/2], Step [248/600], Loss: 0.2719\n",
            "Epoch [1/2], Step [249/600], Loss: 0.2896\n",
            "Epoch [1/2], Step [250/600], Loss: 0.2270\n",
            "Epoch [1/2], Step [251/600], Loss: 0.2065\n",
            "Epoch [1/2], Step [252/600], Loss: 0.2751\n",
            "Epoch [1/2], Step [253/600], Loss: 0.2832\n",
            "Epoch [1/2], Step [254/600], Loss: 0.2383\n",
            "Epoch [1/2], Step [255/600], Loss: 0.1369\n",
            "Epoch [1/2], Step [256/600], Loss: 0.3748\n",
            "Epoch [1/2], Step [257/600], Loss: 0.2181\n",
            "Epoch [1/2], Step [258/600], Loss: 0.3876\n",
            "Epoch [1/2], Step [259/600], Loss: 0.3513\n",
            "Epoch [1/2], Step [260/600], Loss: 0.2570\n",
            "Epoch [1/2], Step [261/600], Loss: 0.2925\n",
            "Epoch [1/2], Step [262/600], Loss: 0.3273\n",
            "Epoch [1/2], Step [263/600], Loss: 0.3204\n",
            "Epoch [1/2], Step [264/600], Loss: 0.1874\n",
            "Epoch [1/2], Step [265/600], Loss: 0.2721\n",
            "Epoch [1/2], Step [266/600], Loss: 0.2874\n",
            "Epoch [1/2], Step [267/600], Loss: 0.2804\n",
            "Epoch [1/2], Step [268/600], Loss: 0.1828\n",
            "Epoch [1/2], Step [269/600], Loss: 0.1803\n",
            "Epoch [1/2], Step [270/600], Loss: 0.2897\n",
            "Epoch [1/2], Step [271/600], Loss: 0.1867\n",
            "Epoch [1/2], Step [272/600], Loss: 0.2324\n",
            "Epoch [1/2], Step [273/600], Loss: 0.3498\n",
            "Epoch [1/2], Step [274/600], Loss: 0.2027\n",
            "Epoch [1/2], Step [275/600], Loss: 0.2607\n",
            "Epoch [1/2], Step [276/600], Loss: 0.3129\n",
            "Epoch [1/2], Step [277/600], Loss: 0.2685\n",
            "Epoch [1/2], Step [278/600], Loss: 0.1257\n",
            "Epoch [1/2], Step [279/600], Loss: 0.2465\n",
            "Epoch [1/2], Step [280/600], Loss: 0.2339\n",
            "Epoch [1/2], Step [281/600], Loss: 0.2574\n",
            "Epoch [1/2], Step [282/600], Loss: 0.1774\n",
            "Epoch [1/2], Step [283/600], Loss: 0.2625\n",
            "Epoch [1/2], Step [284/600], Loss: 0.3071\n",
            "Epoch [1/2], Step [285/600], Loss: 0.2555\n",
            "Epoch [1/2], Step [286/600], Loss: 0.1678\n",
            "Epoch [1/2], Step [287/600], Loss: 0.2311\n",
            "Epoch [1/2], Step [288/600], Loss: 0.2417\n",
            "Epoch [1/2], Step [289/600], Loss: 0.2541\n",
            "Epoch [1/2], Step [290/600], Loss: 0.1378\n",
            "Epoch [1/2], Step [291/600], Loss: 0.3508\n",
            "Epoch [1/2], Step [292/600], Loss: 0.2011\n",
            "Epoch [1/2], Step [293/600], Loss: 0.2560\n",
            "Epoch [1/2], Step [294/600], Loss: 0.2520\n",
            "Epoch [1/2], Step [295/600], Loss: 0.2601\n",
            "Epoch [1/2], Step [296/600], Loss: 0.1963\n",
            "Epoch [1/2], Step [297/600], Loss: 0.3304\n",
            "Epoch [1/2], Step [298/600], Loss: 0.3154\n",
            "Epoch [1/2], Step [299/600], Loss: 0.2095\n",
            "Epoch [1/2], Step [300/600], Loss: 0.2387\n",
            "Epoch [1/2], Step [301/600], Loss: 0.2502\n",
            "Epoch [1/2], Step [302/600], Loss: 0.1887\n",
            "Epoch [1/2], Step [303/600], Loss: 0.2692\n",
            "Epoch [1/2], Step [304/600], Loss: 0.2778\n",
            "Epoch [1/2], Step [305/600], Loss: 0.2546\n",
            "Epoch [1/2], Step [306/600], Loss: 0.3328\n",
            "Epoch [1/2], Step [307/600], Loss: 0.1459\n",
            "Epoch [1/2], Step [308/600], Loss: 0.2279\n",
            "Epoch [1/2], Step [309/600], Loss: 0.3612\n",
            "Epoch [1/2], Step [310/600], Loss: 0.2079\n",
            "Epoch [1/2], Step [311/600], Loss: 0.2949\n",
            "Epoch [1/2], Step [312/600], Loss: 0.1477\n",
            "Epoch [1/2], Step [313/600], Loss: 0.3330\n",
            "Epoch [1/2], Step [314/600], Loss: 0.2463\n",
            "Epoch [1/2], Step [315/600], Loss: 0.1885\n",
            "Epoch [1/2], Step [316/600], Loss: 0.2370\n",
            "Epoch [1/2], Step [317/600], Loss: 0.2964\n",
            "Epoch [1/2], Step [318/600], Loss: 0.2789\n",
            "Epoch [1/2], Step [319/600], Loss: 0.2482\n",
            "Epoch [1/2], Step [320/600], Loss: 0.3448\n",
            "Epoch [1/2], Step [321/600], Loss: 0.2233\n",
            "Epoch [1/2], Step [322/600], Loss: 0.1675\n",
            "Epoch [1/2], Step [323/600], Loss: 0.3487\n",
            "Epoch [1/2], Step [324/600], Loss: 0.2450\n",
            "Epoch [1/2], Step [325/600], Loss: 0.3451\n",
            "Epoch [1/2], Step [326/600], Loss: 0.2302\n",
            "Epoch [1/2], Step [327/600], Loss: 0.2130\n",
            "Epoch [1/2], Step [328/600], Loss: 0.1930\n",
            "Epoch [1/2], Step [329/600], Loss: 0.3003\n",
            "Epoch [1/2], Step [330/600], Loss: 0.1772\n",
            "Epoch [1/2], Step [331/600], Loss: 0.2555\n",
            "Epoch [1/2], Step [332/600], Loss: 0.2720\n",
            "Epoch [1/2], Step [333/600], Loss: 0.3862\n",
            "Epoch [1/2], Step [334/600], Loss: 0.3579\n",
            "Epoch [1/2], Step [335/600], Loss: 0.2037\n",
            "Epoch [1/2], Step [336/600], Loss: 0.2789\n",
            "Epoch [1/2], Step [337/600], Loss: 0.1437\n",
            "Epoch [1/2], Step [338/600], Loss: 0.3028\n",
            "Epoch [1/2], Step [339/600], Loss: 0.1740\n",
            "Epoch [1/2], Step [340/600], Loss: 0.1530\n",
            "Epoch [1/2], Step [341/600], Loss: 0.1721\n",
            "Epoch [1/2], Step [342/600], Loss: 0.2214\n",
            "Epoch [1/2], Step [343/600], Loss: 0.2367\n",
            "Epoch [1/2], Step [344/600], Loss: 0.3291\n",
            "Epoch [1/2], Step [345/600], Loss: 0.2006\n",
            "Epoch [1/2], Step [346/600], Loss: 0.1881\n",
            "Epoch [1/2], Step [347/600], Loss: 0.2036\n",
            "Epoch [1/2], Step [348/600], Loss: 0.1513\n",
            "Epoch [1/2], Step [349/600], Loss: 0.3091\n",
            "Epoch [1/2], Step [350/600], Loss: 0.2732\n",
            "Epoch [1/2], Step [351/600], Loss: 0.2640\n",
            "Epoch [1/2], Step [352/600], Loss: 0.3252\n",
            "Epoch [1/2], Step [353/600], Loss: 0.3927\n",
            "Epoch [1/2], Step [354/600], Loss: 0.2325\n",
            "Epoch [1/2], Step [355/600], Loss: 0.2527\n",
            "Epoch [1/2], Step [356/600], Loss: 0.2723\n",
            "Epoch [1/2], Step [357/600], Loss: 0.3322\n",
            "Epoch [1/2], Step [358/600], Loss: 0.1486\n",
            "Epoch [1/2], Step [359/600], Loss: 0.1339\n",
            "Epoch [1/2], Step [360/600], Loss: 0.1950\n",
            "Epoch [1/2], Step [361/600], Loss: 0.1827\n",
            "Epoch [1/2], Step [362/600], Loss: 0.1857\n",
            "Epoch [1/2], Step [363/600], Loss: 0.2345\n",
            "Epoch [1/2], Step [364/600], Loss: 0.1261\n",
            "Epoch [1/2], Step [365/600], Loss: 0.1966\n",
            "Epoch [1/2], Step [366/600], Loss: 0.2539\n",
            "Epoch [1/2], Step [367/600], Loss: 0.3239\n",
            "Epoch [1/2], Step [368/600], Loss: 0.1977\n",
            "Epoch [1/2], Step [369/600], Loss: 0.2215\n",
            "Epoch [1/2], Step [370/600], Loss: 0.2450\n",
            "Epoch [1/2], Step [371/600], Loss: 0.2507\n",
            "Epoch [1/2], Step [372/600], Loss: 0.1852\n",
            "Epoch [1/2], Step [373/600], Loss: 0.2374\n",
            "Epoch [1/2], Step [374/600], Loss: 0.1310\n",
            "Epoch [1/2], Step [375/600], Loss: 0.1954\n",
            "Epoch [1/2], Step [376/600], Loss: 0.1983\n",
            "Epoch [1/2], Step [377/600], Loss: 0.1685\n",
            "Epoch [1/2], Step [378/600], Loss: 0.2420\n",
            "Epoch [1/2], Step [379/600], Loss: 0.3286\n",
            "Epoch [1/2], Step [380/600], Loss: 0.2350\n",
            "Epoch [1/2], Step [381/600], Loss: 0.2908\n",
            "Epoch [1/2], Step [382/600], Loss: 0.0968\n",
            "Epoch [1/2], Step [383/600], Loss: 0.2392\n",
            "Epoch [1/2], Step [384/600], Loss: 0.1383\n",
            "Epoch [1/2], Step [385/600], Loss: 0.1511\n",
            "Epoch [1/2], Step [386/600], Loss: 0.2306\n",
            "Epoch [1/2], Step [387/600], Loss: 0.1302\n",
            "Epoch [1/2], Step [388/600], Loss: 0.2053\n",
            "Epoch [1/2], Step [389/600], Loss: 0.2757\n",
            "Epoch [1/2], Step [390/600], Loss: 0.2582\n",
            "Epoch [1/2], Step [391/600], Loss: 0.1261\n",
            "Epoch [1/2], Step [392/600], Loss: 0.2367\n",
            "Epoch [1/2], Step [393/600], Loss: 0.1110\n",
            "Epoch [1/2], Step [394/600], Loss: 0.3368\n",
            "Epoch [1/2], Step [395/600], Loss: 0.2534\n",
            "Epoch [1/2], Step [396/600], Loss: 0.3298\n",
            "Epoch [1/2], Step [397/600], Loss: 0.2238\n",
            "Epoch [1/2], Step [398/600], Loss: 0.1617\n",
            "Epoch [1/2], Step [399/600], Loss: 0.1473\n",
            "Epoch [1/2], Step [400/600], Loss: 0.0959\n",
            "Epoch [1/2], Step [401/600], Loss: 0.1313\n",
            "Epoch [1/2], Step [402/600], Loss: 0.2124\n",
            "Epoch [1/2], Step [403/600], Loss: 0.1834\n",
            "Epoch [1/2], Step [404/600], Loss: 0.1943\n",
            "Epoch [1/2], Step [405/600], Loss: 0.2962\n",
            "Epoch [1/2], Step [406/600], Loss: 0.1849\n",
            "Epoch [1/2], Step [407/600], Loss: 0.1377\n",
            "Epoch [1/2], Step [408/600], Loss: 0.1891\n",
            "Epoch [1/2], Step [409/600], Loss: 0.1673\n",
            "Epoch [1/2], Step [410/600], Loss: 0.2528\n",
            "Epoch [1/2], Step [411/600], Loss: 0.1134\n",
            "Epoch [1/2], Step [412/600], Loss: 0.2350\n",
            "Epoch [1/2], Step [413/600], Loss: 0.2188\n",
            "Epoch [1/2], Step [414/600], Loss: 0.2346\n",
            "Epoch [1/2], Step [415/600], Loss: 0.2628\n",
            "Epoch [1/2], Step [416/600], Loss: 0.2072\n",
            "Epoch [1/2], Step [417/600], Loss: 0.1194\n",
            "Epoch [1/2], Step [418/600], Loss: 0.2562\n",
            "Epoch [1/2], Step [419/600], Loss: 0.1755\n",
            "Epoch [1/2], Step [420/600], Loss: 0.3481\n",
            "Epoch [1/2], Step [421/600], Loss: 0.3107\n",
            "Epoch [1/2], Step [422/600], Loss: 0.2888\n",
            "Epoch [1/2], Step [423/600], Loss: 0.2689\n",
            "Epoch [1/2], Step [424/600], Loss: 0.1737\n",
            "Epoch [1/2], Step [425/600], Loss: 0.3070\n",
            "Epoch [1/2], Step [426/600], Loss: 0.1934\n",
            "Epoch [1/2], Step [427/600], Loss: 0.1794\n",
            "Epoch [1/2], Step [428/600], Loss: 0.1900\n",
            "Epoch [1/2], Step [429/600], Loss: 0.1201\n",
            "Epoch [1/2], Step [430/600], Loss: 0.2442\n",
            "Epoch [1/2], Step [431/600], Loss: 0.1901\n",
            "Epoch [1/2], Step [432/600], Loss: 0.2264\n",
            "Epoch [1/2], Step [433/600], Loss: 0.1968\n",
            "Epoch [1/2], Step [434/600], Loss: 0.2238\n",
            "Epoch [1/2], Step [435/600], Loss: 0.1647\n",
            "Epoch [1/2], Step [436/600], Loss: 0.2249\n",
            "Epoch [1/2], Step [437/600], Loss: 0.2816\n",
            "Epoch [1/2], Step [438/600], Loss: 0.1877\n",
            "Epoch [1/2], Step [439/600], Loss: 0.2632\n",
            "Epoch [1/2], Step [440/600], Loss: 0.1765\n",
            "Epoch [1/2], Step [441/600], Loss: 0.1695\n",
            "Epoch [1/2], Step [442/600], Loss: 0.1260\n",
            "Epoch [1/2], Step [443/600], Loss: 0.1614\n",
            "Epoch [1/2], Step [444/600], Loss: 0.1431\n",
            "Epoch [1/2], Step [445/600], Loss: 0.1841\n",
            "Epoch [1/2], Step [446/600], Loss: 0.1878\n",
            "Epoch [1/2], Step [447/600], Loss: 0.2176\n",
            "Epoch [1/2], Step [448/600], Loss: 0.3258\n",
            "Epoch [1/2], Step [449/600], Loss: 0.3752\n",
            "Epoch [1/2], Step [450/600], Loss: 0.2022\n",
            "Epoch [1/2], Step [451/600], Loss: 0.1300\n",
            "Epoch [1/2], Step [452/600], Loss: 0.1674\n",
            "Epoch [1/2], Step [453/600], Loss: 0.1425\n",
            "Epoch [1/2], Step [454/600], Loss: 0.1891\n",
            "Epoch [1/2], Step [455/600], Loss: 0.1449\n",
            "Epoch [1/2], Step [456/600], Loss: 0.2484\n",
            "Epoch [1/2], Step [457/600], Loss: 0.1310\n",
            "Epoch [1/2], Step [458/600], Loss: 0.1766\n",
            "Epoch [1/2], Step [459/600], Loss: 0.1801\n",
            "Epoch [1/2], Step [460/600], Loss: 0.1872\n",
            "Epoch [1/2], Step [461/600], Loss: 0.2415\n",
            "Epoch [1/2], Step [462/600], Loss: 0.2304\n",
            "Epoch [1/2], Step [463/600], Loss: 0.2193\n",
            "Epoch [1/2], Step [464/600], Loss: 0.1517\n",
            "Epoch [1/2], Step [465/600], Loss: 0.1744\n",
            "Epoch [1/2], Step [466/600], Loss: 0.2150\n",
            "Epoch [1/2], Step [467/600], Loss: 0.1817\n",
            "Epoch [1/2], Step [468/600], Loss: 0.2196\n",
            "Epoch [1/2], Step [469/600], Loss: 0.2386\n",
            "Epoch [1/2], Step [470/600], Loss: 0.2383\n",
            "Epoch [1/2], Step [471/600], Loss: 0.2321\n",
            "Epoch [1/2], Step [472/600], Loss: 0.2480\n",
            "Epoch [1/2], Step [473/600], Loss: 0.1838\n",
            "Epoch [1/2], Step [474/600], Loss: 0.1425\n",
            "Epoch [1/2], Step [475/600], Loss: 0.2120\n",
            "Epoch [1/2], Step [476/600], Loss: 0.2310\n",
            "Epoch [1/2], Step [477/600], Loss: 0.2674\n",
            "Epoch [1/2], Step [478/600], Loss: 0.1896\n",
            "Epoch [1/2], Step [479/600], Loss: 0.2347\n",
            "Epoch [1/2], Step [480/600], Loss: 0.1063\n",
            "Epoch [1/2], Step [481/600], Loss: 0.2442\n",
            "Epoch [1/2], Step [482/600], Loss: 0.2104\n",
            "Epoch [1/2], Step [483/600], Loss: 0.1985\n",
            "Epoch [1/2], Step [484/600], Loss: 0.1677\n",
            "Epoch [1/2], Step [485/600], Loss: 0.2805\n",
            "Epoch [1/2], Step [486/600], Loss: 0.1204\n",
            "Epoch [1/2], Step [487/600], Loss: 0.2216\n",
            "Epoch [1/2], Step [488/600], Loss: 0.2160\n",
            "Epoch [1/2], Step [489/600], Loss: 0.1776\n",
            "Epoch [1/2], Step [490/600], Loss: 0.2744\n",
            "Epoch [1/2], Step [491/600], Loss: 0.1462\n",
            "Epoch [1/2], Step [492/600], Loss: 0.1972\n",
            "Epoch [1/2], Step [493/600], Loss: 0.0878\n",
            "Epoch [1/2], Step [494/600], Loss: 0.3022\n",
            "Epoch [1/2], Step [495/600], Loss: 0.2469\n",
            "Epoch [1/2], Step [496/600], Loss: 0.2762\n",
            "Epoch [1/2], Step [497/600], Loss: 0.2936\n",
            "Epoch [1/2], Step [498/600], Loss: 0.2117\n",
            "Epoch [1/2], Step [499/600], Loss: 0.2530\n",
            "Epoch [1/2], Step [500/600], Loss: 0.2134\n",
            "Epoch [1/2], Step [501/600], Loss: 0.1667\n",
            "Epoch [1/2], Step [502/600], Loss: 0.2292\n",
            "Epoch [1/2], Step [503/600], Loss: 0.1673\n",
            "Epoch [1/2], Step [504/600], Loss: 0.2461\n",
            "Epoch [1/2], Step [505/600], Loss: 0.1635\n",
            "Epoch [1/2], Step [506/600], Loss: 0.2147\n",
            "Epoch [1/2], Step [507/600], Loss: 0.2435\n",
            "Epoch [1/2], Step [508/600], Loss: 0.1907\n",
            "Epoch [1/2], Step [509/600], Loss: 0.1338\n",
            "Epoch [1/2], Step [510/600], Loss: 0.2270\n",
            "Epoch [1/2], Step [511/600], Loss: 0.3036\n",
            "Epoch [1/2], Step [512/600], Loss: 0.2730\n",
            "Epoch [1/2], Step [513/600], Loss: 0.1864\n",
            "Epoch [1/2], Step [514/600], Loss: 0.1311\n",
            "Epoch [1/2], Step [515/600], Loss: 0.1520\n",
            "Epoch [1/2], Step [516/600], Loss: 0.2353\n",
            "Epoch [1/2], Step [517/600], Loss: 0.2373\n",
            "Epoch [1/2], Step [518/600], Loss: 0.2416\n",
            "Epoch [1/2], Step [519/600], Loss: 0.1317\n",
            "Epoch [1/2], Step [520/600], Loss: 0.1891\n",
            "Epoch [1/2], Step [521/600], Loss: 0.1613\n",
            "Epoch [1/2], Step [522/600], Loss: 0.1106\n",
            "Epoch [1/2], Step [523/600], Loss: 0.2845\n",
            "Epoch [1/2], Step [524/600], Loss: 0.1491\n",
            "Epoch [1/2], Step [525/600], Loss: 0.1875\n",
            "Epoch [1/2], Step [526/600], Loss: 0.2219\n",
            "Epoch [1/2], Step [527/600], Loss: 0.2155\n",
            "Epoch [1/2], Step [528/600], Loss: 0.1622\n",
            "Epoch [1/2], Step [529/600], Loss: 0.1799\n",
            "Epoch [1/2], Step [530/600], Loss: 0.3339\n",
            "Epoch [1/2], Step [531/600], Loss: 0.1797\n",
            "Epoch [1/2], Step [532/600], Loss: 0.1864\n",
            "Epoch [1/2], Step [533/600], Loss: 0.1389\n",
            "Epoch [1/2], Step [534/600], Loss: 0.2049\n",
            "Epoch [1/2], Step [535/600], Loss: 0.2154\n",
            "Epoch [1/2], Step [536/600], Loss: 0.2165\n",
            "Epoch [1/2], Step [537/600], Loss: 0.1949\n",
            "Epoch [1/2], Step [538/600], Loss: 0.3424\n",
            "Epoch [1/2], Step [539/600], Loss: 0.2344\n",
            "Epoch [1/2], Step [540/600], Loss: 0.2100\n",
            "Epoch [1/2], Step [541/600], Loss: 0.2724\n",
            "Epoch [1/2], Step [542/600], Loss: 0.2240\n",
            "Epoch [1/2], Step [543/600], Loss: 0.1877\n",
            "Epoch [1/2], Step [544/600], Loss: 0.1702\n",
            "Epoch [1/2], Step [545/600], Loss: 0.1359\n",
            "Epoch [1/2], Step [546/600], Loss: 0.2102\n",
            "Epoch [1/2], Step [547/600], Loss: 0.1906\n",
            "Epoch [1/2], Step [548/600], Loss: 0.1494\n",
            "Epoch [1/2], Step [549/600], Loss: 0.1389\n",
            "Epoch [1/2], Step [550/600], Loss: 0.1047\n",
            "Epoch [1/2], Step [551/600], Loss: 0.2152\n",
            "Epoch [1/2], Step [552/600], Loss: 0.0987\n",
            "Epoch [1/2], Step [553/600], Loss: 0.2001\n",
            "Epoch [1/2], Step [554/600], Loss: 0.1570\n",
            "Epoch [1/2], Step [555/600], Loss: 0.2132\n",
            "Epoch [1/2], Step [556/600], Loss: 0.2958\n",
            "Epoch [1/2], Step [557/600], Loss: 0.2756\n",
            "Epoch [1/2], Step [558/600], Loss: 0.1539\n",
            "Epoch [1/2], Step [559/600], Loss: 0.1882\n",
            "Epoch [1/2], Step [560/600], Loss: 0.0991\n",
            "Epoch [1/2], Step [561/600], Loss: 0.3090\n",
            "Epoch [1/2], Step [562/600], Loss: 0.1298\n",
            "Epoch [1/2], Step [563/600], Loss: 0.1228\n",
            "Epoch [1/2], Step [564/600], Loss: 0.1764\n",
            "Epoch [1/2], Step [565/600], Loss: 0.1381\n",
            "Epoch [1/2], Step [566/600], Loss: 0.2391\n",
            "Epoch [1/2], Step [567/600], Loss: 0.1536\n",
            "Epoch [1/2], Step [568/600], Loss: 0.1660\n",
            "Epoch [1/2], Step [569/600], Loss: 0.2339\n",
            "Epoch [1/2], Step [570/600], Loss: 0.1866\n",
            "Epoch [1/2], Step [571/600], Loss: 0.2584\n",
            "Epoch [1/2], Step [572/600], Loss: 0.1497\n",
            "Epoch [1/2], Step [573/600], Loss: 0.1480\n",
            "Epoch [1/2], Step [574/600], Loss: 0.1328\n",
            "Epoch [1/2], Step [575/600], Loss: 0.1819\n",
            "Epoch [1/2], Step [576/600], Loss: 0.1099\n",
            "Epoch [1/2], Step [577/600], Loss: 0.1701\n",
            "Epoch [1/2], Step [578/600], Loss: 0.1436\n",
            "Epoch [1/2], Step [579/600], Loss: 0.0941\n",
            "Epoch [1/2], Step [580/600], Loss: 0.2163\n",
            "Epoch [1/2], Step [581/600], Loss: 0.1868\n",
            "Epoch [1/2], Step [582/600], Loss: 0.2563\n",
            "Epoch [1/2], Step [583/600], Loss: 0.1745\n",
            "Epoch [1/2], Step [584/600], Loss: 0.0828\n",
            "Epoch [1/2], Step [585/600], Loss: 0.2394\n",
            "Epoch [1/2], Step [586/600], Loss: 0.1370\n",
            "Epoch [1/2], Step [587/600], Loss: 0.1636\n",
            "Epoch [1/2], Step [588/600], Loss: 0.1773\n",
            "Epoch [1/2], Step [589/600], Loss: 0.1240\n",
            "Epoch [1/2], Step [590/600], Loss: 0.2424\n",
            "Epoch [1/2], Step [591/600], Loss: 0.1511\n",
            "Epoch [1/2], Step [592/600], Loss: 0.2111\n",
            "Epoch [1/2], Step [593/600], Loss: 0.3268\n",
            "Epoch [1/2], Step [594/600], Loss: 0.2057\n",
            "Epoch [1/2], Step [595/600], Loss: 0.2213\n",
            "Epoch [1/2], Step [596/600], Loss: 0.0830\n",
            "Epoch [1/2], Step [597/600], Loss: 0.1858\n",
            "Epoch [1/2], Step [598/600], Loss: 0.2402\n",
            "Epoch [1/2], Step [599/600], Loss: 0.1731\n",
            "Epoch [1/2], Step [600/600], Loss: 0.1568\n",
            "Epoch [2/2], Step [1/600], Loss: 0.2463\n",
            "Epoch [2/2], Step [2/600], Loss: 0.1689\n",
            "Epoch [2/2], Step [3/600], Loss: 0.1761\n",
            "Epoch [2/2], Step [4/600], Loss: 0.0723\n",
            "Epoch [2/2], Step [5/600], Loss: 0.2230\n",
            "Epoch [2/2], Step [6/600], Loss: 0.1275\n",
            "Epoch [2/2], Step [7/600], Loss: 0.1064\n",
            "Epoch [2/2], Step [8/600], Loss: 0.1206\n",
            "Epoch [2/2], Step [9/600], Loss: 0.1173\n",
            "Epoch [2/2], Step [10/600], Loss: 0.1109\n",
            "Epoch [2/2], Step [11/600], Loss: 0.1482\n",
            "Epoch [2/2], Step [12/600], Loss: 0.0841\n",
            "Epoch [2/2], Step [13/600], Loss: 0.1533\n",
            "Epoch [2/2], Step [14/600], Loss: 0.0900\n",
            "Epoch [2/2], Step [15/600], Loss: 0.1457\n",
            "Epoch [2/2], Step [16/600], Loss: 0.1171\n",
            "Epoch [2/2], Step [17/600], Loss: 0.2153\n",
            "Epoch [2/2], Step [18/600], Loss: 0.1367\n",
            "Epoch [2/2], Step [19/600], Loss: 0.1574\n",
            "Epoch [2/2], Step [20/600], Loss: 0.1328\n",
            "Epoch [2/2], Step [21/600], Loss: 0.1169\n",
            "Epoch [2/2], Step [22/600], Loss: 0.1908\n",
            "Epoch [2/2], Step [23/600], Loss: 0.1170\n",
            "Epoch [2/2], Step [24/600], Loss: 0.2820\n",
            "Epoch [2/2], Step [25/600], Loss: 0.1513\n",
            "Epoch [2/2], Step [26/600], Loss: 0.1267\n",
            "Epoch [2/2], Step [27/600], Loss: 0.1494\n",
            "Epoch [2/2], Step [28/600], Loss: 0.1345\n",
            "Epoch [2/2], Step [29/600], Loss: 0.1910\n",
            "Epoch [2/2], Step [30/600], Loss: 0.2060\n",
            "Epoch [2/2], Step [31/600], Loss: 0.1483\n",
            "Epoch [2/2], Step [32/600], Loss: 0.1769\n",
            "Epoch [2/2], Step [33/600], Loss: 0.1914\n",
            "Epoch [2/2], Step [34/600], Loss: 0.1479\n",
            "Epoch [2/2], Step [35/600], Loss: 0.2028\n",
            "Epoch [2/2], Step [36/600], Loss: 0.1256\n",
            "Epoch [2/2], Step [37/600], Loss: 0.1229\n",
            "Epoch [2/2], Step [38/600], Loss: 0.1973\n",
            "Epoch [2/2], Step [39/600], Loss: 0.0766\n",
            "Epoch [2/2], Step [40/600], Loss: 0.1831\n",
            "Epoch [2/2], Step [41/600], Loss: 0.0861\n",
            "Epoch [2/2], Step [42/600], Loss: 0.2025\n",
            "Epoch [2/2], Step [43/600], Loss: 0.1071\n",
            "Epoch [2/2], Step [44/600], Loss: 0.1024\n",
            "Epoch [2/2], Step [45/600], Loss: 0.1293\n",
            "Epoch [2/2], Step [46/600], Loss: 0.1128\n",
            "Epoch [2/2], Step [47/600], Loss: 0.1242\n",
            "Epoch [2/2], Step [48/600], Loss: 0.2228\n",
            "Epoch [2/2], Step [49/600], Loss: 0.1849\n",
            "Epoch [2/2], Step [50/600], Loss: 0.1801\n",
            "Epoch [2/2], Step [51/600], Loss: 0.0703\n",
            "Epoch [2/2], Step [52/600], Loss: 0.1450\n",
            "Epoch [2/2], Step [53/600], Loss: 0.1077\n",
            "Epoch [2/2], Step [54/600], Loss: 0.1311\n",
            "Epoch [2/2], Step [55/600], Loss: 0.1405\n",
            "Epoch [2/2], Step [56/600], Loss: 0.1688\n",
            "Epoch [2/2], Step [57/600], Loss: 0.2013\n",
            "Epoch [2/2], Step [58/600], Loss: 0.1210\n",
            "Epoch [2/2], Step [59/600], Loss: 0.2675\n",
            "Epoch [2/2], Step [60/600], Loss: 0.2435\n",
            "Epoch [2/2], Step [61/600], Loss: 0.1241\n",
            "Epoch [2/2], Step [62/600], Loss: 0.0980\n",
            "Epoch [2/2], Step [63/600], Loss: 0.1661\n",
            "Epoch [2/2], Step [64/600], Loss: 0.2192\n",
            "Epoch [2/2], Step [65/600], Loss: 0.1461\n",
            "Epoch [2/2], Step [66/600], Loss: 0.0712\n",
            "Epoch [2/2], Step [67/600], Loss: 0.0971\n",
            "Epoch [2/2], Step [68/600], Loss: 0.1006\n",
            "Epoch [2/2], Step [69/600], Loss: 0.1053\n",
            "Epoch [2/2], Step [70/600], Loss: 0.1224\n",
            "Epoch [2/2], Step [71/600], Loss: 0.1312\n",
            "Epoch [2/2], Step [72/600], Loss: 0.1320\n",
            "Epoch [2/2], Step [73/600], Loss: 0.1650\n",
            "Epoch [2/2], Step [74/600], Loss: 0.2943\n",
            "Epoch [2/2], Step [75/600], Loss: 0.2054\n",
            "Epoch [2/2], Step [76/600], Loss: 0.1613\n",
            "Epoch [2/2], Step [77/600], Loss: 0.1857\n",
            "Epoch [2/2], Step [78/600], Loss: 0.1263\n",
            "Epoch [2/2], Step [79/600], Loss: 0.2241\n",
            "Epoch [2/2], Step [80/600], Loss: 0.1827\n",
            "Epoch [2/2], Step [81/600], Loss: 0.2018\n",
            "Epoch [2/2], Step [82/600], Loss: 0.1377\n",
            "Epoch [2/2], Step [83/600], Loss: 0.1584\n",
            "Epoch [2/2], Step [84/600], Loss: 0.2000\n",
            "Epoch [2/2], Step [85/600], Loss: 0.1923\n",
            "Epoch [2/2], Step [86/600], Loss: 0.1515\n",
            "Epoch [2/2], Step [87/600], Loss: 0.0937\n",
            "Epoch [2/2], Step [88/600], Loss: 0.1225\n",
            "Epoch [2/2], Step [89/600], Loss: 0.2081\n",
            "Epoch [2/2], Step [90/600], Loss: 0.1139\n",
            "Epoch [2/2], Step [91/600], Loss: 0.0659\n",
            "Epoch [2/2], Step [92/600], Loss: 0.1439\n",
            "Epoch [2/2], Step [93/600], Loss: 0.1776\n",
            "Epoch [2/2], Step [94/600], Loss: 0.1941\n",
            "Epoch [2/2], Step [95/600], Loss: 0.1093\n",
            "Epoch [2/2], Step [96/600], Loss: 0.0900\n",
            "Epoch [2/2], Step [97/600], Loss: 0.2169\n",
            "Epoch [2/2], Step [98/600], Loss: 0.1396\n",
            "Epoch [2/2], Step [99/600], Loss: 0.1774\n",
            "Epoch [2/2], Step [100/600], Loss: 0.2638\n",
            "Epoch [2/2], Step [101/600], Loss: 0.1444\n",
            "Epoch [2/2], Step [102/600], Loss: 0.1803\n",
            "Epoch [2/2], Step [103/600], Loss: 0.1676\n",
            "Epoch [2/2], Step [104/600], Loss: 0.2253\n",
            "Epoch [2/2], Step [105/600], Loss: 0.2433\n",
            "Epoch [2/2], Step [106/600], Loss: 0.0889\n",
            "Epoch [2/2], Step [107/600], Loss: 0.1232\n",
            "Epoch [2/2], Step [108/600], Loss: 0.1354\n",
            "Epoch [2/2], Step [109/600], Loss: 0.1362\n",
            "Epoch [2/2], Step [110/600], Loss: 0.1312\n",
            "Epoch [2/2], Step [111/600], Loss: 0.1090\n",
            "Epoch [2/2], Step [112/600], Loss: 0.1689\n",
            "Epoch [2/2], Step [113/600], Loss: 0.2448\n",
            "Epoch [2/2], Step [114/600], Loss: 0.1333\n",
            "Epoch [2/2], Step [115/600], Loss: 0.1377\n",
            "Epoch [2/2], Step [116/600], Loss: 0.1782\n",
            "Epoch [2/2], Step [117/600], Loss: 0.3067\n",
            "Epoch [2/2], Step [118/600], Loss: 0.1705\n",
            "Epoch [2/2], Step [119/600], Loss: 0.1227\n",
            "Epoch [2/2], Step [120/600], Loss: 0.1509\n",
            "Epoch [2/2], Step [121/600], Loss: 0.1634\n",
            "Epoch [2/2], Step [122/600], Loss: 0.1636\n",
            "Epoch [2/2], Step [123/600], Loss: 0.1296\n",
            "Epoch [2/2], Step [124/600], Loss: 0.2438\n",
            "Epoch [2/2], Step [125/600], Loss: 0.1476\n",
            "Epoch [2/2], Step [126/600], Loss: 0.1043\n",
            "Epoch [2/2], Step [127/600], Loss: 0.1842\n",
            "Epoch [2/2], Step [128/600], Loss: 0.1475\n",
            "Epoch [2/2], Step [129/600], Loss: 0.1500\n",
            "Epoch [2/2], Step [130/600], Loss: 0.1220\n",
            "Epoch [2/2], Step [131/600], Loss: 0.2230\n",
            "Epoch [2/2], Step [132/600], Loss: 0.1046\n",
            "Epoch [2/2], Step [133/600], Loss: 0.1139\n",
            "Epoch [2/2], Step [134/600], Loss: 0.0662\n",
            "Epoch [2/2], Step [135/600], Loss: 0.0880\n",
            "Epoch [2/2], Step [136/600], Loss: 0.1067\n",
            "Epoch [2/2], Step [137/600], Loss: 0.1516\n",
            "Epoch [2/2], Step [138/600], Loss: 0.1284\n",
            "Epoch [2/2], Step [139/600], Loss: 0.0873\n",
            "Epoch [2/2], Step [140/600], Loss: 0.1232\n",
            "Epoch [2/2], Step [141/600], Loss: 0.1332\n",
            "Epoch [2/2], Step [142/600], Loss: 0.1571\n",
            "Epoch [2/2], Step [143/600], Loss: 0.1394\n",
            "Epoch [2/2], Step [144/600], Loss: 0.1043\n",
            "Epoch [2/2], Step [145/600], Loss: 0.1052\n",
            "Epoch [2/2], Step [146/600], Loss: 0.1832\n",
            "Epoch [2/2], Step [147/600], Loss: 0.1684\n",
            "Epoch [2/2], Step [148/600], Loss: 0.0977\n",
            "Epoch [2/2], Step [149/600], Loss: 0.1270\n",
            "Epoch [2/2], Step [150/600], Loss: 0.1307\n",
            "Epoch [2/2], Step [151/600], Loss: 0.1643\n",
            "Epoch [2/2], Step [152/600], Loss: 0.1653\n",
            "Epoch [2/2], Step [153/600], Loss: 0.1797\n",
            "Epoch [2/2], Step [154/600], Loss: 0.1138\n",
            "Epoch [2/2], Step [155/600], Loss: 0.2024\n",
            "Epoch [2/2], Step [156/600], Loss: 0.1147\n",
            "Epoch [2/2], Step [157/600], Loss: 0.1217\n",
            "Epoch [2/2], Step [158/600], Loss: 0.1245\n",
            "Epoch [2/2], Step [159/600], Loss: 0.1339\n",
            "Epoch [2/2], Step [160/600], Loss: 0.1344\n",
            "Epoch [2/2], Step [161/600], Loss: 0.2035\n",
            "Epoch [2/2], Step [162/600], Loss: 0.1163\n",
            "Epoch [2/2], Step [163/600], Loss: 0.1558\n",
            "Epoch [2/2], Step [164/600], Loss: 0.1304\n",
            "Epoch [2/2], Step [165/600], Loss: 0.0797\n",
            "Epoch [2/2], Step [166/600], Loss: 0.0991\n",
            "Epoch [2/2], Step [167/600], Loss: 0.1966\n",
            "Epoch [2/2], Step [168/600], Loss: 0.0936\n",
            "Epoch [2/2], Step [169/600], Loss: 0.1382\n",
            "Epoch [2/2], Step [170/600], Loss: 0.1397\n",
            "Epoch [2/2], Step [171/600], Loss: 0.0992\n",
            "Epoch [2/2], Step [172/600], Loss: 0.1700\n",
            "Epoch [2/2], Step [173/600], Loss: 0.1510\n",
            "Epoch [2/2], Step [174/600], Loss: 0.1658\n",
            "Epoch [2/2], Step [175/600], Loss: 0.2581\n",
            "Epoch [2/2], Step [176/600], Loss: 0.2533\n",
            "Epoch [2/2], Step [177/600], Loss: 0.0736\n",
            "Epoch [2/2], Step [178/600], Loss: 0.2303\n",
            "Epoch [2/2], Step [179/600], Loss: 0.2002\n",
            "Epoch [2/2], Step [180/600], Loss: 0.1051\n",
            "Epoch [2/2], Step [181/600], Loss: 0.1707\n",
            "Epoch [2/2], Step [182/600], Loss: 0.0943\n",
            "Epoch [2/2], Step [183/600], Loss: 0.2087\n",
            "Epoch [2/2], Step [184/600], Loss: 0.1196\n",
            "Epoch [2/2], Step [185/600], Loss: 0.2143\n",
            "Epoch [2/2], Step [186/600], Loss: 0.1541\n",
            "Epoch [2/2], Step [187/600], Loss: 0.1844\n",
            "Epoch [2/2], Step [188/600], Loss: 0.2220\n",
            "Epoch [2/2], Step [189/600], Loss: 0.1659\n",
            "Epoch [2/2], Step [190/600], Loss: 0.1431\n",
            "Epoch [2/2], Step [191/600], Loss: 0.1058\n",
            "Epoch [2/2], Step [192/600], Loss: 0.1079\n",
            "Epoch [2/2], Step [193/600], Loss: 0.0923\n",
            "Epoch [2/2], Step [194/600], Loss: 0.2113\n",
            "Epoch [2/2], Step [195/600], Loss: 0.1907\n",
            "Epoch [2/2], Step [196/600], Loss: 0.2122\n",
            "Epoch [2/2], Step [197/600], Loss: 0.1488\n",
            "Epoch [2/2], Step [198/600], Loss: 0.2668\n",
            "Epoch [2/2], Step [199/600], Loss: 0.0867\n",
            "Epoch [2/2], Step [200/600], Loss: 0.1514\n",
            "Epoch [2/2], Step [201/600], Loss: 0.0827\n",
            "Epoch [2/2], Step [202/600], Loss: 0.1152\n",
            "Epoch [2/2], Step [203/600], Loss: 0.2662\n",
            "Epoch [2/2], Step [204/600], Loss: 0.1944\n",
            "Epoch [2/2], Step [205/600], Loss: 0.2310\n",
            "Epoch [2/2], Step [206/600], Loss: 0.2100\n",
            "Epoch [2/2], Step [207/600], Loss: 0.2109\n",
            "Epoch [2/2], Step [208/600], Loss: 0.1363\n",
            "Epoch [2/2], Step [209/600], Loss: 0.1934\n",
            "Epoch [2/2], Step [210/600], Loss: 0.0931\n",
            "Epoch [2/2], Step [211/600], Loss: 0.1770\n",
            "Epoch [2/2], Step [212/600], Loss: 0.1357\n",
            "Epoch [2/2], Step [213/600], Loss: 0.1514\n",
            "Epoch [2/2], Step [214/600], Loss: 0.1888\n",
            "Epoch [2/2], Step [215/600], Loss: 0.1171\n",
            "Epoch [2/2], Step [216/600], Loss: 0.1521\n",
            "Epoch [2/2], Step [217/600], Loss: 0.2062\n",
            "Epoch [2/2], Step [218/600], Loss: 0.1541\n",
            "Epoch [2/2], Step [219/600], Loss: 0.1495\n",
            "Epoch [2/2], Step [220/600], Loss: 0.1635\n",
            "Epoch [2/2], Step [221/600], Loss: 0.1231\n",
            "Epoch [2/2], Step [222/600], Loss: 0.2261\n",
            "Epoch [2/2], Step [223/600], Loss: 0.1254\n",
            "Epoch [2/2], Step [224/600], Loss: 0.2357\n",
            "Epoch [2/2], Step [225/600], Loss: 0.1724\n",
            "Epoch [2/2], Step [226/600], Loss: 0.0789\n",
            "Epoch [2/2], Step [227/600], Loss: 0.1832\n",
            "Epoch [2/2], Step [228/600], Loss: 0.1246\n",
            "Epoch [2/2], Step [229/600], Loss: 0.1711\n",
            "Epoch [2/2], Step [230/600], Loss: 0.1520\n",
            "Epoch [2/2], Step [231/600], Loss: 0.1098\n",
            "Epoch [2/2], Step [232/600], Loss: 0.1031\n",
            "Epoch [2/2], Step [233/600], Loss: 0.2238\n",
            "Epoch [2/2], Step [234/600], Loss: 0.1203\n",
            "Epoch [2/2], Step [235/600], Loss: 0.2264\n",
            "Epoch [2/2], Step [236/600], Loss: 0.1613\n",
            "Epoch [2/2], Step [237/600], Loss: 0.0693\n",
            "Epoch [2/2], Step [238/600], Loss: 0.1591\n",
            "Epoch [2/2], Step [239/600], Loss: 0.0910\n",
            "Epoch [2/2], Step [240/600], Loss: 0.2150\n",
            "Epoch [2/2], Step [241/600], Loss: 0.1887\n",
            "Epoch [2/2], Step [242/600], Loss: 0.1786\n",
            "Epoch [2/2], Step [243/600], Loss: 0.0565\n",
            "Epoch [2/2], Step [244/600], Loss: 0.0666\n",
            "Epoch [2/2], Step [245/600], Loss: 0.2662\n",
            "Epoch [2/2], Step [246/600], Loss: 0.1090\n",
            "Epoch [2/2], Step [247/600], Loss: 0.1834\n",
            "Epoch [2/2], Step [248/600], Loss: 0.1917\n",
            "Epoch [2/2], Step [249/600], Loss: 0.2997\n",
            "Epoch [2/2], Step [250/600], Loss: 0.0641\n",
            "Epoch [2/2], Step [251/600], Loss: 0.1449\n",
            "Epoch [2/2], Step [252/600], Loss: 0.1383\n",
            "Epoch [2/2], Step [253/600], Loss: 0.1239\n",
            "Epoch [2/2], Step [254/600], Loss: 0.1855\n",
            "Epoch [2/2], Step [255/600], Loss: 0.2339\n",
            "Epoch [2/2], Step [256/600], Loss: 0.1222\n",
            "Epoch [2/2], Step [257/600], Loss: 0.1444\n",
            "Epoch [2/2], Step [258/600], Loss: 0.2024\n",
            "Epoch [2/2], Step [259/600], Loss: 0.1325\n",
            "Epoch [2/2], Step [260/600], Loss: 0.1698\n",
            "Epoch [2/2], Step [261/600], Loss: 0.1054\n",
            "Epoch [2/2], Step [262/600], Loss: 0.1000\n",
            "Epoch [2/2], Step [263/600], Loss: 0.2016\n",
            "Epoch [2/2], Step [264/600], Loss: 0.1931\n",
            "Epoch [2/2], Step [265/600], Loss: 0.0972\n",
            "Epoch [2/2], Step [266/600], Loss: 0.1503\n",
            "Epoch [2/2], Step [267/600], Loss: 0.1032\n",
            "Epoch [2/2], Step [268/600], Loss: 0.2333\n",
            "Epoch [2/2], Step [269/600], Loss: 0.1388\n",
            "Epoch [2/2], Step [270/600], Loss: 0.1509\n",
            "Epoch [2/2], Step [271/600], Loss: 0.1190\n",
            "Epoch [2/2], Step [272/600], Loss: 0.1697\n",
            "Epoch [2/2], Step [273/600], Loss: 0.1206\n",
            "Epoch [2/2], Step [274/600], Loss: 0.2296\n",
            "Epoch [2/2], Step [275/600], Loss: 0.1403\n",
            "Epoch [2/2], Step [276/600], Loss: 0.1416\n",
            "Epoch [2/2], Step [277/600], Loss: 0.0623\n",
            "Epoch [2/2], Step [278/600], Loss: 0.1159\n",
            "Epoch [2/2], Step [279/600], Loss: 0.1371\n",
            "Epoch [2/2], Step [280/600], Loss: 0.1201\n",
            "Epoch [2/2], Step [281/600], Loss: 0.0955\n",
            "Epoch [2/2], Step [282/600], Loss: 0.2193\n",
            "Epoch [2/2], Step [283/600], Loss: 0.1722\n",
            "Epoch [2/2], Step [284/600], Loss: 0.1931\n",
            "Epoch [2/2], Step [285/600], Loss: 0.1486\n",
            "Epoch [2/2], Step [286/600], Loss: 0.0532\n",
            "Epoch [2/2], Step [287/600], Loss: 0.1116\n",
            "Epoch [2/2], Step [288/600], Loss: 0.1101\n",
            "Epoch [2/2], Step [289/600], Loss: 0.1371\n",
            "Epoch [2/2], Step [290/600], Loss: 0.1640\n",
            "Epoch [2/2], Step [291/600], Loss: 0.1007\n",
            "Epoch [2/2], Step [292/600], Loss: 0.1631\n",
            "Epoch [2/2], Step [293/600], Loss: 0.1403\n",
            "Epoch [2/2], Step [294/600], Loss: 0.1434\n",
            "Epoch [2/2], Step [295/600], Loss: 0.1014\n",
            "Epoch [2/2], Step [296/600], Loss: 0.0996\n",
            "Epoch [2/2], Step [297/600], Loss: 0.1038\n",
            "Epoch [2/2], Step [298/600], Loss: 0.1306\n",
            "Epoch [2/2], Step [299/600], Loss: 0.1403\n",
            "Epoch [2/2], Step [300/600], Loss: 0.2105\n",
            "Epoch [2/2], Step [301/600], Loss: 0.1318\n",
            "Epoch [2/2], Step [302/600], Loss: 0.1367\n",
            "Epoch [2/2], Step [303/600], Loss: 0.1007\n",
            "Epoch [2/2], Step [304/600], Loss: 0.1345\n",
            "Epoch [2/2], Step [305/600], Loss: 0.1388\n",
            "Epoch [2/2], Step [306/600], Loss: 0.1144\n",
            "Epoch [2/2], Step [307/600], Loss: 0.0972\n",
            "Epoch [2/2], Step [308/600], Loss: 0.0784\n",
            "Epoch [2/2], Step [309/600], Loss: 0.0608\n",
            "Epoch [2/2], Step [310/600], Loss: 0.1079\n",
            "Epoch [2/2], Step [311/600], Loss: 0.0901\n",
            "Epoch [2/2], Step [312/600], Loss: 0.2503\n",
            "Epoch [2/2], Step [313/600], Loss: 0.1193\n",
            "Epoch [2/2], Step [314/600], Loss: 0.1739\n",
            "Epoch [2/2], Step [315/600], Loss: 0.0924\n",
            "Epoch [2/2], Step [316/600], Loss: 0.1204\n",
            "Epoch [2/2], Step [317/600], Loss: 0.1112\n",
            "Epoch [2/2], Step [318/600], Loss: 0.0819\n",
            "Epoch [2/2], Step [319/600], Loss: 0.1670\n",
            "Epoch [2/2], Step [320/600], Loss: 0.0655\n",
            "Epoch [2/2], Step [321/600], Loss: 0.1292\n",
            "Epoch [2/2], Step [322/600], Loss: 0.1558\n",
            "Epoch [2/2], Step [323/600], Loss: 0.1841\n",
            "Epoch [2/2], Step [324/600], Loss: 0.1776\n",
            "Epoch [2/2], Step [325/600], Loss: 0.1759\n",
            "Epoch [2/2], Step [326/600], Loss: 0.1313\n",
            "Epoch [2/2], Step [327/600], Loss: 0.0621\n",
            "Epoch [2/2], Step [328/600], Loss: 0.2040\n",
            "Epoch [2/2], Step [329/600], Loss: 0.0795\n",
            "Epoch [2/2], Step [330/600], Loss: 0.1611\n",
            "Epoch [2/2], Step [331/600], Loss: 0.0816\n",
            "Epoch [2/2], Step [332/600], Loss: 0.2148\n",
            "Epoch [2/2], Step [333/600], Loss: 0.1549\n",
            "Epoch [2/2], Step [334/600], Loss: 0.2279\n",
            "Epoch [2/2], Step [335/600], Loss: 0.1204\n",
            "Epoch [2/2], Step [336/600], Loss: 0.1038\n",
            "Epoch [2/2], Step [337/600], Loss: 0.0945\n",
            "Epoch [2/2], Step [338/600], Loss: 0.1635\n",
            "Epoch [2/2], Step [339/600], Loss: 0.1153\n",
            "Epoch [2/2], Step [340/600], Loss: 0.1196\n",
            "Epoch [2/2], Step [341/600], Loss: 0.2020\n",
            "Epoch [2/2], Step [342/600], Loss: 0.1000\n",
            "Epoch [2/2], Step [343/600], Loss: 0.1541\n",
            "Epoch [2/2], Step [344/600], Loss: 0.1004\n",
            "Epoch [2/2], Step [345/600], Loss: 0.1011\n",
            "Epoch [2/2], Step [346/600], Loss: 0.1183\n",
            "Epoch [2/2], Step [347/600], Loss: 0.0887\n",
            "Epoch [2/2], Step [348/600], Loss: 0.1102\n",
            "Epoch [2/2], Step [349/600], Loss: 0.0874\n",
            "Epoch [2/2], Step [350/600], Loss: 0.1134\n",
            "Epoch [2/2], Step [351/600], Loss: 0.1611\n",
            "Epoch [2/2], Step [352/600], Loss: 0.2087\n",
            "Epoch [2/2], Step [353/600], Loss: 0.0921\n",
            "Epoch [2/2], Step [354/600], Loss: 0.1168\n",
            "Epoch [2/2], Step [355/600], Loss: 0.1273\n",
            "Epoch [2/2], Step [356/600], Loss: 0.1143\n",
            "Epoch [2/2], Step [357/600], Loss: 0.0521\n",
            "Epoch [2/2], Step [358/600], Loss: 0.1350\n",
            "Epoch [2/2], Step [359/600], Loss: 0.0943\n",
            "Epoch [2/2], Step [360/600], Loss: 0.1330\n",
            "Epoch [2/2], Step [361/600], Loss: 0.0551\n",
            "Epoch [2/2], Step [362/600], Loss: 0.1218\n",
            "Epoch [2/2], Step [363/600], Loss: 0.0597\n",
            "Epoch [2/2], Step [364/600], Loss: 0.1345\n",
            "Epoch [2/2], Step [365/600], Loss: 0.1569\n",
            "Epoch [2/2], Step [366/600], Loss: 0.0511\n",
            "Epoch [2/2], Step [367/600], Loss: 0.0937\n",
            "Epoch [2/2], Step [368/600], Loss: 0.1152\n",
            "Epoch [2/2], Step [369/600], Loss: 0.1429\n",
            "Epoch [2/2], Step [370/600], Loss: 0.2011\n",
            "Epoch [2/2], Step [371/600], Loss: 0.1283\n",
            "Epoch [2/2], Step [372/600], Loss: 0.0760\n",
            "Epoch [2/2], Step [373/600], Loss: 0.1814\n",
            "Epoch [2/2], Step [374/600], Loss: 0.1333\n",
            "Epoch [2/2], Step [375/600], Loss: 0.1131\n",
            "Epoch [2/2], Step [376/600], Loss: 0.0651\n",
            "Epoch [2/2], Step [377/600], Loss: 0.2607\n",
            "Epoch [2/2], Step [378/600], Loss: 0.2010\n",
            "Epoch [2/2], Step [379/600], Loss: 0.1797\n",
            "Epoch [2/2], Step [380/600], Loss: 0.1939\n",
            "Epoch [2/2], Step [381/600], Loss: 0.0909\n",
            "Epoch [2/2], Step [382/600], Loss: 0.1283\n",
            "Epoch [2/2], Step [383/600], Loss: 0.1912\n",
            "Epoch [2/2], Step [384/600], Loss: 0.0873\n",
            "Epoch [2/2], Step [385/600], Loss: 0.0948\n",
            "Epoch [2/2], Step [386/600], Loss: 0.1038\n",
            "Epoch [2/2], Step [387/600], Loss: 0.0987\n",
            "Epoch [2/2], Step [388/600], Loss: 0.1295\n",
            "Epoch [2/2], Step [389/600], Loss: 0.2008\n",
            "Epoch [2/2], Step [390/600], Loss: 0.1940\n",
            "Epoch [2/2], Step [391/600], Loss: 0.0878\n",
            "Epoch [2/2], Step [392/600], Loss: 0.2387\n",
            "Epoch [2/2], Step [393/600], Loss: 0.1586\n",
            "Epoch [2/2], Step [394/600], Loss: 0.2238\n",
            "Epoch [2/2], Step [395/600], Loss: 0.1983\n",
            "Epoch [2/2], Step [396/600], Loss: 0.0931\n",
            "Epoch [2/2], Step [397/600], Loss: 0.0562\n",
            "Epoch [2/2], Step [398/600], Loss: 0.1019\n",
            "Epoch [2/2], Step [399/600], Loss: 0.1847\n",
            "Epoch [2/2], Step [400/600], Loss: 0.0976\n",
            "Epoch [2/2], Step [401/600], Loss: 0.1142\n",
            "Epoch [2/2], Step [402/600], Loss: 0.1458\n",
            "Epoch [2/2], Step [403/600], Loss: 0.1653\n",
            "Epoch [2/2], Step [404/600], Loss: 0.1336\n",
            "Epoch [2/2], Step [405/600], Loss: 0.1704\n",
            "Epoch [2/2], Step [406/600], Loss: 0.1342\n",
            "Epoch [2/2], Step [407/600], Loss: 0.1505\n",
            "Epoch [2/2], Step [408/600], Loss: 0.1827\n",
            "Epoch [2/2], Step [409/600], Loss: 0.0752\n",
            "Epoch [2/2], Step [410/600], Loss: 0.1198\n",
            "Epoch [2/2], Step [411/600], Loss: 0.1181\n",
            "Epoch [2/2], Step [412/600], Loss: 0.1505\n",
            "Epoch [2/2], Step [413/600], Loss: 0.0968\n",
            "Epoch [2/2], Step [414/600], Loss: 0.0898\n",
            "Epoch [2/2], Step [415/600], Loss: 0.1795\n",
            "Epoch [2/2], Step [416/600], Loss: 0.1016\n",
            "Epoch [2/2], Step [417/600], Loss: 0.1378\n",
            "Epoch [2/2], Step [418/600], Loss: 0.1899\n",
            "Epoch [2/2], Step [419/600], Loss: 0.1290\n",
            "Epoch [2/2], Step [420/600], Loss: 0.1384\n",
            "Epoch [2/2], Step [421/600], Loss: 0.1413\n",
            "Epoch [2/2], Step [422/600], Loss: 0.1531\n",
            "Epoch [2/2], Step [423/600], Loss: 0.1130\n",
            "Epoch [2/2], Step [424/600], Loss: 0.1028\n",
            "Epoch [2/2], Step [425/600], Loss: 0.1776\n",
            "Epoch [2/2], Step [426/600], Loss: 0.1357\n",
            "Epoch [2/2], Step [427/600], Loss: 0.1308\n",
            "Epoch [2/2], Step [428/600], Loss: 0.1412\n",
            "Epoch [2/2], Step [429/600], Loss: 0.1243\n",
            "Epoch [2/2], Step [430/600], Loss: 0.1282\n",
            "Epoch [2/2], Step [431/600], Loss: 0.1023\n",
            "Epoch [2/2], Step [432/600], Loss: 0.1337\n",
            "Epoch [2/2], Step [433/600], Loss: 0.1406\n",
            "Epoch [2/2], Step [434/600], Loss: 0.1168\n",
            "Epoch [2/2], Step [435/600], Loss: 0.1006\n",
            "Epoch [2/2], Step [436/600], Loss: 0.1606\n",
            "Epoch [2/2], Step [437/600], Loss: 0.1409\n",
            "Epoch [2/2], Step [438/600], Loss: 0.1087\n",
            "Epoch [2/2], Step [439/600], Loss: 0.1927\n",
            "Epoch [2/2], Step [440/600], Loss: 0.1931\n",
            "Epoch [2/2], Step [441/600], Loss: 0.2216\n",
            "Epoch [2/2], Step [442/600], Loss: 0.0834\n",
            "Epoch [2/2], Step [443/600], Loss: 0.1312\n",
            "Epoch [2/2], Step [444/600], Loss: 0.1393\n",
            "Epoch [2/2], Step [445/600], Loss: 0.3049\n",
            "Epoch [2/2], Step [446/600], Loss: 0.0795\n",
            "Epoch [2/2], Step [447/600], Loss: 0.0844\n",
            "Epoch [2/2], Step [448/600], Loss: 0.1895\n",
            "Epoch [2/2], Step [449/600], Loss: 0.1993\n",
            "Epoch [2/2], Step [450/600], Loss: 0.1624\n",
            "Epoch [2/2], Step [451/600], Loss: 0.1180\n",
            "Epoch [2/2], Step [452/600], Loss: 0.2091\n",
            "Epoch [2/2], Step [453/600], Loss: 0.2167\n",
            "Epoch [2/2], Step [454/600], Loss: 0.0689\n",
            "Epoch [2/2], Step [455/600], Loss: 0.0733\n",
            "Epoch [2/2], Step [456/600], Loss: 0.1755\n",
            "Epoch [2/2], Step [457/600], Loss: 0.1765\n",
            "Epoch [2/2], Step [458/600], Loss: 0.1701\n",
            "Epoch [2/2], Step [459/600], Loss: 0.1787\n",
            "Epoch [2/2], Step [460/600], Loss: 0.1354\n",
            "Epoch [2/2], Step [461/600], Loss: 0.0357\n",
            "Epoch [2/2], Step [462/600], Loss: 0.1349\n",
            "Epoch [2/2], Step [463/600], Loss: 0.0650\n",
            "Epoch [2/2], Step [464/600], Loss: 0.0653\n",
            "Epoch [2/2], Step [465/600], Loss: 0.1808\n",
            "Epoch [2/2], Step [466/600], Loss: 0.1724\n",
            "Epoch [2/2], Step [467/600], Loss: 0.1755\n",
            "Epoch [2/2], Step [468/600], Loss: 0.1018\n",
            "Epoch [2/2], Step [469/600], Loss: 0.0803\n",
            "Epoch [2/2], Step [470/600], Loss: 0.1892\n",
            "Epoch [2/2], Step [471/600], Loss: 0.1167\n",
            "Epoch [2/2], Step [472/600], Loss: 0.0364\n",
            "Epoch [2/2], Step [473/600], Loss: 0.1423\n",
            "Epoch [2/2], Step [474/600], Loss: 0.1462\n",
            "Epoch [2/2], Step [475/600], Loss: 0.1956\n",
            "Epoch [2/2], Step [476/600], Loss: 0.1791\n",
            "Epoch [2/2], Step [477/600], Loss: 0.0695\n",
            "Epoch [2/2], Step [478/600], Loss: 0.1527\n",
            "Epoch [2/2], Step [479/600], Loss: 0.1692\n",
            "Epoch [2/2], Step [480/600], Loss: 0.1609\n",
            "Epoch [2/2], Step [481/600], Loss: 0.1463\n",
            "Epoch [2/2], Step [482/600], Loss: 0.0941\n",
            "Epoch [2/2], Step [483/600], Loss: 0.0835\n",
            "Epoch [2/2], Step [484/600], Loss: 0.1277\n",
            "Epoch [2/2], Step [485/600], Loss: 0.0485\n",
            "Epoch [2/2], Step [486/600], Loss: 0.1906\n",
            "Epoch [2/2], Step [487/600], Loss: 0.0912\n",
            "Epoch [2/2], Step [488/600], Loss: 0.0892\n",
            "Epoch [2/2], Step [489/600], Loss: 0.0931\n",
            "Epoch [2/2], Step [490/600], Loss: 0.0889\n",
            "Epoch [2/2], Step [491/600], Loss: 0.2131\n",
            "Epoch [2/2], Step [492/600], Loss: 0.1025\n",
            "Epoch [2/2], Step [493/600], Loss: 0.1882\n",
            "Epoch [2/2], Step [494/600], Loss: 0.1091\n",
            "Epoch [2/2], Step [495/600], Loss: 0.1328\n",
            "Epoch [2/2], Step [496/600], Loss: 0.1728\n",
            "Epoch [2/2], Step [497/600], Loss: 0.0978\n",
            "Epoch [2/2], Step [498/600], Loss: 0.1298\n",
            "Epoch [2/2], Step [499/600], Loss: 0.1157\n",
            "Epoch [2/2], Step [500/600], Loss: 0.0960\n",
            "Epoch [2/2], Step [501/600], Loss: 0.0382\n",
            "Epoch [2/2], Step [502/600], Loss: 0.1482\n",
            "Epoch [2/2], Step [503/600], Loss: 0.0525\n",
            "Epoch [2/2], Step [504/600], Loss: 0.2347\n",
            "Epoch [2/2], Step [505/600], Loss: 0.1059\n",
            "Epoch [2/2], Step [506/600], Loss: 0.1228\n",
            "Epoch [2/2], Step [507/600], Loss: 0.0525\n",
            "Epoch [2/2], Step [508/600], Loss: 0.0777\n",
            "Epoch [2/2], Step [509/600], Loss: 0.1153\n",
            "Epoch [2/2], Step [510/600], Loss: 0.1064\n",
            "Epoch [2/2], Step [511/600], Loss: 0.1200\n",
            "Epoch [2/2], Step [512/600], Loss: 0.1264\n",
            "Epoch [2/2], Step [513/600], Loss: 0.1077\n",
            "Epoch [2/2], Step [514/600], Loss: 0.1134\n",
            "Epoch [2/2], Step [515/600], Loss: 0.0467\n",
            "Epoch [2/2], Step [516/600], Loss: 0.1743\n",
            "Epoch [2/2], Step [517/600], Loss: 0.1097\n",
            "Epoch [2/2], Step [518/600], Loss: 0.1235\n",
            "Epoch [2/2], Step [519/600], Loss: 0.1847\n",
            "Epoch [2/2], Step [520/600], Loss: 0.0961\n",
            "Epoch [2/2], Step [521/600], Loss: 0.1456\n",
            "Epoch [2/2], Step [522/600], Loss: 0.1456\n",
            "Epoch [2/2], Step [523/600], Loss: 0.1590\n",
            "Epoch [2/2], Step [524/600], Loss: 0.1047\n",
            "Epoch [2/2], Step [525/600], Loss: 0.1232\n",
            "Epoch [2/2], Step [526/600], Loss: 0.1224\n",
            "Epoch [2/2], Step [527/600], Loss: 0.1110\n",
            "Epoch [2/2], Step [528/600], Loss: 0.1298\n",
            "Epoch [2/2], Step [529/600], Loss: 0.1853\n",
            "Epoch [2/2], Step [530/600], Loss: 0.0936\n",
            "Epoch [2/2], Step [531/600], Loss: 0.1157\n",
            "Epoch [2/2], Step [532/600], Loss: 0.1015\n",
            "Epoch [2/2], Step [533/600], Loss: 0.0923\n",
            "Epoch [2/2], Step [534/600], Loss: 0.1299\n",
            "Epoch [2/2], Step [535/600], Loss: 0.1657\n",
            "Epoch [2/2], Step [536/600], Loss: 0.1651\n",
            "Epoch [2/2], Step [537/600], Loss: 0.1958\n",
            "Epoch [2/2], Step [538/600], Loss: 0.1253\n",
            "Epoch [2/2], Step [539/600], Loss: 0.1709\n",
            "Epoch [2/2], Step [540/600], Loss: 0.1999\n",
            "Epoch [2/2], Step [541/600], Loss: 0.1970\n",
            "Epoch [2/2], Step [542/600], Loss: 0.0991\n",
            "Epoch [2/2], Step [543/600], Loss: 0.1796\n",
            "Epoch [2/2], Step [544/600], Loss: 0.1260\n",
            "Epoch [2/2], Step [545/600], Loss: 0.1111\n",
            "Epoch [2/2], Step [546/600], Loss: 0.1504\n",
            "Epoch [2/2], Step [547/600], Loss: 0.0933\n",
            "Epoch [2/2], Step [548/600], Loss: 0.1126\n",
            "Epoch [2/2], Step [549/600], Loss: 0.2539\n",
            "Epoch [2/2], Step [550/600], Loss: 0.2485\n",
            "Epoch [2/2], Step [551/600], Loss: 0.1014\n",
            "Epoch [2/2], Step [552/600], Loss: 0.1250\n",
            "Epoch [2/2], Step [553/600], Loss: 0.0755\n",
            "Epoch [2/2], Step [554/600], Loss: 0.1104\n",
            "Epoch [2/2], Step [555/600], Loss: 0.1854\n",
            "Epoch [2/2], Step [556/600], Loss: 0.1182\n",
            "Epoch [2/2], Step [557/600], Loss: 0.1551\n",
            "Epoch [2/2], Step [558/600], Loss: 0.1362\n",
            "Epoch [2/2], Step [559/600], Loss: 0.1680\n",
            "Epoch [2/2], Step [560/600], Loss: 0.1412\n",
            "Epoch [2/2], Step [561/600], Loss: 0.0862\n",
            "Epoch [2/2], Step [562/600], Loss: 0.1898\n",
            "Epoch [2/2], Step [563/600], Loss: 0.0667\n",
            "Epoch [2/2], Step [564/600], Loss: 0.1550\n",
            "Epoch [2/2], Step [565/600], Loss: 0.0803\n",
            "Epoch [2/2], Step [566/600], Loss: 0.1438\n",
            "Epoch [2/2], Step [567/600], Loss: 0.0976\n",
            "Epoch [2/2], Step [568/600], Loss: 0.1427\n",
            "Epoch [2/2], Step [569/600], Loss: 0.0948\n",
            "Epoch [2/2], Step [570/600], Loss: 0.0531\n",
            "Epoch [2/2], Step [571/600], Loss: 0.0659\n",
            "Epoch [2/2], Step [572/600], Loss: 0.0649\n",
            "Epoch [2/2], Step [573/600], Loss: 0.2147\n",
            "Epoch [2/2], Step [574/600], Loss: 0.0692\n",
            "Epoch [2/2], Step [575/600], Loss: 0.0885\n",
            "Epoch [2/2], Step [576/600], Loss: 0.1017\n",
            "Epoch [2/2], Step [577/600], Loss: 0.1510\n",
            "Epoch [2/2], Step [578/600], Loss: 0.1545\n",
            "Epoch [2/2], Step [579/600], Loss: 0.0911\n",
            "Epoch [2/2], Step [580/600], Loss: 0.0835\n",
            "Epoch [2/2], Step [581/600], Loss: 0.1616\n",
            "Epoch [2/2], Step [582/600], Loss: 0.1469\n",
            "Epoch [2/2], Step [583/600], Loss: 0.1114\n",
            "Epoch [2/2], Step [584/600], Loss: 0.1633\n",
            "Epoch [2/2], Step [585/600], Loss: 0.1119\n",
            "Epoch [2/2], Step [586/600], Loss: 0.1042\n",
            "Epoch [2/2], Step [587/600], Loss: 0.0779\n",
            "Epoch [2/2], Step [588/600], Loss: 0.1099\n",
            "Epoch [2/2], Step [589/600], Loss: 0.0555\n",
            "Epoch [2/2], Step [590/600], Loss: 0.1406\n",
            "Epoch [2/2], Step [591/600], Loss: 0.0722\n",
            "Epoch [2/2], Step [592/600], Loss: 0.2299\n",
            "Epoch [2/2], Step [593/600], Loss: 0.2386\n",
            "Epoch [2/2], Step [594/600], Loss: 0.1642\n",
            "Epoch [2/2], Step [595/600], Loss: 0.1149\n",
            "Epoch [2/2], Step [596/600], Loss: 0.1112\n",
            "Epoch [2/2], Step [597/600], Loss: 0.1987\n",
            "Epoch [2/2], Step [598/600], Loss: 0.0892\n",
            "Epoch [2/2], Step [599/600], Loss: 0.0556\n",
            "Epoch [2/2], Step [600/600], Loss: 0.0657\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cgVh_4jdpY2O",
        "outputId": "9c171857-795b-4218-ed2a-3acb1338b2d8"
      },
      "source": [
        "with torch.no_grad():\n",
        "  n_correct = 0\n",
        "  n_samples = 0\n",
        "  for images, labels in test_loader:\n",
        "    images = images.reshape(-1, 28*28).to(device)\n",
        "    labels = labels.to(device)\n",
        "    outputs = model_batchnorm(images)\n",
        "    _, predicted = torch.max(outputs.data, 1)  #1 indicates return max of row # predicted will store index of the label having highest probability\n",
        "    n_samples += labels.size(0)  \n",
        "    n_correct += (predicted==labels).sum().item()\n",
        "\n",
        "  acc = 100.0 * n_correct / n_samples\n",
        "  print(f'Accuracy of the network on the 10000 test images: {acc} %')\n"
      ],
      "execution_count": 67,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy of the network on the 10000 test images: 96.31 %\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YZ-skacIzFcw"
      },
      "source": [
        "#Accuracy of the network on the 10000 test images without batchnorm: 95.42 %"
      ],
      "execution_count": 68,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VxvoX7IqWazV"
      },
      "source": [
        ""
      ],
      "execution_count": 68,
      "outputs": []
    }
  ]
}