Basic_concept_5_TensorBoard.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/anaconda3/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/anaconda3/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/anaconda3/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/anaconda3/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/anaconda3/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "WARNING:root:This caffe2 python run does not have GPU support. Will run in CPU only mode.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn import MSELoss\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "# import\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "import torch.optim as optim\n",
    "from tqdm import tqdm, trange\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Outlines\n",
    "- Tensorboard\n",
    "    - Import package\n",
    "        - `torch.utils.tensorboard.SummaryWriter`\n",
    "        - `tb_writer.add_scalar`\n",
    "        - `tb_writer.add_histogram`\n",
    "    - Start process\n",
    "        - `tensorboard --logdir=runs` where `runs` is a folder created automatically in current directory\n",
    "    - Open interface\n",
    "        - `http://localhost:6006`\n",
    "    - Usage\n",
    "        - \n",
    "- Device by `torch.device`\n",
    "- [Gradient accumulation](https://www.zhihu.com/question/303070254)\n",
    "- `TensorDataset`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Simple regression model from `Basic_project_1_regression`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dummy training data\n",
    "n_samples = 100\n",
    "x_dummy = np.linspace(-3, 3, n_samples)\n",
    "\n",
    "# cos() function\n",
    "y_dummy = np.cos(x_dummy) + np.random.uniform(-3, 3, n_samples)\n",
    "\n",
    "x = torch.tensor(x_dummy).float().view(n_samples, 1)\n",
    "y = torch.tensor(y_dummy).float().view(n_samples, 1)\n",
    "\n",
    "src_data = TensorDataset(x, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RegressionNet(\n",
       "  (hidden): Linear(in_features=1, out_features=10, bias=True)\n",
       "  (output): Linear(in_features=10, out_features=1, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class RegressionNet(nn.Module):\n",
    "    def __init__(self, in_features):\n",
    "        super(RegressionNet, self).__init__()\n",
    "        self.hidden = nn.Linear(in_features, 10)\n",
    "        self.output = nn.Linear(10, 1)\n",
    "    \n",
    "    def forward(self, x, y=None):\n",
    "        x = F.relu(self.hidden(x))\n",
    "        logits = self.output(x)\n",
    "        outputs = (logits,)\n",
    "        if y is not None:\n",
    "            criterion = MSELoss()\n",
    "            loss = criterion(logits, y)\n",
    "            outputs = outputs + (loss,)\n",
    "        return outputs\n",
    "\n",
    "\n",
    "# model\n",
    "# if gpu is available\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "model = RegressionNet(1)\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train and predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch:   0%|          | 0/30 [00:00<?, ?it/s]\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Iteration: 100%|██████████| 2/2 [00:00<00:00, 147.50it/s]\u001b[A\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Iteration: 100%|██████████| 2/2 [00:00<00:00, 415.03it/s]\u001b[A\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Iteration: 100%|██████████| 2/2 [00:00<00:00, 367.99it/s]\u001b[A\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch:  13%|█▎        | 4/30 [00:00<00:00, 31.19it/s]t/s]\u001b[A\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Iteration: 100%|██████████| 2/2 [00:00<00:00, 242.99it/s]\u001b[A\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Iteration: 100%|██████████| 2/2 [00:00<00:00, 245.28it/s]\u001b[A\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch:  23%|██▎       | 7/30 [00:00<00:00, 30.21it/s]t/s]\u001b[A\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Iteration: 100%|██████████| 2/2 [00:00<00:00, 397.77it/s]\u001b[A\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Iteration: 100%|██████████| 2/2 [00:00<00:00, 355.95it/s]\u001b[A\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Iteration: 100%|██████████| 2/2 [00:00<00:00, 342.94it/s]\u001b[A\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch:  37%|███▋      | 11/30 [00:00<00:00, 30.94it/s]/s]\u001b[A\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Iteration: 100%|██████████| 2/2 [00:00<00:00, 183.98it/s]\u001b[A\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Iteration: 100%|██████████| 2/2 [00:00<00:00, 215.01it/s]\u001b[A\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Iteration: 100%|██████████| 2/2 [00:00<00:00, 282.65it/s]\u001b[A\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch:  50%|█████     | 15/30 [00:00<00:00, 31.09it/s]/s]\u001b[A\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Iteration: 100%|██████████| 2/2 [00:00<00:00, 270.35it/s]\u001b[A\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Iteration: 100%|██████████| 2/2 [00:00<00:00, 364.37it/s]\u001b[A\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Iteration: 100%|██████████| 2/2 [00:00<00:00, 378.22it/s]\u001b[A\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch:  63%|██████▎   | 19/30 [00:00<00:00, 31.81it/s]/s]\u001b[A\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Iteration: 100%|██████████| 2/2 [00:00<00:00, 302.71it/s]\u001b[A\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Iteration: 100%|██████████| 2/2 [00:00<00:00, 279.53it/s]\u001b[A\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch:  73%|███████▎  | 22/30 [00:00<00:00, 30.59it/s]/s]\u001b[A\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Iteration: 100%|██████████| 2/2 [00:00<00:00, 272.83it/s]\u001b[A\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Iteration: 100%|██████████| 2/2 [00:00<00:00, 406.35it/s]\u001b[A\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch:  83%|████████▎ | 25/30 [00:00<00:00, 30.27it/s]/s]\u001b[A\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Iteration: 100%|██████████| 2/2 [00:00<00:00, 265.68it/s]\u001b[A\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Iteration: 100%|██████████| 2/2 [00:00<00:00, 338.41it/s]\u001b[A\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch:  93%|█████████▎| 28/30 [00:00<00:00, 29.67it/s]/s]\u001b[A\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Iteration: 100%|██████████| 2/2 [00:00<00:00, 209.44it/s]\u001b[A\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch: 100%|██████████| 30/30 [00:00<00:00, 30.47it/s]/s]\u001b[A\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD8CAYAAABjAo9vAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xt0VuWdL/DvjxAgChKQgCEQLmpBRIdLDuLEWsULVK1SvJS00zNdXS2nZ53OtD0zdrCu005npiNdzpp21px2TT2jM9M1NugUhFbroBYvhSnWcPPCxQoaJBBBNComSBJ+548n4c0b3v3e9rMvz97fz1rvgiQ7+907735/77N/z+95HlFVEBFRcgyJ+gCIiMguBnYiooRhYCciShgGdiKihGFgJyJKGAZ2IqKEsRbYRaRCRLaLyKO29klERKWz2WL/GoDdFvdHRERlsBLYRWQSgBsB/LON/RERUfmGWtrPDwF8E8CoYjYeN26cTp061dJTExGlw9atW99W1ZpC2/kO7CJyE4AjqrpVRK7Ks90KACsAoL6+Hi0tLX6fmogoVUSktZjtbKRiGgHcLCJvAFgNYJGI/PvgjVT1PlVtUNWGmpqCHzhERFQm34FdVe9S1UmqOhXAcgAbVfWPfB8ZERGVhXXsREQJY6vzFACgqs8AeMbmPomIqDRssRMRJQwDOxFRwlhNxRC5bt32Nty7YS8OdXRhYnUV7lw8A0vn1kV9WEQlYWAn6rNuexvuWvsSurp7AQBtHV24a+1LAMDgTk5hKoaoz70b9p4O6v26untx74a9ER0RUXkY2In6HOroKun7RHHFVAxRn4nVVWjLEcQnVldFcDS5sQ+AisEWO1GfOxfPQFVlRdb3qiorcOfiGREdUbb+PoC2ji4oMn0A67a3RX1oFDMM7ER9ls6twz3LLkFddRUEQF11Fe5ZdklsWsTsA6BiMRVDNMDSuXWxCeSDsQ+AisUWO5EjvHL9ceoDoHhgYCdyRNz7ACg+mIohckR/iohVMVQIAzuRQ+LcB0DxwVQMEVHCsMVO5CgOViIvDOxEIbIVjDlhGeXjO7CLyAgAzwEY3re/n6vqd/zul8hluQI4AGvBON9gJQZ2stFi/wjAIlU9LiKVADaJyOOqusXCvomc49WaHlE5xFow5mAlysd356kax/u+rOx7qN/9ErnKqzX9bmd3zu3LCcYcrET5WKmKEZEKEdkB4AiAJ1X1eRv7JXJRqYG6nGBczmClddvb0LhqI6atfAyNqzZy8rAEsxLYVbVXVecAmARggYjMHryNiKwQkRYRaTl69KiNpyWKJa9AXV1VaW3kaKkTlnFmyHQRVbtZExH5NoBOVf07r20aGhq0paXF6vMSFRJWeeDgHDtgAvg9yy4BEM3I0cZVG3PONV9XXYXNKxcF/vxkh4hsVdWGQtvZqIqpAdCtqh0iUgXgOgDf97tfIpvKKQ8s94Og0ND/KKpW2NmaLjaqYmoB/JuIVMCkdh5W1Uct7JfImlLLA/3Wicdt6L8Lq0ORPTaqYl5U1bmqeqmqzlbVv7JxYEQ2ldpiTdqiFpwZMl04VwylQqnlgUlLXcR9dSiyi1MKUCrcuXhGzg5NrxZrElMXcUsPUXDYYqdUKLXFGkbqgnXlFBS22Ck1SmmxBr2oBSfxoiAxsBN5CDJ1wUm8KEhMxRBFIGmdsxQvDOxEEeAkXhQkBnaiCLCunILEHDtRBILunKV0Y2AnigjryikoTMUQESUMW+xEMRHWtMKUfAzsRDHAAUtkE1MxRDGQtNkkKVoM7EQxwAFLZBNTMRQZ5pQzkjibJEWHLXaKRBIXV/YzWyMHLJFNNtY8nQzgpwAmAFAA96nqP/jdLxXPxZZv0ibBsrGUHsABS0Fy8X1SLhupmB4Af6aq20RkFICtIvKkqu6ysG8qwNVqimJzyq68GW18UHHAUnBcfZ+Uy8aap4dVdVvf/z8AsBtA8v5SMeVaNUV/ukI9fj4wp+xSuoadn/Hm2vvEL6s5dhGZCmAugOdt7pe8uRRQBgbqXAbnlF16M3K2xnhz6X1ig7XALiIjAawB8HVVfT/Hz1eISIuItBw9etTW06aeSwElV6Dul2upunxvxrgtK8fOz3gr9D6J2/Xkl5XALiKVMEH9QVVdm2sbVb1PVRtUtaGmpsbG0xLcCihegVoAbF656Ixcp9ebcXRVZexSNKWuqUrhyvc+cSnlVywbVTEC4H4Au1X17/0fEpUiyGoK2x2XpdZq37l4RlaHF2DejCKIZUUNOz/jK9/7pHHVxlheT37YqIppBPB5AC+JyI6+731LVX9lYd9UhCACShBVBF6B2uvuwuvN+I2HduTcPqn5UrLD632SxPy778Cuqptg7qYpQcop3yvUwi/n7iLXm/HeDXs5SjNGXClJ9ZLEUb+i6lV4FpyGhgZtaWkJ/XmpeNNWPuZZkliX4807uIUPmNZ4EHnmXM8lMKPjch0bBSfM1z0oLp2DiGxV1YaC2zGwUy6NqzZ6liUCZ174XtvXVVdh88pF1o+vv5XY1tF1Oqj3qxwiGDliKDo6u51sQcaRV6s87Nc9KK7cdRQb2DkJGOWUKx8+0OC0TNh5yv4UTa7A0n1K8W5nN4DkjzAMQ77+lqTkp5PW8c1JwCingeV7Xga+eaOqpy8mgMR1UJMr8vW3uDSOIk0Y2MnT0rl12LxykWdwH/jmjaqevtgA4loLMk7ytcpdGkeRJgzsVFAxb96oBujkOrZc2IIsX75WOQdmxRNz7FRQsWWKUeQpBx/b6KpKfHiyB929me5UtiD9KTT+IGn56SRgVQwljisVDi7h3zQeWBVDqVVqC5JBKzf+XdzFwE6plrYFGIrFv4vb2HlKqebSnO9h4t/FbQzslGpJGWBjG/8ubmNgp1TjAJvc+Hdxm7OBPWkrnlA0OMAmN/5d3OZk5yk7dsiWIBcqcRn/Lm5zso49KTPKUWlYfkdpl+g6dpc7dhicysO7NKLi2VrM+gEROSIiL9vYXyGuduwkcdHcsLD8Ljjsr0oeW52n/wpgiaV9FeRqxw6DU/lcvkuLMzY2kslKYFfV5wC8Y2NfxXB1RjkGp/K5epcWd2xsJJOTOXbAzRnlkrhoblgKzTBI5WFjI5lCC+wisgLACgCor68P62kDVWpHaL7gxE7V/Fh+Fww2NpLJWrmjiEwF8Kiqzi60bRKm7S13ZfNcARyAM6ukU7KUex1TRpiNskSXO8ZBvtxkvhc1VwqpcdXGsvblCt6NxBfvhPyJaxmulcAuIs0ArgIwTkQOAviOqt5vY99xZTM3meQ8Z1wvfMpwsb8qLspt4AXNVlVMk6rWqmqlqk5KelAH7FZpJLnig1UXlGRxbZQ5OwlY1GzW0rtal1+MuF74RDbEtVHGwF4mm7X0rtblFyOuF74tHLWZbnFtlDk5CRi5I8lVF0k+NyqeV3FAEEUDxVbFMLBT4JJWFdN/PrnqvwHOMkrBfeiz3JFiI0lVF7nesIOx/4CirpZhYKcsSWtd25brDTtYUvoPqHxRFw2w85RO40x/hRV6Y8ah44yiF3XRAAM7ncaa88LyvTGTVM1E/kRdLcNUDJ0W9e2jC7wmcmNAp4GinqqBgZ1O40x/hUX9hk2jMMsJbYqyaIDljnQa67IpbryuyVvn12HN1rbUXassdyxS3D/1w8TWKMWNV79P8/NvondQozQOk2/FRaoDO2cePFOSas7JfV79O4ODeqHt0ybVVTGsAiGKN6/+nQqRkrZPm1QHdlaBEMWbV9lg02WTYzn5VlykOhWTpioQ9iWQi/L1+zRMGctr2kOqq2LSUgWSlvN03smTwJEjwFtvmceRI8DbbwPHjpnHiRNmm54e4JxzgPHjgQkTgI99DJg3D6itjfoMKGChVsWIyBIA/wCgAsA/q+oqG/sNWlqqQKKekCjVjh8/M1j3/3/w9zo6/D3XhAnA5ZcD114LXHMNMGMG4JGLpmTzHdhFpALAjwBcB+AggBdE5BequsvvvsOQhioQ9iVYpmoC8YEDwKFD5nH4cO5g3dlZ/H4rKoCaGhOgJ0wwLfKaGuDcc4GxY4GzzgKGDQOGDgXee8/sv70dePllYNs28/W6deYBAJMmAddfbx7XXAOMGxfM34Nix0aLfQGA11R1PwCIyGoAtwBwIrCnQZr6Eqw5edIE7n37sh/795vHhx8Wt58RIzJBemDA7v//wK/HjgWGmHqGrD6RiircubDAneSpU+b4nn0W+PWvzePgQeCBB8xDBJg/PxPoL7/cfEhQItkI7HUA3hzw9UEAl1nYL1niNb9JXCsIwujoXbe9DT9evxWVrW9gzslj+Oy4blx84lgmgB84YIKllzFjgKlTgbo686itzR3AR40qOR1S1viKIUOACy80jy99yRz7iy8CTzwBbNgAbNoEtLSYx9/+LXD22cDVVwPXXQcsXmzy9EzbJIbvzlMRuQ3AElX9Ut/Xnwdwmap+ddB2KwCsAID6+vr5ra2tvp6XSuNKVYzVjt5Tp0yKZGBre98+vPPSbsi+/RjT9b7374oAkycD558PTJ9u/h34/zFjyjzDwhpXbcy7OlNZr11nJ/DccybIP/kk8Mor2T+vr89O24wdW+bRF+bKtRhHoS2NJyKXA/hLVV3c9/VdAKCq93j9TlyqYih+vIKa53JzH30EvP76GcEb+/aZ75844flcJ4YOw4HR56F1zHk4UF2L92rr8b//5ydN8J46FRg+3OKZFW/ayseQ711ppaKprc0E+CeeMP++/XbmZyLAggXAsmXArbeaDzJLWKHlT5iBfSiAVwFcA6ANwAsAPquqr3j9DgM7eckV1EZ3fYCpHYex/pO12cF7/36TR853DdfUZLe4zz8ftz/5FlpHn4cjI8dmpR8EwOurbgzkvEqRr8Xez+q6qqdOAdu3ZwL95s2mj6HfnDnA8uXmMWWKr6cq+YObsoRW7qiqPSLyVQAbYModH8gX1CkdSrrd7u01AXr/fvyP32/EOYcOoP7ddkzpOIz6jnaM/qivo/KnOX63osIEm1zpkunTTb33IIcOb8SRGHcm5+oTGcxqRdOQIaZjdf58YOVK0zH8xBPAz38O/PKXwI4d5rFypel0bWoC7rjD9COUiBVa4Uj1ACUKRq7b7THoxt/NH4Vrhn94ZqXJG28A3d2e+/twWBV6pkzF6Nkzz8x519cDlZW+jy9u6YD+D8Z8ufZQWrgnTpi8fHMz8ItfAF19xzNkCLBokWnFL1tWdJ8DW+z+hJaKKQcDu9tytsbnTASOHgX27cN3f/hLjDp0AFM6DmNyRzumdLRjwvF38u/0vPNOB+vdZ9Xg4XcqsXPYufhoyjR8edllWDpvUvDnEJOgPlCsPoSOHzfBffVq4D//M/NhPGwYsGSJacl/6lOm4sZDrM7HQQzsFIh1LQfwwP97DLNad2Hau4dQ39GOqe+144IP3kJlp3dt98khQ9E2ejymXXbpma3uadPyBoO0i+WH0DvvAGvXmiD/9NOZ0tCzzgJuvtkE+cWLc3ZAx/J8HMHATnYcPw48/7zpUNu8Gcef3YyRH3kE8NGjgenTsbF7FPaePR6t1eehdUwtDlTX4vCoc1E7diRvt5OovR14+GGTrtmyJfP96mqTpmlqMjXzFRXe+6CiMLCTp7wtpra200EcmzebTrPe7E68g+eMx7a6mdg7bgoO9AXv1upajDyvBncumQkAvN1Oq9dfBx56yAT5F1/MfH/CBNPhuny56YDlYKiyMLAnXLm3swNznKKn8LG3D6Dh4C7Mb9uNhYd2Y+K77dm/UFEBzJ0LNDYCjY24Zdsp7NSRnvvvD+BA8idXowJ27TIBfvVq4LXXMt+fMsUE+KYm4NJLGeRLwMCeELkCOFBmi7izE//r6z/B1D3b0dC2C/Pa9mRKCfscH1aFD+ctwIRPLgKuuMIMVBmZCeS5Or8GY4VDttTnlFWBrVtNkH/oIXNX2G/mTBPgm5rMdAgxE7fXjoE9AbwqCEZUDsG7nWeWBw4OqI8/tQO/+df1mP7qDlzevhcXHf49hvT0ZP1O26gatEyahZZJF2Fr3SzsqZlSMBdeqBQvLgN94oBVIIOcOmXmrWluBv7jP8w88/3mzTMB/jOfMdM5RCyOrx0DewIUMwLxNFVccOwgnlo41HRy/voZjHzzjaxNemUIXpswFVsmmiDeMukiHDpn/Bm7KjYwsya5MP6N8ujuNrNQNjcDjzwCfPBB5mcf/7gJ8rfdZkYPRyCOr12oC21QMPKNxhvecxKz219DQ9uuvhz5Hozteh+43/x8JIDOyuHYPnEGWupmoWXSLGyfOBNDq0fjo55TeVMpxY7AdG3WyChwpGUelZWm/n3JEuCf/gl4/HET5B99FPjNb8zjT/7ELBzS1AQsXWoqr0Li8mvHwB5jA+dRr+56H/PbduO/HdyFyw7twcWH9mJYb3ZapWvcBFRdfSXQ2Iibtyl2jZ+Gnorsl1i6uvGDz8w5nUoRIGtullICswsrUEWdI+Vc+EWqqjKlkcuWmZb7+vUmyPdPO7xhg6mJv+EGE+Rvusn8ToBcfu2YiokjVWDfPmz92aPY/8gGzHnzFVx47M0zNts3YRq21M7Aax+bg8s/fzOuv3Hh6QqDYm8jow58QYpDjjQOx+C0Y8eANWtMkH/22cyEbyNHmhb88uVmquESp5UoRhxfO+bYXdLdbWbX27QpUz/+1ltZm5wYOgy7J8/EWVd/AjNuXWJqgfPMzxHHizJsccmRJvnDM1SHDmUGQv3ud5nvjx1rcvHLlwNXXml1IFTcXjsG9jjr6AB++1sTwDdtMhdp16AAVFNjasevuML8O29eyUuZxe2iDJvXvOas2kmA/ftNfXxzs1nztV9tramqWb7clOomrEaegT0uVIHW1kwQ37zZXIiD/+4zZ54eBIQrrgAuuCBxF2XY4tJip4C9/HImyO/fn/n+9OmZgVCzZ0d3fBYxsEelpwfYuTOTUtm0ydxCDlRZCTQ0ZFrjf/iHkZV0JRnTUSmjCrzwQmYg1OHDmZ/Nnp0J8tOnR3eMPjGwh+WDD8zER/1BfMuWM1ewHzvWBO/+1nhDg1m9ngKX9nRU0hT9evb2mnLJ5mazYMg7A6aNXrDABPk77jALkTuEgT0oBw9mp1V27jxzNfvzz89Oq8ycaRYmIKKylX0H1t1tlv1rbgbWrTMzlgIm1XnllSbI33YbMG5cwGfgXyiBXURuB/CXAC4CsEBVi4rWzgT23l6TvxuYVjlwIHuboUPNJFn9aZXGRrNoBBFZZaXPpLMTeOwxk5N/7DGzGDpg3sfXXZcZCDVqlMUjtyeskacvA1gG4Cc+9xMPH35oKlT6g/hvfwu8/372Nueck51WWbDALC5ARIGyMhL0rLOA2283j/feMy341atNi/7xx81jxAjgxhtNkL/hhsAHQgXBV2BX1d0AIK5Wb7S3Z889vm2b6fwcaMqU7LLDiy/mggFEEbA+EnT0aOCP/9g8jh41ufjmZpObX7PGPEaNMi34piYztUEAA6GCYCXHLiLPAPjzfKkYEVkBYAUA1NfXz29tbfX9vCU5dQrYsyc7rbJvX/Y2Q4YAf/AH2YF8kt21NomoPKFVOb35pqmqWb3aTDfcb9w4k4tvajLxIYJ+M2s5dhF5CkCupPHdqrq+b5tnUCCwDxRKjv3ECaClJdPJ+V//ld0zDphhyQsXZnLjCxfGNrdGFIa4VxGFfnyvvmoC/OrVwO7dme/X1ZmBUE1NwPz5oY05CbUqJurAvm57G+5b+ztM2rUNn3j7VSx5bz/O3f0icPJk9oZ1ddmt8UsvNZ0mRMS6/3xUzVJ//StCDcw4XHBBpkZ+1qxADyPZgV0V+P3vgc2b0bp+A3o3bcb0YwezNxGBzJ6daY1//ONAfT1HcxJ54EjdIqma8SrNzWbumoHzOl16qQnyy5cD06ZZf+qwyh0/DeAfAdQA6ACwQ1UXF/q9sgP7k08CP/6xSa0cPZr1o66hw7Gz9kK8MOlibK27CIdnzcWG795c+nM4Lu630hRfnFunDD09ZtbJ5mbT2drRkfnZwoWmFX/HHdZKoEMpd1TVRwA84mcfJWlvN+VJADB+PNDYiL/pGIsXJs3CrgnT0V2R6bGWj0I7qtgYfCvd1tGFu9a+BAAM7j6l4QPT5fnHbSrptR46FLjmGvP40Y/M/PHNzWY++S1bzOMb3wCuusoE+VtvzTsrqy1uDYe89lrgX/7FpGHa24G1a/H49Z/FzokzsoI6kL6LETALXgxeGamruxf3btgb0RElQ/8HZltHFxSZD8x129sK/q5L7lw8A1WV2aW8aVsRy9drPXw48KlPAT/7GXDkiAnwN99sgv/GjcCXvwxMmADs2BH4ebgV2GtrgS98IWvmQ16MGS4v5RVnafnAXDq3DvcsuwR11VUQmNx62jpOrb3WZ59t8uzr15tG6P33m4ZpTQ1wySUWjzg350tCXFieLSy8lQ5Gmj4wl86tS+V7p18gr/WYMcAXv2geXV2hDHB0PrADvBj7cXHpYPADMz0Cf61Dmp7ArVQM5cVb6WAw3ZceSXmtE9FipwzevdjnWrovDRU8QXHttfbC+dhDwDcahYWjR5Ot2Dp2pmIClpZSOYqHtFTwUH5MxRTBT4s73xuNLSiyLU0VPOSNLfYC/La4+UajMHlVb7CCJ10Y2Avwe2vLNxqFKSlVHeQPUzEF+G1xl1Nbzs5WKtXAa2Z0VSVGVA5BR2e3U9cPr3t7GNgL8DtgodTyKU7kRaUafM10dHWjqrICP/jMnLzXWRBB1Gu/hZ6P171dLHccINfFByDU8jHOiU2lynfN3Ll4RmjXtFep5a3z67Bma1ve5+N1XxyWO5bIq5MUQKijOdnZSqXyujb6r+HB1/R3f/lKICWRXv1Rzc+/WfD5eN3bxVRMn3ydpJtXLgrtdpDzklCpvK6ZCpGc1/Tg7/XzG0S9fr/XIyswcHte93axxd7Hdoth3fY2NK7aiGkrH0Pjqo1Fl0eyqoFK5XXNeAVUL36DqNfvV3gsRzlwe173dvkK7CJyr4jsEZEXReQREam2dWC2FQq0NssS/dS+cyIvKpXXNVPnce1WV1UGEkS9gnPTZZMLPh+ve7v8rnl6PYCNqtojIt8HAFX9i0K/F3bnaTHzZ9icY4MdQRQH+a5pIJiJrsqtiqHihLKY9aAn/DSA21T1c4W2DTuwFxtobV18XBSY4oIBNVlCWcx6kC8CeCjPAa0AsAIA6uvrLT5tYcXmz21NecuOoNIxAAWD0zinU8Ecu4g8JSIv53jcMmCbuwH0AHjQaz+qep+qNqhqQ01NjZ2jL1LYw/rZEVQazoBJZFfBwK6q16rq7ByP9QAgIl8AcBOAz2kUo52KEHagZUdQaTjVLJFdvlIxIrIEwDcBfEJVO+0ckn1RrIrCW+DicXAKkV1+c+z/F8BwAE+KqVXdoqpf8X1UASg10DLnGx72SRDZ5Suwq+oFtg4kTjghUbjKmQGTiLxx5GkOzPmGi30SRHZxrpgcmPMNH/skiOxhYM8hrJxvqXl85v2JqBgM7DmEkfMtNY/PvD+Rt8ErSIkAHZ3dWf9PU2OIOfYcwsj5lprHZ96fKLfBA9w6urrxbmf3Gf9P08C3xLXYbaUrgs75lprHZ96fKLdcjR4v/Y2hpLfaE9Vid2loeqnTHIQ9LQKRK0pt3KShMZSowO5SuqLUaQ44/wy5ptzFZkpVauMmDY2hRAV2l9IVpebxWetNLgnz7jlXo8dLWhpDicqxuzY0vdQ8Pmu9yRX57p5tX8OD54JiVUzCAjuHphPFQ7F3z64UO7gmUYE9ilkciehMxdw9c2xGcBIV2AF+chPFQTF3z4WKHdhAK1/iAjsRBauY9Ekxd89e6Zr+ljtb8uVjYCeiopWSPil09+yVrqkQCbzjNenzLiWq3JGIgmVzrIjX2IxejxU2bZUtuzSQsVy+AruI/LWIvCgiO0TkCRGZaOvAiCh+bI4V8RqbURfwKGuXBjKWy28q5l5V/T8AICJ/CuDbAGK5NB4R+Wd7rIhXuibIsmWXBjKWy1eLXVXfH/Dl2QBy30MRUSKEMbVF0KOs0zDvku/OUxH5HoD/DuA9AFf7PiIiiq2wxooEWbachoGMoh4dFac3EHkKwHk5fnS3qq4fsN1dAEao6nc89rMCwAoAqK+vn9/a2lr2QfuV9B5xIsrP1RggIltVtaHgdoUCewlPWA/gV6o6u9C2DQ0N2tLSYuV5SzW4XAswn9Z+b/VcvVCIyB3FBna/VTEXDvjyFgB7/OwvDEH0iKehfIqI3OE3x75KRGYAOAWgFQFXxNhoFQfRIx7mTHZERIX4CuyqequtAynE1oRBQUztm4byKSJyhzMjT22lUPyWa+VaFSYN5VNE5A5nArutVrGfGlmvXPrVM2u4bB0RxYYzk4DZTKGUWyPrddfw9J6juGfZJayKIaJYcCawx2FQQb67Bs4DT0Rx4Uxgj8PqSK6tqUpE9rg0VsWZwA5EvzpSHO4aiCh8ri3j50znaRwEPTkREcWTa1P9OtVij4Oo7xqIKHyujVVhi52IqADXxqowsBMRFRDGPPQ2MRVDRFRAHKrySsHATkRUhGL61+JSEsnATkRkQZxKIhnYiYhKlKtlHqfpuxMd2ONyW0REyeHVMh8c1PtFURKZ2MAep9siIkoOr5Z5hQh6cyw1GkVJZGLLHV0bKUZEbvBqgfeqxqYk0kpgF5E/ExEVkXE29meDayPFiMgNXi3w/ilG4jDliO9UjIhMBnA9gAP+D8cezsRIREHINxmgV0lk2P19NlrsPwDwTQBnJpci5NpIMSJyQ6mTAXqtvLZue1tgx+irxS4itwBoU9WdImLpkOxwbaQYEbmjlMkAoyiDLBjYReQpAOfl+NHdAL4Fk4YpSERWAFgBAPX19SUcYvk4EyMRRS2K/r6CgV1Vr831fRG5BMA0AP2t9UkAtonIAlVtz7Gf+wDcBwANDQ2xStsQEQUliv6+snPsqvqSqo5X1amqOhXAQQDzcgV1IqK0iqK/L7EDlIiI4iCK/j5rgb2v1U5ERIOE3d+X2JGnRERpxcBORJQwDOxERAnDwE5ElDAM7ERECSOaY/7gwJ9U5CiAVh+7GAfgbUuHE6WknAeQnHNJynkAPJfc05gcAAADbElEQVQ48nseU1S1ptBGkQR2v0SkRVUboj4Ov5JyHkByziUp5wHwXOIorPNgKoaIKGEY2ImIEsbVwH5f1AdgSVLOA0jOuSTlPACeSxyFch5O5tiJiMibqy12IiLy4GRgF5G/FpEXRWSHiDwhIhOjPqZyici9IrKn73weEZHqqI+pXCJyu4i8IiKnRMS5CgYRWSIie0XkNRFZGfXxlEtEHhCRIyLyctTH4oeITBaRp0VkV9919bWoj6lcIjJCRH4nIjv7zuW7gT6fi6kYETlHVd/v+/+fApilql+J+LDKIiLXA9ioqj0i8n0AUNW/iPiwyiIiFwE4BeAnAP5cVVsiPqSiiUgFgFcBXAeztsALAJpUdVekB1YGEbkSwHEAP1XV2VEfT7lEpBZArapuE5FRALYCWOroayIAzlbV4yJSCWATgK+p6pYgns/JFnt/UO9zNmK2kHYpVPUJVe3p+3ILzEpUTlLV3aq6N+rjKNMCAK+p6n5VPQlgNYBbIj6msqjqcwDeifo4/FLVw6q6re//HwDYDcDJtS7VON73ZWXfI7C45WRgBwAR+Z6IvAngcwC+HfXxWPJFAI9HfRApVQfgzQFfH4SjQSSJRGQqgLkAno/2SMonIhUisgPAEQBPqmpg5xLbwC4iT4nIyzketwCAqt6tqpMBPAjgq9EebX6FzqVvm7sB9MCcT2wVcy5ENonISABrAHx90N26U1S1V1XnwNyVLxCRwNJksV0az2sR7RweBPArAN8J8HB8KXQuIvIFADcBuEZj3ulRwuvimjYAkwd8PanvexShvnz0GgAPquraqI/HBlXtEJGnASwBEEgHd2xb7PmIyIUDvrwFwJ6ojsUvEVkC4JsAblbVzqiPJ8VeAHChiEwTkWEAlgP4RcTHlGp9HY73A9itqn8f9fH4ISI1/RVvIlIF00kfWNxytSpmDYAZMBUYrQC+oqpOtq5E5DUAwwEc6/vWFocrfD4N4B8B1ADoALBDVRdHe1TFE5EbAPwQQAWAB1T1exEfUllEpBnAVTAzCb4F4Duqen+kB1UGEbkCwG8AvATzXgeAb6nqr6I7qvKIyKUA/g3m2hoC4GFV/avAns/FwE5ERN6cTMUQEZE3BnYiooRhYCciShgGdiKihGFgJyJKGAZ2IqKEYWAnIkoYBnYiooT5/x++8uf3xtI4AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# data\n",
    "tb_writer = SummaryWriter(comment='summary writer')\n",
    "#tb_writer.add_graph(model)\n",
    "train_dataloader = DataLoader(src_data, batch_size=50)\n",
    "\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.01)\n",
    "\n",
    "global_step = 0\n",
    "# gradient accumulation\n",
    "gradient_accumulation_steps = 2\n",
    "model.zero_grad()\n",
    "train_iterator = trange(30, desc='Epoch')\n",
    "for epoch in train_iterator:\n",
    "    epoch_iterator = tqdm(train_dataloader, desc='Iteration')\n",
    "    for step, batch in enumerate(epoch_iterator):\n",
    "        model.train()\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "        inputs = {'x': batch[0],\n",
    "                  'y': batch[1]}\n",
    "        outputs = model(**inputs)\n",
    "        loss = outputs[1]\n",
    "        loss = loss / gradient_accumulation_steps\n",
    "        # backward to compute gradients\n",
    "        loss.backward()\n",
    "        nn.utils.clip_grad_norm_(model.parameters(), 5.0)\n",
    "        \n",
    "        if (step + 1) % gradient_accumulation_steps == 0:\n",
    "            optimizer.step()\n",
    "            model.zero_grad()\n",
    "            global_step += 1\n",
    "            # item() to get a number from a tensor containing a single value\n",
    "            tb_writer.add_scalar('loss', loss.item(), global_step)\n",
    "    for name, param in model.named_parameters():\n",
    "        tb_writer.add_histogram(name, param, epoch)\n",
    "        tb_writer.add_histogram(f'{name}.grad', param.grad, epoch)\n",
    "    \n",
    "tb_writer.close()\n",
    "model.eval()\n",
    "inputs = {'x': x}\n",
    "outputs = model(**inputs)\n",
    "predictions = outputs[0]\n",
    "plt.scatter(x.numpy(), y.numpy())\n",
    "plt.plot(x.numpy(), predictions.data.numpy(), 'r', lw=2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}