lesson7_human_numbers.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "lesson7-human-numbers.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [
        "nv_TDLKR3_L4",
        "xC8VjuwG3_MY",
        "a627iOGk3_Mi",
        "BQvtI9l_3_M_",
        "fhdKG4p93_NP",
        "2PK2YyEP3_Nh",
        "Rm0iRgZj3_Nq"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K454H5BD3_Jo",
        "colab_type": "text"
      },
      "source": [
        "# Human numbers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KldD_EUu3_Jr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from fastai.text import *"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x8K7hlz73_Jv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "bs=64"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SIOsfpBD3_Jy",
        "colab_type": "text"
      },
      "source": [
        "## Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vfRQoIfW3_Jz",
        "colab_type": "code",
        "outputId": "38e5d841-67f7-403e-bb90-c7c54b304d03",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "path = untar_data(URLs.HUMAN_NUMBERS)\n",
        "path.ls()"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[PosixPath('/root/.fastai/data/human_numbers/train.txt'),\n",
              " PosixPath('/root/.fastai/data/human_numbers/valid.txt')]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XlQpUWch3_J7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def readnums(d): return [', '.join(o.strip() for o in open(path/d).readlines())]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bl3Qresh3_J-",
        "colab_type": "code",
        "outputId": "93d4e859-b5db-4fe7-e564-d0e9d98fb492",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "train_txt = readnums('train.txt'); train_txt[0][:80]"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'one, two, three, four, five, six, seven, eight, nine, ten, eleven, twelve, thirt'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yIFPtwtU3_KD",
        "colab_type": "code",
        "outputId": "8578cdc4-edc0-4b55-e2d7-4f3589c03e32",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "valid_txt = readnums('valid.txt'); valid_txt[0][-80:]"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "' nine thousand nine hundred ninety eight, nine thousand nine hundred ninety nine'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LuEspm_U3_KI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train = TextList(train_txt, path=path)\n",
        "valid = TextList(valid_txt, path=path)\n",
        "\n",
        "src = ItemLists(path=path, train=train, valid=valid).label_for_lm()\n",
        "data = src.databunch(bs=bs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F6QFe2Cb3_KL",
        "colab_type": "code",
        "outputId": "c8ca14d4-371c-45fd-de0a-935553cb40da",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "train[0].text[:80]"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'xxbos one , two , three , four , five , six , seven , eight , nine , ten , eleve'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m_yO33Bt3_KQ",
        "colab_type": "code",
        "outputId": "2aad0b62-8178-4919-bdec-7d3d26624614",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "len(data.valid_ds[0][0].data)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "13017"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "op1TwsJI3_KV",
        "colab_type": "code",
        "outputId": "b1634f11-4c51-460c-a59a-a9783ecc7d4d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "data.bptt, len(data.valid_dl)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(70, 3)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "855upyvm3_Ka",
        "colab_type": "code",
        "outputId": "bd090632-273c-46a8-b9a5-a2ae891cd078",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "13017/70/bs"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2.905580357142857"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "muNgjShU3_Ke",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "it = iter(data.valid_dl)\n",
        "x1,y1 = next(it)\n",
        "x2,y2 = next(it)\n",
        "x3,y3 = next(it)\n",
        "it.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OtTYilUu3_Kh",
        "colab_type": "code",
        "outputId": "4f8cd6e4-4dc1-42d9-b5a5-41f01a592309",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "x1.numel()+x2.numel()+x3.numel()"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "13440"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rZmMPBMe3_Kn",
        "colab_type": "code",
        "outputId": "713e5c2f-1a24-428d-b8c1-7668b3163bcd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "x1.shape,y1.shape"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(torch.Size([64, 70]), torch.Size([64, 70]))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8X5tPOJC3_Ku",
        "colab_type": "code",
        "outputId": "f35db22f-24c7-47df-978a-2b1c96df6194",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "x2.shape,y2.shape"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(torch.Size([64, 70]), torch.Size([64, 70]))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T3QkVzda3_K1",
        "colab_type": "code",
        "outputId": "4575a7ef-d5c0-4b5b-b708-ac360dfd4bd5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "x1[:,0]"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([ 2,  9, 11, 12, 13, 11, 10,  9, 10, 14, 19, 25, 19, 15, 16, 11, 19,  9,\n",
              "        10,  9, 19, 25, 19, 11, 19, 11, 10,  9, 19, 20, 11, 26, 20, 23, 20, 20,\n",
              "        24, 20, 11, 14, 11, 11,  9, 14,  9, 20, 10, 20, 35, 17, 11, 10,  9, 17,\n",
              "         9, 20, 10, 20, 11, 20, 11, 20, 20, 20], device='cuda:0')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oByVntqD3_K-",
        "colab_type": "code",
        "outputId": "8819dc31-81d7-4a5c-d981-81586465f26c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "y1[:,0]"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([19, 19, 27, 10,  9, 12, 32, 19, 26, 10, 11, 15, 11, 10,  9, 15, 11, 19,\n",
              "        26, 19, 11, 18, 11, 18,  9, 18, 21, 19, 10, 10, 20,  9, 11, 16, 11, 11,\n",
              "        13, 11, 13,  9, 13, 14, 20, 10, 20, 11, 24, 11,  9,  9, 16, 17, 20, 10,\n",
              "        20, 11, 24, 11, 19,  9, 19, 11, 11, 10], device='cuda:0')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dOoi_4Eh3_LE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "v = data.valid_ds.vocab"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mosIO-ag3_LI",
        "colab_type": "code",
        "outputId": "185e86e1-f6de-4d01-9efb-9687cb126f19",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "v.textify(x1[0])"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'xxbos eight thousand one , eight thousand two , eight thousand three , eight thousand four , eight thousand five , eight thousand six , eight thousand seven , eight thousand eight , eight thousand nine , eight thousand ten , eight thousand eleven , eight thousand twelve , eight thousand thirteen , eight thousand fourteen , eight thousand fifteen , eight thousand sixteen , eight thousand seventeen , eight'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6qP-swBv3_LQ",
        "colab_type": "code",
        "outputId": "0f27f860-291f-41d2-8129-4efa371ed459",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "v.textify(y1[0])"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'eight thousand one , eight thousand two , eight thousand three , eight thousand four , eight thousand five , eight thousand six , eight thousand seven , eight thousand eight , eight thousand nine , eight thousand ten , eight thousand eleven , eight thousand twelve , eight thousand thirteen , eight thousand fourteen , eight thousand fifteen , eight thousand sixteen , eight thousand seventeen , eight thousand'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TbYRNq9a3_LU",
        "colab_type": "code",
        "outputId": "4e95a519-2c9d-450d-df41-4e3baae4751d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "v.textify(x2[0])"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'thousand eighteen , eight thousand nineteen , eight thousand twenty , eight thousand twenty one , eight thousand twenty two , eight thousand twenty three , eight thousand twenty four , eight thousand twenty five , eight thousand twenty six , eight thousand twenty seven , eight thousand twenty eight , eight thousand twenty nine , eight thousand thirty , eight thousand thirty one , eight thousand thirty two ,'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4yyIFS633_LZ",
        "colab_type": "code",
        "outputId": "f2d1ce11-e6d1-4cb4-adc3-5211c74c55f2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "v.textify(x3[0])"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'eight thousand thirty three , eight thousand thirty four , eight thousand thirty five , eight thousand thirty six , eight thousand thirty seven , eight thousand thirty eight , eight thousand thirty nine , eight thousand forty , eight thousand forty one , eight thousand forty two , eight thousand forty three , eight thousand forty four , eight thousand forty five , eight thousand forty six , eight'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nB3iIO-e3_Li",
        "colab_type": "code",
        "outputId": "46b41f9a-1b6d-40f4-f0fd-22fc63a05fe6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "v.textify(x1[1])"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "', eight thousand forty six , eight thousand forty seven , eight thousand forty eight , eight thousand forty nine , eight thousand fifty , eight thousand fifty one , eight thousand fifty two , eight thousand fifty three , eight thousand fifty four , eight thousand fifty five , eight thousand fifty six , eight thousand fifty seven , eight thousand fifty eight , eight thousand fifty nine ,'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lYeD03AN3_Lm",
        "colab_type": "code",
        "outputId": "35fda18e-7050-4155-d2a6-472739542ce4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "v.textify(x2[1])"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'eight thousand sixty , eight thousand sixty one , eight thousand sixty two , eight thousand sixty three , eight thousand sixty four , eight thousand sixty five , eight thousand sixty six , eight thousand sixty seven , eight thousand sixty eight , eight thousand sixty nine , eight thousand seventy , eight thousand seventy one , eight thousand seventy two , eight thousand seventy three , eight thousand'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l5X-AVyF3_Lt",
        "colab_type": "code",
        "outputId": "9b297353-5387-4797-a6d6-b93e53cb8a49",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "v.textify(x3[1])"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'seventy four , eight thousand seventy five , eight thousand seventy six , eight thousand seventy seven , eight thousand seventy eight , eight thousand seventy nine , eight thousand eighty , eight thousand eighty one , eight thousand eighty two , eight thousand eighty three , eight thousand eighty four , eight thousand eighty five , eight thousand eighty six , eight thousand eighty seven , eight thousand eighty'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BqnvCykT3_Lx",
        "colab_type": "code",
        "outputId": "17845be8-0795-45cc-96b3-c651b5b3d292",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "v.textify(x3[-1])"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'ninety , nine thousand nine hundred ninety one , nine thousand nine hundred ninety two , nine thousand nine hundred ninety three , nine thousand nine hundred ninety four , nine thousand nine hundred ninety five , nine thousand nine hundred ninety six , nine thousand nine hundred ninety seven , nine thousand nine hundred ninety eight , nine thousand nine hundred ninety nine xxbos eight thousand one , eight'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8-QO9VJ93_L0",
        "colab_type": "code",
        "outputId": "9d534dcb-567a-40ca-e3f4-389bdeb09a49",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 374
        }
      },
      "source": [
        "data.show_batch(ds_type=DatasetType.Valid)"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th>idx</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>thousand forty seven , eight thousand forty eight , eight thousand forty nine , eight thousand fifty , eight thousand fifty one , eight thousand fifty two , eight thousand fifty three , eight thousand fifty four , eight thousand fifty five , eight thousand fifty six , eight thousand fifty seven , eight thousand fifty eight , eight thousand fifty nine , eight thousand sixty , eight thousand sixty</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>eight , eight thousand eighty nine , eight thousand ninety , eight thousand ninety one , eight thousand ninety two , eight thousand ninety three , eight thousand ninety four , eight thousand ninety five , eight thousand ninety six , eight thousand ninety seven , eight thousand ninety eight , eight thousand ninety nine , eight thousand one hundred , eight thousand one hundred one , eight thousand one</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>thousand one hundred twenty four , eight thousand one hundred twenty five , eight thousand one hundred twenty six , eight thousand one hundred twenty seven , eight thousand one hundred twenty eight , eight thousand one hundred twenty nine , eight thousand one hundred thirty , eight thousand one hundred thirty one , eight thousand one hundred thirty two , eight thousand one hundred thirty three , eight thousand</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>three , eight thousand one hundred fifty four , eight thousand one hundred fifty five , eight thousand one hundred fifty six , eight thousand one hundred fifty seven , eight thousand one hundred fifty eight , eight thousand one hundred fifty nine , eight thousand one hundred sixty , eight thousand one hundred sixty one , eight thousand one hundred sixty two , eight thousand one hundred sixty three</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>thousand one hundred eighty three , eight thousand one hundred eighty four , eight thousand one hundred eighty five , eight thousand one hundred eighty six , eight thousand one hundred eighty seven , eight thousand one hundred eighty eight , eight thousand one hundred eighty nine , eight thousand one hundred ninety , eight thousand one hundred ninety one , eight thousand one hundred ninety two , eight thousand</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nv_TDLKR3_L4",
        "colab_type": "text"
      },
      "source": [
        "## Single fully connected model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BapeEbS33_L4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data = src.databunch(bs=bs, bptt=3)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mpgcdiT83_L6",
        "colab_type": "code",
        "outputId": "88b6ae6f-d705-4ac9-cfec-ff57fc0f3b3c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "x,y = data.one_batch()\n",
        "x.shape,y.shape"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(torch.Size([64, 3]), torch.Size([64, 3]))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CLN17ia23_L-",
        "colab_type": "code",
        "outputId": "603af9b3-516d-4b55-c841-c1fa0bdd0ee3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "nv = len(v.itos); nv"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "40"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CI1UPXfB3_MB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "nh=64"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I9c5bMhs3_MD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def loss4(input,target): return F.cross_entropy(input, target[:,-1])\n",
        "def acc4 (input,target): return accuracy(input, target[:,-1])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_D3oUbAM3_MI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Model0(nn.Module):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.i_h = nn.Embedding(nv,nh)  # green arrow\n",
        "        self.h_h = nn.Linear(nh,nh)     # brown arrow\n",
        "        self.h_o = nn.Linear(nh,nv)     # blue arrow\n",
        "        self.bn = nn.BatchNorm1d(nh)\n",
        "        \n",
        "    def forward(self, x):\n",
        "        h = self.bn(F.relu(self.h_h(self.i_h(x[:,0]))))\n",
        "        if x.shape[1]>1:\n",
        "            h = h + self.i_h(x[:,1])\n",
        "            h = self.bn(F.relu(self.h_h(h)))\n",
        "        if x.shape[1]>2:\n",
        "            h = h + self.i_h(x[:,2])\n",
        "            h = self.bn(F.relu(self.h_h(h)))\n",
        "        return self.h_o(h)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IB-Iwhfk3_MQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "learn = Learner(data, Model0(), loss_func=loss4, metrics=acc4)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o0cTBgLv3_MT",
        "colab_type": "code",
        "outputId": "2910d205-cd59-48ce-8ee7-e8520e096e61",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 235
        }
      },
      "source": [
        "learn.fit_one_cycle(6, 1e-4)"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>acc4</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>3.662185</td>\n",
              "      <td>3.734218</td>\n",
              "      <td>0.020221</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>3.200958</td>\n",
              "      <td>3.311808</td>\n",
              "      <td>0.253447</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>2.641509</td>\n",
              "      <td>2.850472</td>\n",
              "      <td>0.384421</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>2.290921</td>\n",
              "      <td>2.580583</td>\n",
              "      <td>0.399816</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>2.148529</td>\n",
              "      <td>2.481218</td>\n",
              "      <td>0.395910</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>2.118700</td>\n",
              "      <td>2.466454</td>\n",
              "      <td>0.395910</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xC8VjuwG3_MY",
        "colab_type": "text"
      },
      "source": [
        "## Same thing with a loop"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "id5ZROw13_MZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Model1(nn.Module):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.i_h = nn.Embedding(nv,nh)  # green arrow\n",
        "        self.h_h = nn.Linear(nh,nh)     # brown arrow\n",
        "        self.h_o = nn.Linear(nh,nv)     # blue arrow\n",
        "        self.bn = nn.BatchNorm1d(nh)\n",
        "        \n",
        "    def forward(self, x):\n",
        "        h = torch.zeros(x.shape[0], nh).to(device=x.device)\n",
        "        for i in range(x.shape[1]):\n",
        "            h = h + self.i_h(x[:,i])\n",
        "            h = self.bn(F.relu(self.h_h(h)))\n",
        "        return self.h_o(h)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cY3G0QNo3_Mb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "learn = Learner(data, Model1(), loss_func=loss4, metrics=acc4)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FA0x0PvX3_Mf",
        "colab_type": "code",
        "outputId": "ece3d410-0aff-486e-bb69-87ae69f78e6d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 235
        }
      },
      "source": [
        "learn.fit_one_cycle(6, 1e-4)"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>acc4</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>3.647422</td>\n",
              "      <td>3.649930</td>\n",
              "      <td>0.051011</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>3.075131</td>\n",
              "      <td>3.034867</td>\n",
              "      <td>0.377987</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>2.514988</td>\n",
              "      <td>2.551021</td>\n",
              "      <td>0.463235</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>2.177379</td>\n",
              "      <td>2.298244</td>\n",
              "      <td>0.464154</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>2.037625</td>\n",
              "      <td>2.212026</td>\n",
              "      <td>0.462546</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>2.009076</td>\n",
              "      <td>2.200106</td>\n",
              "      <td>0.462776</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a627iOGk3_Mi",
        "colab_type": "text"
      },
      "source": [
        "## Multi fully connected model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aoTQ5tUQ3_Mj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data = src.databunch(bs=bs, bptt=20)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pR1iaAfR3_Mp",
        "colab_type": "code",
        "outputId": "6305dd59-a87f-4d00-da59-d51641a853aa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "x,y = data.one_batch()\n",
        "x.shape,y.shape"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(torch.Size([64, 20]), torch.Size([64, 20]))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-We-k9K13_Mv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Model2(nn.Module):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.i_h = nn.Embedding(nv,nh)\n",
        "        self.h_h = nn.Linear(nh,nh)\n",
        "        self.h_o = nn.Linear(nh,nv)\n",
        "        self.bn = nn.BatchNorm1d(nh)\n",
        "        \n",
        "    def forward(self, x):\n",
        "        h = torch.zeros(x.shape[0], nh).to(device=x.device)\n",
        "        res = []\n",
        "        for i in range(x.shape[1]):\n",
        "            h = h + self.i_h(x[:,i])\n",
        "            h = F.relu(self.h_h(h))\n",
        "            res.append(self.h_o(self.bn(h)))\n",
        "        return torch.stack(res, dim=1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ylALg6jQ3_My",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "learn = Learner(data, Model2(), metrics=accuracy)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BGba63zh3_M6",
        "colab_type": "code",
        "outputId": "a6e5fcfa-10b3-4799-9d3c-426c8b01a3d3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 359
        }
      },
      "source": [
        "learn.fit_one_cycle(10, 1e-4, pct_start=0.1)"
      ],
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>3.800973</td>\n",
              "      <td>3.729707</td>\n",
              "      <td>0.022230</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>3.696275</td>\n",
              "      <td>3.603117</td>\n",
              "      <td>0.025497</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>3.567799</td>\n",
              "      <td>3.485523</td>\n",
              "      <td>0.121094</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>3.439255</td>\n",
              "      <td>3.383646</td>\n",
              "      <td>0.141832</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>3.322295</td>\n",
              "      <td>3.302094</td>\n",
              "      <td>0.219460</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>3.223532</td>\n",
              "      <td>3.242494</td>\n",
              "      <td>0.259020</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>3.147387</td>\n",
              "      <td>3.203842</td>\n",
              "      <td>0.273651</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>3.094224</td>\n",
              "      <td>3.182674</td>\n",
              "      <td>0.280611</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>3.061521</td>\n",
              "      <td>3.174519</td>\n",
              "      <td>0.284659</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>3.044448</td>\n",
              "      <td>3.173323</td>\n",
              "      <td>0.285440</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BQvtI9l_3_M_",
        "colab_type": "text"
      },
      "source": [
        "## Maintain state"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w-hjaEbT3_NA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Model3(nn.Module):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.i_h = nn.Embedding(nv,nh)\n",
        "        self.h_h = nn.Linear(nh,nh)\n",
        "        self.h_o = nn.Linear(nh,nv)\n",
        "        self.bn = nn.BatchNorm1d(nh)\n",
        "        self.h = torch.zeros(bs, nh).cuda()\n",
        "        \n",
        "    def forward(self, x):\n",
        "        res = []\n",
        "        h = self.h\n",
        "        for i in range(x.shape[1]):\n",
        "            h = h + self.i_h(x[:,i])\n",
        "            h = F.relu(self.h_h(h))\n",
        "            res.append(self.bn(h))\n",
        "        self.h = h.detach()\n",
        "        res = torch.stack(res, dim=1)\n",
        "        res = self.h_o(res)\n",
        "        return res"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_YaMn_FP3_ND",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "learn = Learner(data, Model3(), metrics=accuracy)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_We1k_Xp3_NK",
        "colab_type": "code",
        "outputId": "62845ae4-99cc-4077-e336-b15df19ef356",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 669
        }
      },
      "source": [
        "learn.fit_one_cycle(20, 3e-3)"
      ],
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>3.647981</td>\n",
              "      <td>3.540949</td>\n",
              "      <td>0.112216</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>3.302816</td>\n",
              "      <td>2.893462</td>\n",
              "      <td>0.450497</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>2.634717</td>\n",
              "      <td>1.966482</td>\n",
              "      <td>0.466335</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>2.031778</td>\n",
              "      <td>1.891439</td>\n",
              "      <td>0.398295</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.704693</td>\n",
              "      <td>1.834678</td>\n",
              "      <td>0.437003</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>1.493576</td>\n",
              "      <td>1.763799</td>\n",
              "      <td>0.468040</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>1.320778</td>\n",
              "      <td>1.760281</td>\n",
              "      <td>0.523722</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>1.189989</td>\n",
              "      <td>1.755907</td>\n",
              "      <td>0.523580</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>1.090191</td>\n",
              "      <td>1.810392</td>\n",
              "      <td>0.508239</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.997321</td>\n",
              "      <td>1.863108</td>\n",
              "      <td>0.507457</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.911781</td>\n",
              "      <td>1.874390</td>\n",
              "      <td>0.528409</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.833397</td>\n",
              "      <td>1.963326</td>\n",
              "      <td>0.535795</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.765010</td>\n",
              "      <td>2.010844</td>\n",
              "      <td>0.550994</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.710084</td>\n",
              "      <td>2.030426</td>\n",
              "      <td>0.556534</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.663980</td>\n",
              "      <td>2.046339</td>\n",
              "      <td>0.564773</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.628800</td>\n",
              "      <td>2.067131</td>\n",
              "      <td>0.566761</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.603532</td>\n",
              "      <td>2.118631</td>\n",
              "      <td>0.557315</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.585641</td>\n",
              "      <td>2.140126</td>\n",
              "      <td>0.561719</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.573890</td>\n",
              "      <td>2.130257</td>\n",
              "      <td>0.568040</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.567488</td>\n",
              "      <td>2.132430</td>\n",
              "      <td>0.565128</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fhdKG4p93_NP",
        "colab_type": "text"
      },
      "source": [
        "## nn.RNN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7O50DVK23_NQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Model4(nn.Module):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.i_h = nn.Embedding(nv,nh)\n",
        "        self.rnn = nn.RNN(nh,nh, batch_first=True)\n",
        "        self.h_o = nn.Linear(nh,nv)\n",
        "        self.bn = BatchNorm1dFlat(nh)\n",
        "        self.h = torch.zeros(1, bs, nh).cuda()\n",
        "        \n",
        "    def forward(self, x):\n",
        "        res,h = self.rnn(self.i_h(x), self.h)\n",
        "        self.h = h.detach()\n",
        "        return self.h_o(self.bn(res))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qA8qMK5o3_NV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "learn = Learner(data, Model4(), metrics=accuracy)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kHvrKtXV3_Ne",
        "colab_type": "code",
        "outputId": "b5a1e065-14a9-4708-f185-10dd9b2ce467",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 669
        }
      },
      "source": [
        "learn.fit_one_cycle(20, 3e-3)"
      ],
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>3.618752</td>\n",
              "      <td>3.475577</td>\n",
              "      <td>0.136577</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>3.202406</td>\n",
              "      <td>2.688467</td>\n",
              "      <td>0.453267</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>2.510317</td>\n",
              "      <td>1.926023</td>\n",
              "      <td>0.467259</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.962517</td>\n",
              "      <td>1.873861</td>\n",
              "      <td>0.362216</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.666820</td>\n",
              "      <td>1.790041</td>\n",
              "      <td>0.445952</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>1.437092</td>\n",
              "      <td>1.643206</td>\n",
              "      <td>0.468111</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>1.220581</td>\n",
              "      <td>1.583685</td>\n",
              "      <td>0.525781</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>1.041729</td>\n",
              "      <td>1.590503</td>\n",
              "      <td>0.538707</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.895967</td>\n",
              "      <td>1.588321</td>\n",
              "      <td>0.582528</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.777269</td>\n",
              "      <td>1.515340</td>\n",
              "      <td>0.594034</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.670584</td>\n",
              "      <td>1.492727</td>\n",
              "      <td>0.582884</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.581003</td>\n",
              "      <td>1.404665</td>\n",
              "      <td>0.586790</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.507030</td>\n",
              "      <td>1.320654</td>\n",
              "      <td>0.616193</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.448248</td>\n",
              "      <td>1.347034</td>\n",
              "      <td>0.617330</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.400520</td>\n",
              "      <td>1.312607</td>\n",
              "      <td>0.627344</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.364552</td>\n",
              "      <td>1.340127</td>\n",
              "      <td>0.620668</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.337766</td>\n",
              "      <td>1.354721</td>\n",
              "      <td>0.621023</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.319293</td>\n",
              "      <td>1.370037</td>\n",
              "      <td>0.623651</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.307397</td>\n",
              "      <td>1.407944</td>\n",
              "      <td>0.621520</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.300418</td>\n",
              "      <td>1.416663</td>\n",
              "      <td>0.620384</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2PK2YyEP3_Nh",
        "colab_type": "text"
      },
      "source": [
        "## 2-layer GRU"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NXR_XvOR3_Ni",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Model5(nn.Module):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.i_h = nn.Embedding(nv,nh)\n",
        "        self.rnn = nn.GRU(nh, nh, 2, batch_first=True)\n",
        "        self.h_o = nn.Linear(nh,nv)\n",
        "        self.bn = BatchNorm1dFlat(nh)\n",
        "        self.h = torch.zeros(2, bs, nh).cuda()\n",
        "        \n",
        "    def forward(self, x):\n",
        "        res,h = self.rnn(self.i_h(x), self.h)\n",
        "        self.h = h.detach()\n",
        "        return self.h_o(self.bn(res))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dzbv7a_N3_Nj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "learn = Learner(data, Model5(), metrics=accuracy)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oLRROAkX3_Nn",
        "colab_type": "code",
        "outputId": "dacd42e6-e43d-4290-8a03-44a0e2b726fc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 359
        }
      },
      "source": [
        "learn.fit_one_cycle(10, 1e-2)"
      ],
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>2.989351</td>\n",
              "      <td>2.276178</td>\n",
              "      <td>0.461648</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.882157</td>\n",
              "      <td>1.417959</td>\n",
              "      <td>0.553267</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.983393</td>\n",
              "      <td>1.266026</td>\n",
              "      <td>0.768466</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.477263</td>\n",
              "      <td>1.074785</td>\n",
              "      <td>0.826918</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.239261</td>\n",
              "      <td>1.045795</td>\n",
              "      <td>0.814418</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.125659</td>\n",
              "      <td>0.952196</td>\n",
              "      <td>0.838849</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.069592</td>\n",
              "      <td>0.963979</td>\n",
              "      <td>0.826065</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.041627</td>\n",
              "      <td>0.966188</td>\n",
              "      <td>0.828977</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.026565</td>\n",
              "      <td>1.026441</td>\n",
              "      <td>0.824148</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.018878</td>\n",
              "      <td>1.023104</td>\n",
              "      <td>0.822159</td>\n",
              "      <td>00:00</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Rm0iRgZj3_Nq",
        "colab_type": "text"
      },
      "source": [
        "## fin"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bbLERXYK3_Nr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}