03_02_exercise.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "39409e2b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/lucasvanwalstijn/miniconda3/envs/fastai/lib/python3.8/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3f81ddba",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./names.txt') as file:\n",
    "    names = file.read().splitlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "75be6f2e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['emma', 'olivia', 'ava']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "names[0:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ea0f8f74",
   "metadata": {},
   "outputs": [],
   "source": [
    "letters = sorted(list(set([letter for name in names for letter in name])))\n",
    "i_to_s = {i+1: s for i, s in enumerate(letters)}\n",
    "i_to_s[0] = '.'\n",
    "s_to_i = {s: i for i, s in i_to_s.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6290438d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4db8a1d3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e50bd0ed",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 324,
   "id": "71a52214",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model:\n",
    "    \n",
    "    def __init__(self, config):\n",
    "        self.unique_items = config.get('unique_items', 27)\n",
    "        self.embedding_depth = config.get('embedding_depth', 2)\n",
    "        self.n_hidden_layer = config.get('n_hidden_layer', 100)\n",
    "        self.block_size = config.get('block_size', 3)\n",
    "        self.batch_size = config.get('batch_size', 32)\n",
    "        \n",
    "        self.train_size = config.get('train_size', 0.8)\n",
    "        self.val_size = config.get('train_size', 0.1)\n",
    "        self.test_size = 1.0 - self.train_size - self.val_size\n",
    "        \n",
    "        self.names = config.get('names').copy() #copy here to make sure we don't shuffle the original names\n",
    "        \n",
    "        # Build the model here\n",
    "        self.g = torch.Generator().manual_seed(2147483647)\n",
    "        \n",
    "        # embedding matrix C, size: (27,2) by default\n",
    "        self.C = torch.randn((self.unique_items, self.embedding_depth), requires_grad=True, generator=self.g)\n",
    "        \n",
    "        # Weight matrix of the hidden layer, size: (3 x 2, 100) by default\n",
    "        self.W1 = torch.randn([self.block_size * self.embedding_depth, self.n_hidden_layer], requires_grad=True, generator=self.g)\n",
    "        self.b1 = torch.randn([1, self.n_hidden_layer], requires_grad=True, generator=self.g)\n",
    "        \n",
    "        # Weight matrix of the logits layer, size: (100, 27) by default\n",
    "        self.W2 = torch.randn([self.n_hidden_layer, self.unique_items], requires_grad=True, generator=self.g)\n",
    "        self.b2 = torch.randn([1, self.unique_items,], requires_grad=True, generator=self.g)\n",
    "        \n",
    "        self.parameters = [self.C, self.W1, self.b1, self.W2, self.b2]\n",
    "        self.n_elements = sum(p.nelement() for p in self.parameters)\n",
    "        \n",
    "        # Building the datasets\n",
    "        \n",
    "        split1 = int(self.train_size * len(self.names))\n",
    "        split2 = int((self.train_size + self.val_size) * len(self.names))\n",
    "        \n",
    "        random.seed(42)\n",
    "        random.shuffle(self.names)\n",
    "\n",
    "        self.names_train = self.names[:split1]\n",
    "        self.names_val = self.names[split1:split2]\n",
    "        self.names_test = self.names[split2:]\n",
    "        \n",
    "        def build_dataset(names, block_size):\n",
    "            X, Y = [], []\n",
    "            for name in names:\n",
    "                context = block_size * [0]\n",
    "                for letter in name + \".\":\n",
    "                    ix = s_to_i[letter]\n",
    "                    X.append(context)\n",
    "                    Y.append(ix)\n",
    "                    context = context[1:] + [ix]\n",
    "            X = torch.tensor(X)\n",
    "            Y = torch.tensor(Y)\n",
    "            return X,Y\n",
    "        \n",
    "        self.X, self.Y = build_dataset(self.names, self.block_size)\n",
    "        \n",
    "        self.X_train, self.Y_train = build_dataset(self.names_train, self.block_size)\n",
    "        self.X_val, self.Y_val = build_dataset(self.names_val, self.block_size)\n",
    "        self.X_test, self.Y_test = build_dataset(self.names_test, self.block_size)\n",
    "        \n",
    "    \n",
    "    def forward(self, X, Y):\n",
    "        # Compute embeddings for the Xs\n",
    "        embs = self.C[X]\n",
    "        # Stretch out the embeddings, and do linear transformation\n",
    "        z1 = embs.view(-1, self.block_size * self.embedding_depth) @ self.W1 + self.b1\n",
    "        # Tanh activation\n",
    "        a1 = z1.tanh()\n",
    "        # Second linear transformation (==> computes the logits)\n",
    "        z2 = a1 @ self.W2 + self.b2\n",
    "        # Compute the loss\n",
    "        loss = F.cross_entropy(z2, Y)\n",
    "        \n",
    "        return loss\n",
    "    \n",
    "    \n",
    "    def get_loss(self, data_set='train'):\n",
    "        with torch.no_grad():\n",
    "            if data_set == 'train':\n",
    "                return self.forward(self.X_train, self.Y_train).item()\n",
    "            if data_set == 'val':\n",
    "                return self.forward(self.X_val, self.Y_val).item()\n",
    "            if data_set == 'test':\n",
    "                return self.forward(self.X_val, self.Y_val).item()\n",
    "        print(\"data_set should be either train, val or test\")\n",
    "    \n",
    "    \n",
    "    def backward(self, loss):\n",
    "        for p in self.parameters:\n",
    "            p.grad = None\n",
    "        loss.backward()\n",
    "        \n",
    "        \n",
    "    def update(self, learning_rate=1e-1):\n",
    "        for p in self.parameters:\n",
    "            p.data += - learning_rate * p.grad\n",
    "        \n",
    "        \n",
    "    def train(self, epochs, learning_rate=1e-1, print_every=None):\n",
    "        for _ in range(epochs):\n",
    "            loss = self.forward(self.X, self.Y)\n",
    "            self.backward(loss)\n",
    "            self.update(learning_rate)\n",
    "            if type(print_every) == int and _ % print_every == 0:\n",
    "                print(loss.item())\n",
    "                \n",
    "                \n",
    "    def train_minibatches(self, epochs, learning_rate=1e-1, print_every=None):\n",
    "        for _ in range(epochs):\n",
    "            batch_ixs = torch.randint(0, len(self.X_train), (self.batch_size,), generator=self.g)\n",
    "            X_batch = self.X_train[batch_ixs]\n",
    "            Y_batch = self.Y_train[batch_ixs]\n",
    "            \n",
    "            loss = self.forward(X_batch, Y_batch)\n",
    "            self.backward(loss)\n",
    "            self.update(learning_rate)\n",
    "            if type(print_every) == int and _ % print_every == 0:\n",
    "                print(loss.item())\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3f9530b7",
   "metadata": {},
   "source": [
    "### This is the final model from Andrej"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 325,
   "id": "ac358d8c",
   "metadata": {},
   "outputs": [],
   "source": [
    "m = Model({\n",
    "    'names': names,\n",
    "    'embedding_depth': 10,\n",
    "    'n_hidden_layer': 200,\n",
    "    'block_size': 3,\n",
    "    'batch_size': 32\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 326,
   "id": "fdb4d256",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.3885087966918945"
      ]
     },
     "execution_count": 326,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.train_minibatches(100000, 0.1)\n",
    "m.get_loss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 327,
   "id": "83d86c6e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.124298334121704"
      ]
     },
     "execution_count": 327,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.train_minibatches(100000, 0.01)\n",
    "m.get_loss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 328,
   "id": "8d1d7516",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.165433645248413"
      ]
     },
     "execution_count": 328,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.get_loss('val')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "08a3e192",
   "metadata": {},
   "source": [
    "### I think the embedding depth is too high with 10D, let's try 5 and see if it performs similarly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 329,
   "id": "b5931d46",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8762"
      ]
     },
     "execution_count": 329,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m = Model({\n",
    "    'names': names,\n",
    "    'embedding_depth': 5,\n",
    "    'n_hidden_layer': 200,\n",
    "    'block_size': 3,\n",
    "    'batch_size': 32\n",
    "})\n",
    "m.n_elements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 330,
   "id": "4d602f69",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.3803389072418213"
      ]
     },
     "execution_count": 330,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.train_minibatches(100000, 0.1)\n",
    "m.get_loss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "id": "433935c4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.1736955642700195"
      ]
     },
     "execution_count": 331,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.train_minibatches(100000, 0.01)\n",
    "m.get_loss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "id": "2195fa0c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.195449113845825"
      ]
     },
     "execution_count": 332,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.get_loss('val')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ce049c07",
   "metadata": {},
   "source": [
    "### Does seem to impact the model performance"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5e354009",
   "metadata": {},
   "source": [
    "### Let's try with higher embedding_depth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "id": "d4ab29f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15032"
      ]
     },
     "execution_count": 334,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m = Model({\n",
    "    'names': names,\n",
    "    'embedding_depth': 15,\n",
    "    'n_hidden_layer': 200,\n",
    "    'block_size': 3,\n",
    "    'batch_size': 32\n",
    "})\n",
    "m.n_elements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "id": "67c04b35",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.0927231311798096"
      ]
     },
     "execution_count": 335,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.train_minibatches(100000, 0.1)\n",
    "m.train_minibatches(100000, 0.01)\n",
    "m.get_loss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 336,
   "id": "ee4c5555",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.149803400039673"
      ]
     },
     "execution_count": 336,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.get_loss('val')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ce6b093",
   "metadata": {},
   "source": [
    "### Even larger"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "id": "9305d372",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18167"
      ]
     },
     "execution_count": 337,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m = Model({\n",
    "    'names': names,\n",
    "    'embedding_depth': 20,\n",
    "    'n_hidden_layer': 200,\n",
    "    'block_size': 3,\n",
    "    'batch_size': 32\n",
    "})\n",
    "m.n_elements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "id": "2b6f3150",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.068345308303833"
      ]
     },
     "execution_count": 338,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.train_minibatches(100000, 0.1)\n",
    "m.train_minibatches(100000, 0.01)\n",
    "m.get_loss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "id": "a953e06d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.1444251537323"
      ]
     },
     "execution_count": 339,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.get_loss('val')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4ff51d0a",
   "metadata": {},
   "source": [
    "### Even larger"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "id": "f88ce2a9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21302"
      ]
     },
     "execution_count": 340,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m = Model({\n",
    "    'names': names,\n",
    "    'embedding_depth': 25,\n",
    "    'n_hidden_layer': 200,\n",
    "    'block_size': 3,\n",
    "    'batch_size': 32\n",
    "})\n",
    "m.n_elements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 341,
   "id": "88463951",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.0630736351013184"
      ]
     },
     "execution_count": 341,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.train_minibatches(100000, 0.1)\n",
    "m.train_minibatches(100000, 0.01)\n",
    "m.get_loss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "id": "e22da2b7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.14540958404541"
      ]
     },
     "execution_count": 342,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.get_loss('val')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a0242fd3",
   "metadata": {},
   "source": [
    "### 20 seems to be enough, let's see if we can reduce n_hidden_layer size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 343,
   "id": "63a77d03",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13767"
      ]
     },
     "execution_count": 343,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m = Model({\n",
    "    'names': names,\n",
    "    'embedding_depth': 20,\n",
    "    'n_hidden_layer': 150,\n",
    "    'block_size': 3,\n",
    "    'batch_size': 32\n",
    "})\n",
    "m.n_elements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "id": "70cde15b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.0878281593322754"
      ]
     },
     "execution_count": 344,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.train_minibatches(100000, 0.1)\n",
    "m.train_minibatches(100000, 0.01)\n",
    "m.get_loss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 345,
   "id": "5a489bc8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.1424944400787354"
      ]
     },
     "execution_count": 345,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.get_loss('val')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "865b7bbd",
   "metadata": {},
   "source": [
    "### Let's reduce even more"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 346,
   "id": "e5c01d9d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9367"
      ]
     },
     "execution_count": 346,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m = Model({\n",
    "    'names': names,\n",
    "    'embedding_depth': 20,\n",
    "    'n_hidden_layer': 100,\n",
    "    'block_size': 3,\n",
    "    'batch_size': 32\n",
    "})\n",
    "m.n_elements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 347,
   "id": "ff2caa6a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.12813401222229"
      ]
     },
     "execution_count": 347,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.train_minibatches(100000, 0.1)\n",
    "m.train_minibatches(100000, 0.01)\n",
    "m.get_loss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 348,
   "id": "5abc47f2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.160646677017212"
      ]
     },
     "execution_count": 348,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.get_loss('val')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "668f99ef",
   "metadata": {},
   "source": [
    "### Let's keep it at 150, and increase block_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 349,
   "id": "4ccd36c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19767"
      ]
     },
     "execution_count": 349,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m = Model({\n",
    "    'names': names,\n",
    "    'embedding_depth': 20,\n",
    "    'n_hidden_layer': 150,\n",
    "    'block_size': 5,\n",
    "    'batch_size': 32\n",
    "})\n",
    "m.n_elements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 350,
   "id": "ecc57641",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.169311046600342"
      ]
     },
     "execution_count": 350,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.train_minibatches(100000, 0.1)\n",
    "m.train_minibatches(100000, 0.01)\n",
    "m.get_loss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 351,
   "id": "2b686fce",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.2097270488739014"
      ]
     },
     "execution_count": 351,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.get_loss('val')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a356789",
   "metadata": {},
   "source": [
    "### Longer training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 352,
   "id": "567e4a71",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.118191957473755"
      ]
     },
     "execution_count": 352,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m = Model({\n",
    "    'names': names,\n",
    "    'embedding_depth': 20,\n",
    "    'n_hidden_layer': 150,\n",
    "    'block_size': 5,\n",
    "    'batch_size': 32\n",
    "})\n",
    "m.train_minibatches(200000, 0.1)\n",
    "m.train_minibatches(200000, 0.01)\n",
    "m.get_loss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "id": "a283cc30",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.178915023803711"
      ]
     },
     "execution_count": 353,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.get_loss('val')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64b54df9",
   "metadata": {},
   "source": [
    "### Even longer training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 441,
   "id": "09fc369c",
   "metadata": {},
   "outputs": [],
   "source": [
    "m = Model({\n",
    "    'names': names,\n",
    "    'embedding_depth': 20,\n",
    "    'n_hidden_layer': 150,\n",
    "    'block_size': 5,\n",
    "    'batch_size': 32\n",
    "})\n",
    "m.train_minibatches(200000, 0.1)\n",
    "m.train_minibatches(400000, 0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 442,
   "id": "376123a9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2.111159086227417, 2.174161911010742)"
      ]
     },
     "execution_count": 442,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.get_loss(), m.get_loss('val')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f69a140",
   "metadata": {},
   "source": [
    "### Only small improvement.. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 444,
   "id": "4a7c6a66",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "26182"
      ]
     },
     "execution_count": 444,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m = Model({\n",
    "    'names': names,\n",
    "    'embedding_depth': 15,\n",
    "    'n_hidden_layer': 250,\n",
    "    'block_size': 5,\n",
    "    'batch_size': 32\n",
    "})\n",
    "m.n_elements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 445,
   "id": "be1acb91",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2.423304319381714, 2.456299066543579)"
      ]
     },
     "execution_count": 445,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.train_minibatches(100000, 0.1)\n",
    "m.get_loss(), m.get_loss('val')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22e234bf",
   "metadata": {},
   "source": [
    "### let's get back to the best model so far:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 358,
   "id": "4a93cdf7",
   "metadata": {},
   "outputs": [],
   "source": [
    "m = Model({\n",
    "    'names': names,\n",
    "    'embedding_depth': 20,\n",
    "    'n_hidden_layer': 150,\n",
    "    'block_size': 3,\n",
    "    'batch_size': 32\n",
    "})\n",
    "m.n_elements\n",
    "\n",
    "m.train_minibatches(50000, 0.1)\n",
    "m.train_minibatches(50000, 0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 359,
   "id": "1a571d4a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2.128970146179199, 2.1740517616271973)"
      ]
     },
     "execution_count": 359,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.get_loss(), m.get_loss('val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 360,
   "id": "89332d7d",
   "metadata": {},
   "outputs": [],
   "source": [
    "m = Model({\n",
    "    'names': names,\n",
    "    'embedding_depth': 20,\n",
    "    'n_hidden_layer': 150,\n",
    "    'block_size': 3,\n",
    "    'batch_size': 32\n",
    "})\n",
    "m.n_elements\n",
    "\n",
    "m.train_minibatches(150000, 0.1)\n",
    "m.train_minibatches(150000, 0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 361,
   "id": "342186bf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2.070401191711426, 2.1311147212982178)"
      ]
     },
     "execution_count": 361,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.get_loss(), m.get_loss('val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 362,
   "id": "d26c8258",
   "metadata": {},
   "outputs": [],
   "source": [
    "m.train_minibatches(100000, 0.001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 363,
   "id": "ef54b8fe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2.061901807785034, 2.125061273574829)"
      ]
     },
     "execution_count": 363,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.get_loss(), m.get_loss('val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 364,
   "id": "80220804",
   "metadata": {},
   "outputs": [],
   "source": [
    "m.train_minibatches(100000, 0.001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 365,
   "id": "f885f204",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2.0613131523132324, 2.1244380474090576)"
      ]
     },
     "execution_count": 365,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.get_loss(), m.get_loss('val')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a184980",
   "metadata": {},
   "source": [
    "### Larger n_hidden_layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 366,
   "id": "bd1bf724",
   "metadata": {},
   "outputs": [],
   "source": [
    "m = Model({\n",
    "    'names': names,\n",
    "    'embedding_depth': 20,\n",
    "    'n_hidden_layer': 200,\n",
    "    'block_size': 3,\n",
    "    'batch_size': 32\n",
    "})\n",
    "m.n_elements\n",
    "\n",
    "m.train_minibatches(100000, 0.1)\n",
    "m.train_minibatches(100000, 0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 367,
   "id": "8c6f1902",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2.068345308303833, 2.1444251537323)"
      ]
     },
     "execution_count": 367,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.get_loss(), m.get_loss('val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "333081b5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4dcf12de",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 400,
   "id": "1d5a3197",
   "metadata": {},
   "outputs": [],
   "source": [
    "m = Model({\n",
    "    'names': names,\n",
    "    'embedding_depth': 20,\n",
    "    'n_hidden_layer': 150,\n",
    "    'block_size': 3,\n",
    "    'batch_size': 32\n",
    "})\n",
    "m.n_elements\n",
    "\n",
    "m.train_minibatches(100000, 0.1)\n",
    "m.train_minibatches(100000, 0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 401,
   "id": "1b4f78b3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2.0878281593322754, 2.1424944400787354)"
      ]
     },
     "execution_count": 401,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.get_loss(), m.get_loss('val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 402,
   "id": "1612bcb0",
   "metadata": {},
   "outputs": [],
   "source": [
    "m.train_minibatches(100000, 0.001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 403,
   "id": "b54311bb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2.079813241958618, 2.1356570720672607)"
      ]
     },
     "execution_count": 403,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.get_loss(), m.get_loss('val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 404,
   "id": "f37b5f68",
   "metadata": {},
   "outputs": [],
   "source": [
    "m.train_minibatches(100000, 0.001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 405,
   "id": "17ad6b69",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2.079188108444214, 2.1362316608428955)"
      ]
     },
     "execution_count": 405,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.get_loss(), m.get_loss('val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6399138c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8b06371",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 417,
   "id": "f8929047",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "23.754575729370117"
      ]
     },
     "execution_count": 417,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m = Model({\n",
    "    'names': names,\n",
    "    'embedding_depth': 20,\n",
    "    'n_hidden_layer': 150,\n",
    "    'block_size': 3,\n",
    "    'batch_size': 32\n",
    "})\n",
    "m.get_loss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 436,
   "id": "1e2e026b",
   "metadata": {},
   "outputs": [],
   "source": [
    "m = Model({\n",
    "    'names': names,\n",
    "    'embedding_depth': 20,\n",
    "    'n_hidden_layer': 150,\n",
    "    'block_size': 3,\n",
    "    'batch_size': 32\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 437,
   "id": "a25c35de",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(23.754575729370117, 23.730976104736328)"
      ]
     },
     "execution_count": 437,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.get_loss(), m.get_loss('val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 438,
   "id": "a292af26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(23.754575729370117, 23.730976104736328)"
      ]
     },
     "execution_count": 438,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.get_loss(), m.get_loss('val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 439,
   "id": "7826b34a",
   "metadata": {},
   "outputs": [],
   "source": [
    "m.train_minibatches(100000, 0.1)\n",
    "m.train_minibatches(100000, 0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 440,
   "id": "ca002e5a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2.0878281593322754, 2.1424944400787354)"
      ]
     },
     "execution_count": 440,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.get_loss(), m.get_loss('val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35911665",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}