PyTorch seq2seq template based on the g2p task.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We'll write a simple template for seq2seq using PyTorch. For demonstration, we attack the g2p task. G2p is a task of converting graphemes (spelling) to phonemes (pronunciation). It's a very good source for this purpose as it's simple enough for you to up and run. If you want to know more about g2p, see my [repo](https:/github.com/kyubyong/g2p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "__author__ = \"kyubyong\"\n",
    "__address__ = \"https://github.com/kyubyong/nlp_made_easy\"\n",
    "__email__ = \"kbpark.linguist@gmail.com\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "TEXqpZ_U738q"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "from distance import levenshtein\n",
    "import os\n",
    "import math\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.utils import data\n",
    "from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "_7vuctbU7381",
    "outputId": "f8ee2cbf-1f04-432f-ba42-d25fec61669b"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.2.0'"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "B6te4HKk738_"
   },
   "source": [
    "# Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "CWS2hkce739C"
   },
   "outputs": [],
   "source": [
    "class Hparams:\n",
    "    batch_size = 128\n",
    "    enc_maxlen = 20\n",
    "    dec_maxlen = 20\n",
    "    num_epochs = 10\n",
    "    hidden_units = 128\n",
    "    emb_units = 64\n",
    "    graphemes = [\"<pad>\", \"<unk>\", \"</s>\"] + list(\"abcdefghijklmnopqrstuvwxyz\")\n",
    "    phonemes = [\"<pad>\", \"<unk>\", \"<s>\", \"</s>\"] + ['AA0', 'AA1', 'AA2', 'AE0', 'AE1', 'AE2', 'AH0', 'AH1', 'AH2', 'AO0',\n",
    "                    'AO1', 'AO2', 'AW0', 'AW1', 'AW2', 'AY0', 'AY1', 'AY2', 'B', 'CH', 'D', 'DH',\n",
    "                    'EH0', 'EH1', 'EH2', 'ER0', 'ER1', 'ER2', 'EY0', 'EY1', 'EY2', 'F', 'G', 'HH',\n",
    "                    'IH0', 'IH1', 'IH2', 'IY0', 'IY1', 'IY2', 'JH', 'K', 'L', 'M', 'N', 'NG', 'OW0', 'OW1',\n",
    "                    'OW2', 'OY0', 'OY1', 'OY2', 'P', 'R', 'S', 'SH', 'T', 'TH', 'UH0', 'UH1', 'UH2', 'UW',\n",
    "                    'UW0', 'UW1', 'UW2', 'V', 'W', 'Y', 'Z', 'ZH']\n",
    "    lr = 0.001\n",
    "    logdir = \"log/01\"\n",
    "hp = Hparams()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "nz-hD6dn739L"
   },
   "source": [
    "# Prepare Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-as4PHs-739N"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['R', 'AH0', 'F', 'Y', 'UW1', 'Z'],\n",
       " ['R', 'EH1', 'F', 'Y', 'UW2', 'Z'],\n",
       " ['R', 'IH0', 'F', 'Y', 'UW1', 'Z']]"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "# nltk.download('cmudict')# <- if you haven't downloaded, do this.\n",
    "from nltk.corpus import cmudict\n",
    "cmu = cmudict.dict()\n",
    "cmu[\"refuse\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "39gQ3vOi739S"
   },
   "outputs": [],
   "source": [
    "def load_vocab():\n",
    "    g2idx = {g: idx for idx, g in enumerate(hp.graphemes)}\n",
    "    idx2g = {idx: g for idx, g in enumerate(hp.graphemes)}\n",
    "\n",
    "    p2idx = {p: idx for idx, p in enumerate(hp.phonemes)}\n",
    "    idx2p = {idx: p for idx, p in enumerate(hp.phonemes)}\n",
    "\n",
    "    return g2idx, idx2g, p2idx, idx2p # note that g and p mean grapheme and phoneme, respectively.\n",
    "\n",
    "g2idx, idx2g, p2idx, idx2p = load_vocab()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Zslytxn6739Z"
   },
   "outputs": [],
   "source": [
    "def prepare_data():\n",
    "    words = [\" \".join(list(word)) for word, prons in cmu.items()]\n",
    "    prons = [\" \".join(prons[0]) for word, prons in cmu.items()]\n",
    "    indices = list(range(len(words)))\n",
    "    from random import shuffle\n",
    "    shuffle(indices)\n",
    "    words = [words[idx] for idx in indices]\n",
    "    prons = [prons[idx] for idx in indices]\n",
    "    num_train, num_test = int(len(words)*.8), int(len(words)*.1)\n",
    "    train_words, eval_words, test_words = words[:num_train], \\\n",
    "                                          words[num_train:-num_test],\\\n",
    "                                          words[-num_test:]\n",
    "    train_prons, eval_prons, test_prons = prons[:num_train], \\\n",
    "                                          prons[num_train:-num_test],\\\n",
    "                                          prons[-num_test:]    \n",
    "    return train_words, eval_words, test_words, train_prons, eval_prons, test_prons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WHBXkAPG739j"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "f l a p j a c k\n",
      "F L AE1 P JH AE2 K\n"
     ]
    }
   ],
   "source": [
    "train_words, eval_words, test_words, train_prons, eval_prons, test_prons = prepare_data()\n",
    "print(train_words[0])\n",
    "print(train_prons[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "def drop_lengthy_samples(words, prons, enc_maxlen, dec_maxlen):\n",
    "    \"\"\"We only include such samples less than maxlen.\"\"\"\n",
    "    _words, _prons = [], []\n",
    "    for w, p in zip(words, prons):\n",
    "        if len(w.split()) + 1 > enc_maxlen: continue\n",
    "        if len(p.split()) + 1 > dec_maxlen: continue # 1: <EOS>\n",
    "        _words.append(w)\n",
    "        _prons.append(p)\n",
    "    return _words, _prons          "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_words, train_prons = drop_lengthy_samples(train_words, train_prons, hp.enc_maxlen, hp.dec_maxlen)\n",
    "# We do NOT apply this constraint to eval and test datasets."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "def encode(inp, type, dict):\n",
    "    '''convert string into ids\n",
    "    type: \"x\" or \"y\"\n",
    "    dict: g2idx for 'x', p2idx for 'y'\n",
    "    '''\n",
    "    if type==\"x\": tokens = inp.split() + [\"</s>\"]\n",
    "    else: tokens = [\"<s>\"] + inp.split() + [\"</s>\"]\n",
    "\n",
    "    x = [dict.get(t, dict[\"<unk>\"]) for t in tokens]\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "class G2pDataset(data.Dataset):\n",
    "\n",
    "    def __init__(self, words, prons):\n",
    "        \"\"\"\n",
    "        words: list of words. e.g., [\"w o r d\", ]\n",
    "        prons: list of prons. e.g., ['W ER1 D',]\n",
    "        \"\"\"\n",
    "        self.words = words\n",
    "        self.prons = prons\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.words)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        word, pron = self.words[idx], self.prons[idx]\n",
    "        x = encode(word, \"x\", g2idx)\n",
    "        y = encode(pron, \"y\", p2idx)\n",
    "        decoder_input, y = y[:-1], y[1:]\n",
    "\n",
    "        x_seqlen, y_seqlen = len(x), len(y)\n",
    "                \n",
    "        return x, x_seqlen, word, decoder_input, y, y_seqlen, pron"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad(batch):\n",
    "    '''Pads zeros such that the length of all samples in a batch is the same.'''\n",
    "    f = lambda x: [sample[x] for sample in batch]\n",
    "    x_seqlens = f(1)\n",
    "    y_seqlens = f(5)\n",
    "    words = f(2)\n",
    "    prons = f(-1)\n",
    "    \n",
    "    x_maxlen = np.array(x_seqlens).max()\n",
    "    y_maxlen = np.array(y_seqlens).max()\n",
    "    \n",
    "    f = lambda x, maxlen, batch: [sample[x]+[0]*(maxlen-len(sample[x])) for sample in batch]\n",
    "    x = f(0, x_maxlen, batch)\n",
    "    decoder_inputs = f(3, y_maxlen, batch)\n",
    "    y = f(4, y_maxlen, batch)\n",
    "    \n",
    "    f = torch.LongTensor\n",
    "    return f(x), x_seqlens, words, f(decoder_inputs), f(y), y_seqlens, prons"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "22mif4xf73-M"
   },
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder(nn.Module):\n",
    "    global g2idx, idx2g, p2idx, idx2p\n",
    "    def __init__(self, emb_units, hidden_units):\n",
    "        super().__init__()\n",
    "        self.emb_units = emb_units\n",
    "        self.hidden_units = hidden_units\n",
    "        self.emb = nn.Embedding(len(g2idx), emb_units)\n",
    "        self.rnn = nn.GRU(emb_units, hidden_units, batch_first=True)\n",
    "        \n",
    "    def forward(self, x, seqlens):\n",
    "        x = self.emb(x)\n",
    "            \n",
    "        # packing -> rnn -> unpacking -> position recovery: note that enforce_sorted is set to False.\n",
    "        packed_input = pack_padded_sequence(x, seqlens, batch_first=True, enforce_sorted=False)   \n",
    "        outputs, last_hidden = self.rnn(packed_input)\n",
    "#         outputs, _ = pad_packed_sequence(outputs, batch_first=True, total_length=x.size()[1])\n",
    "\n",
    "        # last hidden\n",
    "        last_hidden = last_hidden.permute(1, 2, 0)\n",
    "        last_hidden = last_hidden.view(last_hidden.size()[0], -1)\n",
    "        \n",
    "        return last_hidden\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Decoder(nn.Module):\n",
    "    global g2idx, idx2g, p2idx, idx2p\n",
    "    def __init__(self, emb_units, hidden_units):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.emb_units = emb_units\n",
    "        self.hidden_units = hidden_units\n",
    "        self.emb = nn.Embedding(len(p2idx), emb_units)\n",
    "        self.rnn = nn.GRU(emb_units, hidden_units, batch_first=True)\n",
    "        self.fc = nn.Linear(hidden_units, len(p2idx))\n",
    "        \n",
    "    def forward(self, decoder_inputs, h0):\n",
    "        decoder_inputs = self.emb(decoder_inputs)\n",
    "           \n",
    "        outputs, last_hidden = self.rnn(decoder_inputs, h0)\n",
    "        logits = self.fc(outputs) # (N, T, V)\n",
    "        y_hat = logits.argmax(-1)\n",
    "        \n",
    "        return logits, y_hat, last_hidden\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "HA39FU4-73-O"
   },
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    global g2idx, idx2g, p2idx, idx2p\n",
    "    \n",
    "    def __init__(self, encoder, decoder): \n",
    "        super().__init__()\n",
    "        self.encoder = encoder\n",
    "        self.decoder = decoder\n",
    "    \n",
    "    def forward(self, x, seqlens, decoder_inputs, teacher_forcing=True, dec_maxlen=None):  \n",
    "        '''\n",
    "        At training, decoder inputs (ground truth) and teacher forcing is applied. \n",
    "        At evaluation, decoder inputs are ignored, and the decoding keeps for `dec_maxlen` steps.\n",
    "        '''\n",
    "        last_hidden = self.encoder(x, seqlens)\n",
    "        h0 = last_hidden.unsqueeze(0)\n",
    "        \n",
    "        if teacher_forcing: # training\n",
    "            logits, y_hat, h0 = self.decoder(decoder_inputs, h0)\n",
    "        else: # evaluation\n",
    "            decoder_inputs = decoder_inputs[:, :1] # \"<s>\"\n",
    "            logits, y_hat = [], []\n",
    "            for t in range(dec_maxlen):\n",
    "                _logits, _y_hat, h0 =self.decoder(decoder_inputs, h0) # _logits: (N, 1, V), _y_hat: (N, 1), h0: (1, N, N)\n",
    "                logits.append(_logits)\n",
    "                y_hat.append(_y_hat)\n",
    "                decoder_inputs = _y_hat\n",
    "        \n",
    "            logits = torch.cat(logits, 1)\n",
    "            y_hat = torch.cat(y_hat, 1)\n",
    "        \n",
    "        return logits, y_hat\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train & Eval functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, iterator, optimizer, criterion, device):\n",
    "    model.train()\n",
    "    for i, batch in enumerate(iterator):\n",
    "        x, x_seqlens, words, decoder_inputs, y, y_seqlens, prons = batch\n",
    "        \n",
    "        x, decoder_inputs = x.to(device), decoder_inputs.to(device) \n",
    "        y = y.to(device)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        logits, y_hat = model(x, x_seqlens, decoder_inputs)\n",
    "        \n",
    "        # calc loss\n",
    "        logits = logits.view(-1, logits.shape[-1]) # (N*T, VOCAB)\n",
    "        y = y.view(-1) # (N*T,)\n",
    "        loss = criterion(logits, y)\n",
    "        loss.backward()\n",
    "        \n",
    "        optimizer.step()\n",
    "        \n",
    "        if i and i%100==0:\n",
    "            print(f\"step: {i}, loss: {loss.item()}\")\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calc_per(Y_true, Y_pred):\n",
    "    '''Calc phoneme error rate\n",
    "    Y_true: list of predicted phoneme sequences. e.g., [[\"B\", \"L\", \"AA1\", \"K\", \"HH\", \"AW2\", \"S\"], ...]\n",
    "    Y_pred: list of ground truth phoneme sequences. e.g., [[\"B\", \"L\", \"AA1\", \"K\", \"HH\", \"AW2\", \"S\"], ...]\n",
    "    '''\n",
    "    num_phonemes, num_erros = 0, 0\n",
    "    for y_true, y_pred in zip(Y_true, Y_pred):\n",
    "        num_phonemes += len(y_true)\n",
    "        num_erros += levenshtein(y_true, y_pred)\n",
    "\n",
    "    per = round(num_erros / num_phonemes, 2)\n",
    "    return per, num_erros"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_ids_to_phonemes(ids, idx2p):\n",
    "    phonemes = []\n",
    "    for idx in ids:\n",
    "        if idx == 3: # 3: </s>\n",
    "            break\n",
    "        p = idx2p[idx]\n",
    "        phonemes.append(p)\n",
    "    return phonemes\n",
    "        \n",
    "            \n",
    "\n",
    "def eval(model, iterator, device, dec_maxlen):\n",
    "    model.eval()\n",
    "\n",
    "    Y_true, Y_pred = [], []\n",
    "    with torch.no_grad():\n",
    "        for i, batch in enumerate(iterator):\n",
    "            x, x_seqlens, words, decoder_inputs, y, y_seqlens, prons = batch\n",
    "            x, decoder_inputs = x.to(device), decoder_inputs.to(device) \n",
    "\n",
    "            _, y_hat = model(x, x_seqlens, decoder_inputs, False, dec_maxlen) # <- teacher forcing is suppressed.\n",
    "            \n",
    "            y = y.to('cpu').numpy().tolist()\n",
    "            y_hat = y_hat.to('cpu').numpy().tolist()\n",
    "            for yy, yy_hat in zip(y, y_hat):\n",
    "                y_true = convert_ids_to_phonemes(yy, idx2p)\n",
    "                y_pred = convert_ids_to_phonemes(yy_hat, idx2p)\n",
    "                Y_true.append(y_true)\n",
    "                Y_pred.append(y_pred)\n",
    "    \n",
    "    # calc per.\n",
    "    per, num_errors = calc_per(Y_true, Y_pred)\n",
    "    print(\"per: %.2f\" % per, \"num errors: \", num_errors)\n",
    "    \n",
    "    with open(\"result\", \"w\") as fout:\n",
    "        for y_true, y_pred in zip(Y_true, Y_pred):\n",
    "            fout.write(\" \".join(y_true) + \"\\n\")\n",
    "            fout.write(\" \".join(y_pred) + \"\\n\\n\")\n",
    "    \n",
    "    return per\n",
    "            "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "PKllLnfp73-V"
   },
   "source": [
    "# Train & Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = G2pDataset(train_words, train_prons)\n",
    "eval_dataset = G2pDataset(eval_words, eval_prons)\n",
    "\n",
    "train_iter = data.DataLoader(train_dataset, batch_size=hp.batch_size, shuffle=True, collate_fn=pad)\n",
    "eval_iter = data.DataLoader(eval_dataset, batch_size=hp.batch_size, shuffle=False, collate_fn=pad)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 121
    },
    "colab_type": "code",
    "id": "aF0cJceg73-m",
    "outputId": "ab78b80a-d6e3-4408-af21-acff58944a79",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 1\n",
      "step: 100, loss: 2.729764461517334\n",
      "step: 200, loss: 2.06953763961792\n",
      "step: 300, loss: 1.6415880918502808\n",
      "step: 400, loss: 1.3378574848175049\n",
      "step: 500, loss: 1.2205088138580322\n",
      "step: 600, loss: 0.987713098526001\n",
      "step: 700, loss: 0.9300493597984314\n",
      "per: 0.39 num errors:  30495\n",
      "\n",
      "epoch: 2\n",
      "step: 100, loss: 0.862144947052002\n",
      "step: 200, loss: 0.8274500370025635\n",
      "step: 300, loss: 0.7277541160583496\n",
      "step: 400, loss: 0.8145508766174316\n",
      "step: 500, loss: 0.6055648922920227\n",
      "step: 600, loss: 0.6670782566070557\n",
      "step: 700, loss: 0.7308872938156128\n",
      "per: 0.30 num errors:  23436\n",
      "\n",
      "epoch: 3\n",
      "step: 100, loss: 0.6280044913291931\n",
      "step: 200, loss: 0.6292356848716736\n",
      "step: 300, loss: 0.6315799951553345\n",
      "step: 400, loss: 0.6083210110664368\n",
      "step: 500, loss: 0.5998032093048096\n",
      "step: 600, loss: 0.634599506855011\n",
      "step: 700, loss: 0.6139586567878723\n",
      "per: 0.25 num errors:  19908\n",
      "\n",
      "epoch: 4\n",
      "step: 100, loss: 0.5246961116790771\n",
      "step: 200, loss: 0.4894694685935974\n",
      "step: 300, loss: 0.5112945437431335\n",
      "step: 400, loss: 0.42986905574798584\n",
      "step: 500, loss: 0.598124623298645\n",
      "step: 600, loss: 0.4181916415691376\n",
      "step: 700, loss: 0.4648551046848297\n",
      "per: 0.23 num errors:  18315\n",
      "\n",
      "epoch: 5\n",
      "step: 100, loss: 0.5175663232803345\n",
      "step: 200, loss: 0.429765909910202\n",
      "step: 300, loss: 0.40104925632476807\n",
      "step: 400, loss: 0.5041416883468628\n",
      "step: 500, loss: 0.4927402436733246\n",
      "step: 600, loss: 0.4579430818557739\n",
      "step: 700, loss: 0.4033070504665375\n",
      "per: 0.22 num errors:  17284\n",
      "\n",
      "epoch: 6\n",
      "step: 100, loss: 0.40969985723495483\n",
      "step: 200, loss: 0.4264414310455322\n",
      "step: 300, loss: 0.4089752435684204\n",
      "step: 400, loss: 0.4005317986011505\n",
      "step: 500, loss: 0.46420779824256897\n",
      "step: 600, loss: 0.3897724449634552\n",
      "step: 700, loss: 0.3946235775947571\n",
      "per: 0.21 num errors:  16358\n",
      "\n",
      "epoch: 7\n",
      "step: 100, loss: 0.36400753259658813\n",
      "step: 200, loss: 0.43446823954582214\n",
      "step: 300, loss: 0.419318288564682\n",
      "step: 400, loss: 0.3300051987171173\n",
      "step: 500, loss: 0.4172106385231018\n",
      "step: 600, loss: 0.4165189862251282\n",
      "step: 700, loss: 0.37570977210998535\n",
      "per: 0.20 num errors:  15657\n",
      "\n",
      "epoch: 8\n",
      "step: 100, loss: 0.28185898065567017\n",
      "step: 200, loss: 0.3737757205963135\n",
      "step: 300, loss: 0.31888020038604736\n",
      "step: 400, loss: 0.31222569942474365\n",
      "step: 500, loss: 0.36826616525650024\n",
      "step: 600, loss: 0.32874614000320435\n",
      "step: 700, loss: 0.41009703278541565\n",
      "per: 0.19 num errors:  14824\n",
      "\n",
      "epoch: 9\n",
      "step: 100, loss: 0.3402358591556549\n",
      "step: 200, loss: 0.34929829835891724\n",
      "step: 300, loss: 0.3066072463989258\n",
      "step: 400, loss: 0.403970867395401\n",
      "step: 500, loss: 0.39007294178009033\n",
      "step: 600, loss: 0.36635467410087585\n",
      "step: 700, loss: 0.371225506067276\n",
      "per: 0.19 num errors:  14593\n",
      "\n",
      "epoch: 10\n",
      "step: 100, loss: 0.3137436509132385\n",
      "step: 200, loss: 0.3691956102848053\n",
      "step: 300, loss: 0.3513341546058655\n",
      "step: 400, loss: 0.32838159799575806\n",
      "step: 500, loss: 0.2848772406578064\n",
      "step: 600, loss: 0.3028918504714966\n",
      "step: 700, loss: 0.41611844301223755\n",
      "per: 0.18 num errors:  14374\n"
     ]
    }
   ],
   "source": [
    "encoder = Encoder(hp.emb_units, hp.hidden_units)\n",
    "decoder = Decoder(hp.emb_units, hp.hidden_units)\n",
    "model = Net(encoder, decoder)\n",
    "model.to(device)\n",
    "\n",
    "optimizer = optim.Adam(model.parameters(), lr = hp.lr)\n",
    "criterion = nn.CrossEntropyLoss(ignore_index=0)\n",
    "\n",
    "for epoch in range(1, hp.num_epochs+1):\n",
    "    print(f\"\\nepoch: {epoch}\")\n",
    "    train(model, train_iter, optimizer, criterion, device)\n",
    "    eval(model, eval_iter, device, hp.dec_maxlen)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "82t4Dmwp73--"
   },
   "source": [
    "# Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "jUyYlI4S73_O",
    "outputId": "ae0592d3-14b0-4f3b-94f8-ebc293c48304"
   },
   "outputs": [],
   "source": [
    "test_dataset = G2pDataset(test_words, test_prons)\n",
    "test_iter = data.DataLoader(test_dataset, batch_size=hp.batch_size, shuffle=False, collate_fn=pad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "per: 0.18 num errors:  14045\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.18"
      ]
     },
     "execution_count": 245,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval(model, test_iter, device, hp.dec_maxlen)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check the results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['',\n",
       " 'G L IY1 M D',\n",
       " 'G L IY1 M D',\n",
       " '',\n",
       " 'P EY1 D AH0 N',\n",
       " 'P EY1 D AH0 N',\n",
       " '',\n",
       " 'B L UW1 N AH0 S',\n",
       " 'B L UW1 N AH0 S',\n",
       " '',\n",
       " 'HH OW1 L B R UH0 K S',\n",
       " 'HH OW1 L B R UH2 K S',\n",
       " '',\n",
       " 'B AE1 R IH0 S T ER0 Z',\n",
       " 'B AE1 R IH0 S T ER0 Z',\n",
       " '',\n",
       " 'P EH1 L T',\n",
       " 'P EH1 L T',\n",
       " '',\n",
       " 'M AA1 R K AH0 L',\n",
       " 'M AA1 R K AH0 L',\n",
       " '',\n",
       " 'F EY1 G ER0 S T R AH0 M',\n",
       " 'F EY1 G ER0 S T R AH0 M',\n",
       " '',\n",
       " 'P EH1 R AH0 SH UW2 T',\n",
       " 'P EH1 R AH0 CH UW0 T',\n",
       " '',\n",
       " 'B EH2 L W OW0 M IY1 N IY0',\n",
       " 'B EH0 L UW1 M IY0 N IY0',\n",
       " '',\n",
       " 'L AA1 HH AH0 V IH0 CH',\n",
       " 'L AH0 SH OW1 IH0 K S',\n",
       " '',\n",
       " 'F EY1 G AH0 N',\n",
       " 'F AE1 G AH0 N',\n",
       " '',\n",
       " 'P IH1 S ER0 EH0 K',\n",
       " 'P IH0 S AA1 R EH0 K',\n",
       " '',\n",
       " 'R IY1 D ER0 M AH0 N',\n",
       " 'R IY1 D ER0 M AH0 N',\n",
       " '',\n",
       " 'K AA1 K AH0 T UW2 Z',\n",
       " 'K AH0 K EY1 T OW0 Z',\n",
       " '',\n",
       " 'R IY0 B AH1 F IH0 NG',\n",
       " 'R IH0 F AH1 B IH0 NG',\n",
       " '',\n",
       " 'S AW1 TH D AW2 N',\n",
       " 'S AW1 TH D AW2 N',\n",
       " '',\n",
       " 'B AE1 L AH0 N T R EY2',\n",
       " 'B AE1 L AH0 N T R EY2',\n",
       " '',\n",
       " 'S L OW1 P S',\n",
       " 'S L OW1 P S',\n",
       " '',\n",
       " 'V AE1 N D ER0 V L IY2 T',\n",
       " 'V AE1 N D ER0 V L AY2 T',\n",
       " '',\n",
       " 'F AY1 R B AA2 M D',\n",
       " 'F AY1 ER0 B OW2 M B AH0 JH',\n",
       " '',\n",
       " 'P AH0 L UW1 T ER0',\n",
       " 'P AA1 L AH0 T ER0',\n",
       " '',\n",
       " 'D AO1 F IH0 NG',\n",
       " 'D AO1 F IH0 NG',\n",
       " '',\n",
       " 'P AE1 L K OW0',\n",
       " 'P AE1 L K OW0',\n",
       " '',\n",
       " 'SH AH0 HH IH1 N IY0 AH0 N',\n",
       " 'SH AH0 HH IY1 N IY0 AH0 N',\n",
       " '',\n",
       " 'K L EH1 N CH',\n",
       " 'K L EH1 N CH',\n",
       " '',\n",
       " 'P AE1 S AH0 B L IY0',\n",
       " 'P AE1 S AH0 B L IY0',\n",
       " '',\n",
       " 'JH AA1 S AH0 L D',\n",
       " 'JH AA1 S T AH0 L',\n",
       " '',\n",
       " 'AA1 M N IH0 B UH2 K',\n",
       " 'AA1 M N IH0 B UH2 K',\n",
       " '',\n",
       " 'K L IH1 N IH0 K S',\n",
       " 'K L IH1 N IH0 K S',\n",
       " '',\n",
       " 'K AA1 N R IY0',\n",
       " 'K AA1 N R IY0',\n",
       " '',\n",
       " 'R AO1 F',\n",
       " 'R AO1 F',\n",
       " '',\n",
       " 'AH0 M AE1 N AH0',\n",
       " 'AH0 M AA1 N AH0',\n",
       " '']"
      ]
     },
     "execution_count": 246,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "open('result', 'r').read().splitlines()[-100:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "name": "Seq2seq tutorial with g2p.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}