08.1 Sequence to sequence mapping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from gutenberg.acquire import load_etext\n",
    "from gutenberg.cleanup import strip_headers\n",
    "\n",
    "import nltk\n",
    "from nltk.corpus import wordnet as wn\n",
    "import inflect\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras import layers\n",
    "import numpy as np\n",
    "from collections import Counter, defaultdict\n",
    "\n",
    "from gensim.utils import tokenize\n",
    "from itertools import groupby\n",
    "\n",
    "from keras.models import Input, Model\n",
    "from keras.layers import Dense, Dropout\n",
    "from keras.layers import LSTM, RepeatVector\n",
    "from keras.layers.wrappers import TimeDistributed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "p = inflect.engine()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "67176"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pairs = {}\n",
    "for synset in wn.all_synsets('n'):\n",
    "    word = synset.name().split('.', 1)[0]\n",
    "    if not word in pairs:\n",
    "        pairs[word] = p.plural(word)\n",
    "len(pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('data/plurals.txt', 'w') as fout:\n",
    "    for k in sorted(pairs):\n",
    "        if '_' in k or '-' in k:\n",
    "            continue\n",
    "        if k.isdigit():\n",
    "            continue\n",
    "        fout.write('%s\\t%s\\n' % (k, pairs[k]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'noes'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p.plural('no')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class CharacterTable(object):\n",
    "    \"\"\"Given a set of characters:\n",
    "    + Encode them to a one hot integer representation\n",
    "    + Decode the one hot integer representation to their character output\n",
    "    + Decode a vector of probabilities to their character output\n",
    "    \"\"\"\n",
    "    def __init__(self, chars):\n",
    "        \"\"\"Initialize character table.\n",
    "        # Arguments\n",
    "            chars: Characters that can appear in the input.\n",
    "        \"\"\"\n",
    "        self.chars = sorted(set(chars))\n",
    "        self.char_indices = dict((c, i) for i, c in enumerate(self.chars))\n",
    "        self.indices_char = dict((i, c) for i, c in enumerate(self.chars))\n",
    "\n",
    "    def encode(self, C, num_rows):\n",
    "        \"\"\"One hot encode given string C.\n",
    "        # Arguments\n",
    "            num_rows: Number of rows in the returned one hot encoding. This is\n",
    "                used to keep the # of rows for each data the same.\n",
    "        \"\"\"\n",
    "        x = np.zeros((num_rows, len(self.chars)))\n",
    "        for i, c in enumerate(C):\n",
    "            x[i, self.char_indices[c]] = 1\n",
    "        return x\n",
    "\n",
    "    def decode(self, x, calc_argmax=True):\n",
    "        if calc_argmax:\n",
    "            x = x.argmax(axis=-1)\n",
    "        return ''.join(self.indices_char[x] for x in x)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Parameters for the model and dataset.\n",
    "INVERT = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total addition questions: 39929\n"
     ]
    }
   ],
   "source": [
    "questions = []\n",
    "expected = []\n",
    "seen = set()\n",
    "#with open('data/en_de.txt') as fin:\n",
    "with open('data/plurals.txt') as fin:\n",
    "    for line in fin:\n",
    "        en, de = line.strip().split('\\t')\n",
    "        questions.append(en)\n",
    "        expected.append(de)\n",
    "\n",
    "max_question_len = max(len(q) for q in questions)\n",
    "max_expected_len = max(len(e) for e in expected)\n",
    "questions = [' ' * (max_question_len - len(q)) + q for q in questions]\n",
    "expected = [e + ' ' * (max_expected_len - len(e)) for e in expected]\n",
    "if INVERT:\n",
    "    questions = [q[::-1] for q in questions]\n",
    "\n",
    "print('Total addition questions:', len(questions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "40"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chars = set(ch for k, v in zip(questions, expected) for ch in k + v)\n",
    "ctable = CharacterTable(chars)\n",
    "len(chars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vectorization...\n",
      "done\n"
     ]
    }
   ],
   "source": [
    "print('Vectorization...')\n",
    "x = np.zeros((len(questions), max_question_len, len(chars)), dtype=np.bool)\n",
    "y = np.zeros((len(questions), max_expected_len, len(chars)), dtype=np.bool)\n",
    "for i, sentence in enumerate(questions):\n",
    "    x[i] = ctable.encode(sentence, max_question_len)\n",
    "for i, sentence in enumerate(expected):\n",
    "    y[i] = ctable.encode(sentence, max_expected_len)\n",
    "print('done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Data:\n",
      "(35937, 31, 40)\n",
      "(35937, 32, 40)\n",
      "Validation Data:\n",
      "(3992, 31, 40)\n",
      "(3992, 32, 40)\n"
     ]
    }
   ],
   "source": [
    "# Shuffle (x, y) in unison as the later parts of x will almost all be larger\n",
    "# digits.\n",
    "indices = np.arange(len(y))\n",
    "np.random.shuffle(indices)\n",
    "x = x[indices]\n",
    "y = y[indices]\n",
    "\n",
    "# Explicitly set apart 10% for validation data that we never train over.\n",
    "split_at = len(x) - len(x) // 10\n",
    "(x_train, x_val) = x[:split_at], x[split_at:]\n",
    "(y_train, y_val) = y[:split_at], y[split_at:]\n",
    "\n",
    "print('Training Data:')\n",
    "print(x_train.shape)\n",
    "print(y_train.shape)\n",
    "\n",
    "print('Validation Data:')\n",
    "print(x_val.shape)\n",
    "print(y_val.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Build model...\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "lstm_1 (LSTM)                (None, 128)               86528     \n",
      "_________________________________________________________________\n",
      "repeat_vector_1 (RepeatVecto (None, 32, 128)           0         \n",
      "_________________________________________________________________\n",
      "lstm_2 (LSTM)                (None, 32, 128)           131584    \n",
      "_________________________________________________________________\n",
      "time_distributed_1 (TimeDist (None, 32, 40)            5160      \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 32, 40)            0         \n",
      "=================================================================\n",
      "Total params: 223,272\n",
      "Trainable params: 223,272\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# The below is taken from: https://github.com/keras-team/keras/blob/master/examples/addition_rnn.py\n",
    "RNN = layers.LSTM\n",
    "HIDDEN_SIZE = 128\n",
    "LAYERS = 1\n",
    "\n",
    "print('Build model...')\n",
    "model = Sequential()\n",
    "# \"Encode\" the input sequence using an RNN, producing an output of HIDDEN_SIZE.\n",
    "# Note: In a situation where your input sequences have a variable length,\n",
    "# use input_shape=(None, num_feature).\n",
    "model.add(RNN(HIDDEN_SIZE, input_shape=(max_question_len, len(chars))))\n",
    "# As the decoder RNN's input, repeatedly provide with the last hidden state of\n",
    "# RNN for each time step. Repeat 'DIGITS + 1' times as that's the maximum\n",
    "# length of output, e.g., when DIGITS=3, max output is 999+999=1998.\n",
    "#model.add(layers.Dropout(DROP_OUT))\n",
    "model.add(layers.RepeatVector(max_expected_len))\n",
    "# The decoder RNN could be multiple layers stacked or a single layer.\n",
    "for _ in range(LAYERS):\n",
    "    # By setting return_sequences to True, return not only the last output but\n",
    "    # all the outputs so far in the form of (num_samples, timesteps,\n",
    "    # output_dim). This is necessary as TimeDistributed in the below expects\n",
    "    # the first dimension to be the timesteps.\n",
    "    model.add(RNN(HIDDEN_SIZE, return_sequences=True))\n",
    "#    model.add(layers.Dropout(DROP_OUT))\n",
    "\n",
    "# Apply a dense layer to the every temporal slice of an input. For each of step\n",
    "# of the output sequence, decide which character should be chosen.\n",
    "model.add(layers.TimeDistributed(layers.Dense(len(chars))))\n",
    "model.add(layers.Activation('softmax'))\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='adam',\n",
    "              metrics=['accuracy'])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 35937 samples, validate on 3992 samples\n",
      "Epoch 1/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0158 - acc: 0.9963 - val_loss: 0.0628 - val_acc: 0.9835\n",
      "Epoch 2/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0157 - acc: 0.9964 - val_loss: 0.0626 - val_acc: 0.9839\n",
      "Epoch 3/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0155 - acc: 0.9965 - val_loss: 0.0624 - val_acc: 0.9836\n",
      "Epoch 4/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0151 - acc: 0.9966 - val_loss: 0.0622 - val_acc: 0.9838\n",
      "Epoch 5/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0150 - acc: 0.9966 - val_loss: 0.0624 - val_acc: 0.9839\n",
      "Epoch 6/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0150 - acc: 0.9966 - val_loss: 0.0625 - val_acc: 0.9838\n",
      "Epoch 7/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0146 - acc: 0.9968 - val_loss: 0.0624 - val_acc: 0.9840\n",
      "Epoch 8/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0145 - acc: 0.9968 - val_loss: 0.0624 - val_acc: 0.9838\n",
      "Epoch 9/10\n",
      "35937/35937 [==============================] - 2s 44us/step - loss: 0.0145 - acc: 0.9968 - val_loss: 0.0627 - val_acc: 0.9840\n",
      "Epoch 10/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0146 - acc: 0.9967 - val_loss: 0.0624 - val_acc: 0.9840\n",
      "\n",
      "--------------------------------------------------\n",
      "Iteration 190\n",
      "                       schiller (schillers                       ) - schillers                       \n",
      "                         chorea (choreas                         ) - choreas                         \n",
      "                        spokane (spokanes                        ) - spokanes                        \n",
      "                schistosomiasis (schistosomiases                 ) - schistosomissms                 \n",
      "                      euphonium (euphoniums                      ) - euphoniums                      \n",
      "                      submitter (submitters                      ) - submitters                      \n",
      "                     percentage (percentages                     ) - percentages                     \n",
      "                unassertiveness (unassertivenesses               ) - unassersivenesses               \n",
      "                        protest (protests                        ) - protests                        \n",
      "                      cherbourg (cherbourgs                      ) - cherborrgs                      \n",
      "Train on 35937 samples, validate on 3992 samples\n",
      "Epoch 1/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0143 - acc: 0.9968 - val_loss: 0.0622 - val_acc: 0.9840\n",
      "Epoch 2/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0140 - acc: 0.9970 - val_loss: 0.0622 - val_acc: 0.9840\n",
      "Epoch 3/10\n",
      "35937/35937 [==============================] - 2s 44us/step - loss: 0.0140 - acc: 0.9970 - val_loss: 0.0625 - val_acc: 0.9838\n",
      "Epoch 4/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0137 - acc: 0.9971 - val_loss: 0.0621 - val_acc: 0.9839\n",
      "Epoch 5/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0135 - acc: 0.9972 - val_loss: 0.0621 - val_acc: 0.9840\n",
      "Epoch 6/10\n",
      "35937/35937 [==============================] - 2s 44us/step - loss: 0.0134 - acc: 0.9972 - val_loss: 0.0624 - val_acc: 0.9838\n",
      "Epoch 7/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0133 - acc: 0.9972 - val_loss: 0.0620 - val_acc: 0.9842\n",
      "Epoch 8/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0130 - acc: 0.9973 - val_loss: 0.0618 - val_acc: 0.9842\n",
      "Epoch 9/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0130 - acc: 0.9973 - val_loss: 0.0621 - val_acc: 0.9841\n",
      "Epoch 10/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0126 - acc: 0.9975 - val_loss: 0.0622 - val_acc: 0.9842\n",
      "\n",
      "--------------------------------------------------\n",
      "Iteration 191\n",
      "                   admirability (admirabilities                  ) - admimabilities                  \n",
      "                        dasyure (dasyures                        ) - dasyures                        \n",
      "                       partisan (partisans                       ) - partisans                       \n",
      "                         copout (copouts                         ) - copouts                         \n",
      "                         nodule (nodules                         ) - nodules                         \n",
      "                         arthur (arthurs                         ) - arthurs                         \n",
      "                       lamppost (lampposts                       ) - lampposts                       \n",
      "                           surd (surds                           ) - surds                           \n",
      "                       djibouti (djiboutis                       ) - djiboutis                       \n",
      "                          spoke (spokes                          ) - spokes                          \n",
      "Train on 35937 samples, validate on 3992 samples\n",
      "Epoch 1/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0125 - acc: 0.9975 - val_loss: 0.0618 - val_acc: 0.9842\n",
      "Epoch 2/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0125 - acc: 0.9975 - val_loss: 0.0623 - val_acc: 0.9842\n",
      "Epoch 3/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0122 - acc: 0.9976 - val_loss: 0.0614 - val_acc: 0.9844\n",
      "Epoch 4/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0122 - acc: 0.9976 - val_loss: 0.0619 - val_acc: 0.9843\n",
      "Epoch 5/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0121 - acc: 0.9977 - val_loss: 0.0623 - val_acc: 0.9841\n",
      "Epoch 6/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0122 - acc: 0.9976 - val_loss: 0.0618 - val_acc: 0.9844\n",
      "Epoch 7/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0121 - acc: 0.9976 - val_loss: 0.0619 - val_acc: 0.9844\n",
      "Epoch 8/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0121 - acc: 0.9976 - val_loss: 0.0633 - val_acc: 0.9839\n",
      "Epoch 9/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0126 - acc: 0.9974 - val_loss: 0.0623 - val_acc: 0.9841\n",
      "Epoch 10/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0119 - acc: 0.9977 - val_loss: 0.0624 - val_acc: 0.9841\n",
      "\n",
      "--------------------------------------------------\n",
      "Iteration 192\n",
      "                       goodyear (goodyears                       ) - goodyears                       \n",
      "                          sebum (sebums                          ) - sebums                          \n",
      "                       ephesian (ephesians                       ) - ephesians                       \n",
      "                          galea (galeas                          ) - galeas                          \n",
      "                          honey (honeys                          ) - honeys                          \n",
      "                        jellaba (jellabas                        ) - jellabas                        \n",
      "                         egeria (egerias                         ) - egerias                         \n",
      "                     slopseller (slopsellers                     ) - slopsellers                     \n",
      "                      kiswahili (kiswahilis                      ) - kisaahilis                      \n",
      "                          peeve (peeves                          ) - peeves                          \n",
      "Train on 35937 samples, validate on 3992 samples\n",
      "Epoch 1/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0115 - acc: 0.9979 - val_loss: 0.0622 - val_acc: 0.9843\n",
      "Epoch 2/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0114 - acc: 0.9979 - val_loss: 0.0615 - val_acc: 0.9843\n",
      "Epoch 3/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0113 - acc: 0.9979 - val_loss: 0.0628 - val_acc: 0.9842\n",
      "Epoch 4/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0116 - acc: 0.9977 - val_loss: 0.0614 - val_acc: 0.9843\n",
      "Epoch 5/10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0114 - acc: 0.9979 - val_loss: 0.0621 - val_acc: 0.9842\n",
      "Epoch 6/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0113 - acc: 0.9978 - val_loss: 0.0619 - val_acc: 0.9842\n",
      "Epoch 7/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0113 - acc: 0.9978 - val_loss: 0.0622 - val_acc: 0.9844\n",
      "Epoch 8/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0111 - acc: 0.9979 - val_loss: 0.0624 - val_acc: 0.9841\n",
      "Epoch 9/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0110 - acc: 0.9980 - val_loss: 0.0619 - val_acc: 0.9846\n",
      "Epoch 10/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0111 - acc: 0.9979 - val_loss: 0.0627 - val_acc: 0.9842\n",
      "\n",
      "--------------------------------------------------\n",
      "Iteration 193\n",
      "                       culiacan (culiacans                       ) - culiacans                       \n",
      "                     translator (translators                     ) - translators                     \n",
      "                      pomolobus (pomolobuses                     ) - pomolobuses                     \n",
      "                      obbligato (obbligatoes                     ) - obblignties                     \n",
      "                   inaudibility (inaudibilities                  ) - inaaiibilities                  \n",
      "                      larvacide (larvacides                      ) - lauvacides                      \n",
      "                        tribute (tributes                        ) - tributes                        \n",
      "                mythologization (mythologizations                ) - mythologizations                \n",
      "                    funambulist (funambulists                    ) - funambulists                    \n",
      "                          gloss (glosses                         ) - gloeses                         \n",
      "Train on 35937 samples, validate on 3992 samples\n",
      "Epoch 1/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0113 - acc: 0.9978 - val_loss: 0.0631 - val_acc: 0.9840\n",
      "Epoch 2/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0113 - acc: 0.9978 - val_loss: 0.0626 - val_acc: 0.9844\n",
      "Epoch 3/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0110 - acc: 0.9979 - val_loss: 0.0626 - val_acc: 0.9842\n",
      "Epoch 4/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0106 - acc: 0.9980 - val_loss: 0.0617 - val_acc: 0.9844\n",
      "Epoch 5/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0103 - acc: 0.9982 - val_loss: 0.0615 - val_acc: 0.9846\n",
      "Epoch 6/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0101 - acc: 0.9983 - val_loss: 0.0619 - val_acc: 0.9845\n",
      "Epoch 7/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0101 - acc: 0.9983 - val_loss: 0.0613 - val_acc: 0.9846\n",
      "Epoch 8/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0099 - acc: 0.9984 - val_loss: 0.0623 - val_acc: 0.9844\n",
      "Epoch 9/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0103 - acc: 0.9982 - val_loss: 0.0635 - val_acc: 0.9839\n",
      "Epoch 10/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0105 - acc: 0.9981 - val_loss: 0.0625 - val_acc: 0.9845\n",
      "\n",
      "--------------------------------------------------\n",
      "Iteration 194\n",
      "                      kidnapper (kidnappers                      ) - kienappers                      \n",
      "                         weight (weights                         ) - weights                         \n",
      "                       cambarus (cambaruses                      ) - cambaruses                      \n",
      "                       anatidae (anatidaes                       ) - anatidaes                       \n",
      "                         felloe (felloes                         ) - felloes                         \n",
      "                     stablemate (stablemates                     ) - stablemates                     \n",
      "                         kuvasz (kuvaszzes                       ) - kuvaczses                       \n",
      "                         bhakti (bhaktis                         ) - bhakkis                         \n",
      "                        shingle (shingles                        ) - shingles                        \n",
      "                          kesey (keseys                          ) - keseys                          \n",
      "Train on 35937 samples, validate on 3992 samples\n",
      "Epoch 1/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0102 - acc: 0.9982 - val_loss: 0.0618 - val_acc: 0.9846\n",
      "Epoch 2/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0099 - acc: 0.9983 - val_loss: 0.0620 - val_acc: 0.9846\n",
      "Epoch 3/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0101 - acc: 0.9982 - val_loss: 0.0629 - val_acc: 0.9844\n",
      "Epoch 4/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0101 - acc: 0.9982 - val_loss: 0.0625 - val_acc: 0.9845\n",
      "Epoch 5/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0100 - acc: 0.9982 - val_loss: 0.0625 - val_acc: 0.9843\n",
      "Epoch 6/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0096 - acc: 0.9984 - val_loss: 0.0624 - val_acc: 0.9846\n",
      "Epoch 7/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0094 - acc: 0.9984 - val_loss: 0.0620 - val_acc: 0.9848\n",
      "Epoch 8/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0097 - acc: 0.9983 - val_loss: 0.0628 - val_acc: 0.9844\n",
      "Epoch 9/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0094 - acc: 0.9985 - val_loss: 0.0621 - val_acc: 0.9847\n",
      "Epoch 10/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0091 - acc: 0.9986 - val_loss: 0.0618 - val_acc: 0.9847\n",
      "\n",
      "--------------------------------------------------\n",
      "Iteration 195\n",
      "                        spitter (spitters                        ) - spitters                        \n",
      "                      audiotape (audiotapes                      ) - audiotapes                      \n",
      "                        daphnia (daphnias                        ) - daphnias                        \n",
      "                          locus (loci                            ) - locuses                         \n",
      "                      phasmidae (phasmidaes                      ) - phasmidaes                      \n",
      "                       drudgery (drudgeries                      ) - drudgeries                      \n",
      "                           ashe (ashes                           ) - ashes                           \n",
      "                      entrecote (entrecotes                      ) - entrecotes                      \n",
      "                         mbundu (mbundus                         ) - muundus                         \n",
      "                            key (keys                            ) - kers                            \n",
      "Train on 35937 samples, validate on 3992 samples\n",
      "Epoch 1/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0091 - acc: 0.9985 - val_loss: 0.0626 - val_acc: 0.9846\n",
      "Epoch 2/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0091 - acc: 0.9986 - val_loss: 0.0626 - val_acc: 0.9846\n",
      "Epoch 3/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0093 - acc: 0.9984 - val_loss: 0.0642 - val_acc: 0.9844\n",
      "Epoch 4/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0095 - acc: 0.9984 - val_loss: 0.0616 - val_acc: 0.9848\n",
      "Epoch 5/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0092 - acc: 0.9985 - val_loss: 0.0637 - val_acc: 0.9843\n",
      "Epoch 6/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0091 - acc: 0.9985 - val_loss: 0.0626 - val_acc: 0.9847\n",
      "Epoch 7/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0091 - acc: 0.9985 - val_loss: 0.0622 - val_acc: 0.9846\n",
      "Epoch 8/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0090 - acc: 0.9985 - val_loss: 0.0629 - val_acc: 0.9848\n",
      "Epoch 9/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0087 - acc: 0.9986 - val_loss: 0.0635 - val_acc: 0.9848\n",
      "Epoch 10/10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0090 - acc: 0.9985 - val_loss: 0.0622 - val_acc: 0.9851\n",
      "\n",
      "--------------------------------------------------\n",
      "Iteration 196\n",
      "                         oilman (oilmen                          ) - oilmen                          \n",
      "                  butterfingers (butterfinger                    ) - butterfingrr                    \n",
      "                        chapter (chapters                        ) - chapters                        \n",
      "                 nibelungenlied (nibelungenlieds                 ) - nieblengeeeeees                 \n",
      "                          talus (taluses                         ) - taluses                         \n",
      "                        rydberg (rydbergs                        ) - rydbergs                        \n",
      "                        limiter (limiters                        ) - limiters                        \n",
      "                      hyperpnea (hyperpneas                      ) - hyperpheas                      \n",
      "                           lyre (lyres                           ) - lyres                           \n",
      "                   hairsplitter (hairsplitters                   ) - hairsplitters                   \n",
      "Train on 35937 samples, validate on 3992 samples\n",
      "Epoch 1/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0086 - acc: 0.9987 - val_loss: 0.0635 - val_acc: 0.9847\n",
      "Epoch 2/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.3795 - acc: 0.9432 - val_loss: 0.5020 - val_acc: 0.8955\n",
      "Epoch 3/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.2639 - acc: 0.9292 - val_loss: 0.1491 - val_acc: 0.9562\n",
      "Epoch 4/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0901 - acc: 0.9710 - val_loss: 0.0904 - val_acc: 0.9734\n",
      "Epoch 5/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0491 - acc: 0.9839 - val_loss: 0.0718 - val_acc: 0.9796\n",
      "Epoch 6/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0317 - acc: 0.9900 - val_loss: 0.0661 - val_acc: 0.9820\n",
      "Epoch 7/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0231 - acc: 0.9932 - val_loss: 0.0638 - val_acc: 0.9832\n",
      "Epoch 8/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0188 - acc: 0.9949 - val_loss: 0.0626 - val_acc: 0.9835\n",
      "Epoch 9/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0162 - acc: 0.9959 - val_loss: 0.0626 - val_acc: 0.9837\n",
      "Epoch 10/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0147 - acc: 0.9965 - val_loss: 0.0620 - val_acc: 0.9840\n",
      "\n",
      "--------------------------------------------------\n",
      "Iteration 197\n",
      "                       coccyzus (coccyzuses                      ) - coccyzuses                      \n",
      "                   antidiabetic (antidiabetics                   ) - antidibaittes                   \n",
      "                     perisoreus (perisoreuses                    ) - periooreuses                    \n",
      "                     telegraphy (telegraphies                    ) - telegraphies                    \n",
      "                       novation (novations                       ) - novations                       \n",
      "                       nautilus (nautiluses                      ) - nautiluses                      \n",
      "                          mania (manias                          ) - manias                          \n",
      "                   pyrocephalus (pyrocephaluses                  ) - pyrocephaluses                  \n",
      "                     leishmania (leishmanias                     ) - leishmanias                     \n",
      "                          atole (atoles                          ) - atoles                          \n",
      "Train on 35937 samples, validate on 3992 samples\n",
      "Epoch 1/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0134 - acc: 0.9971 - val_loss: 0.0618 - val_acc: 0.9841\n",
      "Epoch 2/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0126 - acc: 0.9974 - val_loss: 0.0620 - val_acc: 0.9839\n",
      "Epoch 3/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0120 - acc: 0.9976 - val_loss: 0.0617 - val_acc: 0.9844\n",
      "Epoch 4/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0115 - acc: 0.9978 - val_loss: 0.0616 - val_acc: 0.9842\n",
      "Epoch 5/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0111 - acc: 0.9980 - val_loss: 0.0617 - val_acc: 0.9844\n",
      "Epoch 6/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0108 - acc: 0.9981 - val_loss: 0.0619 - val_acc: 0.9844\n",
      "Epoch 7/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0105 - acc: 0.9982 - val_loss: 0.0616 - val_acc: 0.9845\n",
      "Epoch 8/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0103 - acc: 0.9983 - val_loss: 0.0618 - val_acc: 0.9845\n",
      "Epoch 9/10\n",
      "35937/35937 [==============================] - 2s 46us/step - loss: 0.0101 - acc: 0.9983 - val_loss: 0.0616 - val_acc: 0.9845\n",
      "Epoch 10/10\n",
      "35937/35937 [==============================] - 2s 46us/step - loss: 0.0098 - acc: 0.9984 - val_loss: 0.0615 - val_acc: 0.9845\n",
      "\n",
      "--------------------------------------------------\n",
      "Iteration 198\n",
      "                       diastole (diastoles                       ) - diastoles                       \n",
      "                   salviniaceae (salviniaceaes                   ) - salviniaceaes                   \n",
      "                   pyrocephalus (pyrocephaluses                  ) - pyrocephaluses                  \n",
      "                              x (xes                             ) - xes                             \n",
      "                       vivacity (vivacities                      ) - vivacities                      \n",
      "              pentylenetetrazol (pentylenetetrazols              ) - penchllnetenntiols              \n",
      "                   paleontology (paleontologies                  ) - paleontologies                  \n",
      "                   chiropractic (chiropractics                   ) - chiropractics                   \n",
      "                      submitter (submitters                      ) - submitters                      \n",
      "                        concord (concords                        ) - concords                        \n",
      "Train on 35937 samples, validate on 3992 samples\n",
      "Epoch 1/10\n",
      "35937/35937 [==============================] - 2s 46us/step - loss: 0.0096 - acc: 0.9985 - val_loss: 0.0615 - val_acc: 0.9846\n",
      "Epoch 2/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0095 - acc: 0.9986 - val_loss: 0.0620 - val_acc: 0.9846\n",
      "Epoch 3/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0093 - acc: 0.9986 - val_loss: 0.0614 - val_acc: 0.9846\n",
      "Epoch 4/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0092 - acc: 0.9987 - val_loss: 0.0613 - val_acc: 0.9847\n",
      "Epoch 5/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0091 - acc: 0.9987 - val_loss: 0.0617 - val_acc: 0.9846\n",
      "Epoch 6/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0090 - acc: 0.9987 - val_loss: 0.0619 - val_acc: 0.9847\n",
      "Epoch 7/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0089 - acc: 0.9987 - val_loss: 0.0619 - val_acc: 0.9848\n",
      "Epoch 8/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0088 - acc: 0.9988 - val_loss: 0.0619 - val_acc: 0.9847\n",
      "Epoch 9/10\n",
      "35937/35937 [==============================] - 2s 45us/step - loss: 0.0087 - acc: 0.9988 - val_loss: 0.0616 - val_acc: 0.9847\n",
      "Epoch 10/10\n",
      "35937/35937 [==============================] - 2s 46us/step - loss: 0.0086 - acc: 0.9988 - val_loss: 0.0620 - val_acc: 0.9847\n",
      "\n",
      "--------------------------------------------------\n",
      "Iteration 199\n",
      "                          silex (silices                         ) - silexes                         \n",
      "                       nautilus (nautiluses                      ) - nautiluses                      \n",
      "                        grizzly (grizzlies                       ) - grizzlies                       \n",
      "                      blandness (blandnesses                     ) - blandnesses                     \n",
      "                     percentage (percentages                     ) - percentages                     \n",
      "                       airspace (airspaces                       ) - airspaces                       \n",
      "                        comoros (comoro                          ) - comoro                          \n",
      "                         phlegm (phlegms                         ) - phlegms                         \n",
      "                      syneresis (synereses                       ) - synereses                       \n",
      "                        mercury (mercuries                       ) - mercuries                       \n"
     ]
    }
   ],
   "source": [
    "BATCH_SIZE = 2048\n",
    "\n",
    "# Train the model each generation and show predictions against the validation\n",
    "# dataset.\n",
    "for iteration in range(1, 200):\n",
    "    model.fit(x_train, y_train,\n",
    "              batch_size=BATCH_SIZE,\n",
    "              epochs=10,\n",
    "              validation_data=(x_val, y_val))\n",
    "    print()\n",
    "    print('-' * 50)\n",
    "    print('Iteration', iteration)\n",
    "    # Select 10 samples from the validation set at random so we can visualize\n",
    "    # errors.\n",
    "    for i in range(10):\n",
    "        ind = np.random.randint(0, len(x_val))\n",
    "        rowx, rowy = x_val[np.array([ind])], y_val[np.array([ind])]\n",
    "        preds = model.predict_classes(rowx, verbose=0)\n",
    "        q = ctable.decode(rowx[0])\n",
    "        correct = ctable.decode(rowy[0])\n",
    "        guess = ctable.decode(preds[0], calc_argmax=False)\n",
    "        print(q[::-1] if INVERT else q, '(%s)' % correct, '-', guess)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                        hodeida (hodeidas                        ) - hodeidas                        \n",
      "                         redbud (redbuds                         ) - redbuds                         \n",
      "                        fitment (fitments                        ) - fitments                        \n",
      "                       morrison (morrisons                       ) - morrisons                       \n",
      "                           pung (pungs                           ) - pungs                           \n",
      "                     micromeria (micromerias                     ) - micromerias                     \n",
      "                        albumin (albumins                        ) - albumins                        \n",
      "                        fetlock (fetlocks                        ) - fetlocks                        \n",
      "                        sorbent (sorbents                        ) - sorbents                        \n",
      "                         prague (pragues                         ) - pragues                         \n"
     ]
    }
   ],
   "source": [
    "for i in range(10):\n",
    "    ind = np.random.randint(0, len(x_val))\n",
    "    rowx, rowy = x_val[np.array([ind])], y_val[np.array([ind])]\n",
    "    preds = model.predict_classes(rowx, verbose=0)\n",
    "    q = ctable.decode(rowx[0])\n",
    "    correct = ctable.decode(rowy[0])\n",
    "    guess = ctable.decode(preds[0], calc_argmax=False)\n",
    "    print(q[::-1] if INVERT else q, '(%s)' % correct, '-', guess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Cookbook Kernel",
   "language": "python",
   "name": "cookbook3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}