keras_examples_babi_rnn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def plot_keras_model(model, show_shapes=True, show_layer_names=True):\n",
    "    from IPython.display import SVG\n",
    "    from keras.utils.vis_utils import model_to_dot\n",
    "    return SVG(model_to_dot(model, show_shapes=show_shapes, show_layer_names=show_layer_names).create(prog='dot', format='svg'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "'''Trains two recurrent neural networks based upon a story and a question.\n",
    "\n",
    "The resulting merged vector is then queried to answer a range of bAbI tasks.\n",
    "\n",
    "The results are comparable to those for an LSTM model provided in Weston et al.:\n",
    "\"Towards AI-Complete Question Answering: A Set of Prerequisite Toy Tasks\"\n",
    "http://arxiv.org/abs/1502.05698\n",
    "\n",
    "Task Number                  | FB LSTM Baseline | Keras QA\n",
    "---                          | ---              | ---\n",
    "QA1 - Single Supporting Fact | 50               | 100.0\n",
    "QA2 - Two Supporting Facts   | 20               | 50.0\n",
    "QA3 - Three Supporting Facts | 20               | 20.5\n",
    "QA4 - Two Arg. Relations     | 61               | 62.9\n",
    "QA5 - Three Arg. Relations   | 70               | 61.9\n",
    "QA6 - yes/No Questions       | 48               | 50.7\n",
    "QA7 - Counting               | 49               | 78.9\n",
    "QA8 - Lists/Sets             | 45               | 77.2\n",
    "QA9 - Simple Negation        | 64               | 64.0\n",
    "QA10 - Indefinite Knowledge  | 44               | 47.7\n",
    "QA11 - Basic Coreference     | 72               | 74.9\n",
    "QA12 - Conjunction           | 74               | 76.4\n",
    "QA13 - Compound Coreference  | 94               | 94.4\n",
    "QA14 - Time Reasoning        | 27               | 34.8\n",
    "QA15 - Basic Deduction       | 21               | 32.4\n",
    "QA16 - Basic Induction       | 23               | 50.6\n",
    "QA17 - Positional Reasoning  | 51               | 49.1\n",
    "QA18 - Size Reasoning        | 52               | 90.8\n",
    "QA19 - Path Finding          | 8                | 9.0\n",
    "QA20 - Agent's Motivations   | 91               | 90.7\n",
    "\n",
    "For the resources related to the bAbI project, refer to:\n",
    "https://research.facebook.com/researchers/1543934539189348\n",
    "\n",
    "# Notes\n",
    "\n",
    "- With default word, sentence, and query vector sizes, the GRU model achieves:\n",
    "  - 100% test accuracy on QA1 in 20 epochs (2 seconds per epoch on CPU)\n",
    "  - 50% test accuracy on QA2 in 20 epochs (16 seconds per epoch on CPU)\n",
    "In comparison, the Facebook paper achieves 50% and 20% for the LSTM baseline.\n",
    "\n",
    "- The task does not traditionally parse the question separately. This likely\n",
    "improves accuracy and is a good example of merging two RNNs.\n",
    "\n",
    "- The word vector embeddings are not shared between the story and question RNNs.\n",
    "\n",
    "- See how the accuracy changes given 10,000 training samples (en-10k) instead\n",
    "of only 1000. 1000 was used in order to be comparable to the original paper.\n",
    "\n",
    "- Experiment with GRU, LSTM, and JZS1-3 as they give subtly different results.\n",
    "\n",
    "- The length and noise (i.e. 'useless' story components) impact the ability for\n",
    "LSTMs / GRUs to provide the correct answer. Given only the supporting facts,\n",
    "these RNNs can achieve 100% accuracy on many tasks. Memory networks and neural\n",
    "networks that use attentional processes can efficiently search through this\n",
    "noise to find the relevant statements, improving performance substantially.\n",
    "This becomes especially obvious on QA2 and QA3, both far longer than QA1.\n",
    "'''\n",
    "\n",
    "from __future__ import print_function\n",
    "from functools import reduce\n",
    "import re\n",
    "import tarfile\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from keras.utils.data_utils import get_file\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras import layers\n",
    "from keras.layers import recurrent\n",
    "from keras.models import Model\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "\n",
    "def tokenize(sent):\n",
    "    '''Return the tokens of a sentence including punctuation.\n",
    "\n",
    "    >>> tokenize('Bob dropped the apple. Where is the apple?')\n",
    "    ['Bob', 'dropped', 'the', 'apple', '.', 'Where', 'is', 'the', 'apple', '?']\n",
    "    '''\n",
    "    return [x.strip() for x in re.split('(\\W+)?', sent) if x.strip()]\n",
    "\n",
    "\n",
    "def parse_stories(lines, only_supporting=False):\n",
    "    '''Parse stories provided in the bAbi tasks format\n",
    "\n",
    "    If only_supporting is true,\n",
    "    only the sentences that support the answer are kept.\n",
    "    '''\n",
    "    data = []\n",
    "    story = []\n",
    "    for line in lines:\n",
    "        line = line.decode('utf-8').strip()\n",
    "        nid, line = line.split(' ', 1)\n",
    "        nid = int(nid)\n",
    "        if nid == 1:\n",
    "            story = []\n",
    "        if '\\t' in line:\n",
    "            q, a, supporting = line.split('\\t')\n",
    "            q = tokenize(q)\n",
    "            substory = None\n",
    "            if only_supporting:\n",
    "                # Only select the related substory\n",
    "                supporting = map(int, supporting.split())\n",
    "                substory = [story[i - 1] for i in supporting]\n",
    "            else:\n",
    "                # Provide all the substories\n",
    "                substory = [x for x in story if x]\n",
    "            data.append((substory, q, a))\n",
    "            story.append('')\n",
    "        else:\n",
    "            sent = tokenize(line)\n",
    "            story.append(sent)\n",
    "    return data\n",
    "\n",
    "\n",
    "def get_stories(f, only_supporting=False, max_length=None):\n",
    "    '''Given a file name, read the file, retrieve the stories,\n",
    "    and then convert the sentences into a single story.\n",
    "\n",
    "    If max_length is supplied,\n",
    "    any stories longer than max_length tokens will be discarded.\n",
    "    '''\n",
    "    data = parse_stories(f.readlines(), only_supporting=only_supporting)\n",
    "    flatten = lambda data: reduce(lambda x, y: x + y, data)\n",
    "    data = [(flatten(story), q, answer) for story, q, answer in data if not max_length or len(flatten(story)) < max_length]\n",
    "    return data\n",
    "\n",
    "\n",
    "def vectorize_stories(data, word_idx, story_maxlen, query_maxlen):\n",
    "    xs = []\n",
    "    xqs = []\n",
    "    ys = []\n",
    "    for story, query, answer in data:\n",
    "        x = [word_idx[w] for w in story]\n",
    "        xq = [word_idx[w] for w in query]\n",
    "        # let's not forget that index 0 is reserved\n",
    "        y = np.zeros(len(word_idx) + 1)\n",
    "        y[word_idx[answer]] = 1\n",
    "        xs.append(x)\n",
    "        xqs.append(xq)\n",
    "        ys.append(y)\n",
    "    return pad_sequences(xs, maxlen=story_maxlen), pad_sequences(xqs, maxlen=query_maxlen), np.array(ys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RNN / Embed / Sent / Query = <class 'keras.layers.recurrent.LSTM'>, 50, 100, 100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/re.py:212: FutureWarning: split() requires a non-empty pattern match.\n",
      "  return _compile(pattern, flags).split(string, maxsplit)\n"
     ]
    }
   ],
   "source": [
    "RNN = recurrent.LSTM\n",
    "EMBED_HIDDEN_SIZE = 50\n",
    "SENT_HIDDEN_SIZE = 100\n",
    "QUERY_HIDDEN_SIZE = 100\n",
    "BATCH_SIZE = 32\n",
    "EPOCHS = 40\n",
    "print('RNN / Embed / Sent / Query = {}, {}, {}, {}'.format(RNN,\n",
    "                                                           EMBED_HIDDEN_SIZE,\n",
    "                                                           SENT_HIDDEN_SIZE,\n",
    "                                                           QUERY_HIDDEN_SIZE))\n",
    "\n",
    "try:\n",
    "    path = get_file('babi-tasks-v1-2.tar.gz', origin='https://s3.amazonaws.com/text-datasets/babi_tasks_1-20_v1-2.tar.gz')\n",
    "except:\n",
    "    print('Error downloading dataset, please download it manually:\\n'\n",
    "          '$ wget http://www.thespermwhale.com/jaseweston/babi/tasks_1-20_v1-2.tar.gz\\n'\n",
    "          '$ mv tasks_1-20_v1-2.tar.gz ~/.keras/datasets/babi-tasks-v1-2.tar.gz')\n",
    "    raise\n",
    "tar = tarfile.open(path)\n",
    "# Default QA1 with 1000 samples\n",
    "# challenge = 'tasks_1-20_v1-2/en/qa1_single-supporting-fact_{}.txt'\n",
    "# QA1 with 10,000 samples\n",
    "# challenge = 'tasks_1-20_v1-2/en-10k/qa1_single-supporting-fact_{}.txt'\n",
    "# QA2 with 1000 samples\n",
    "challenge = 'tasks_1-20_v1-2/en/qa2_two-supporting-facts_{}.txt'\n",
    "# QA2 with 10,000 samples\n",
    "# challenge = 'tasks_1-20_v1-2/en-10k/qa2_two-supporting-facts_{}.txt'\n",
    "train = get_stories(tar.extractfile(challenge.format('train')))\n",
    "test = get_stories(tar.extractfile(challenge.format('test')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab = ['.', '?', 'Daniel', 'John', 'Mary', 'Sandra', 'Where', 'apple', 'back', 'bathroom', 'bedroom', 'discarded', 'down', 'dropped', 'football', 'garden', 'got', 'grabbed', 'hallway', 'is', 'journeyed', 'kitchen', 'left', 'milk', 'moved', 'office', 'picked', 'put', 'the', 'there', 'to', 'took', 'travelled', 'up', 'went']\n",
      "x.shape = (1000, 552)\n",
      "xq.shape = (1000, 5)\n",
      "y.shape = (1000, 36)\n",
      "story_maxlen, query_maxlen = 552, 5\n"
     ]
    }
   ],
   "source": [
    "vocab = set()\n",
    "for story, q, answer in train + test:\n",
    "    vocab |= set(story + q + [answer])\n",
    "vocab = sorted(vocab)\n",
    "\n",
    "# Reserve 0 for masking via pad_sequences\n",
    "vocab_size = len(vocab) + 1\n",
    "word_idx = dict((c, i + 1) for i, c in enumerate(vocab))\n",
    "story_maxlen = max(map(len, (x for x, _, _ in train + test)))\n",
    "query_maxlen = max(map(len, (x for _, x, _ in train + test)))\n",
    "\n",
    "x, xq, y = vectorize_stories(train, word_idx, story_maxlen, query_maxlen)\n",
    "tx, txq, ty = vectorize_stories(test, word_idx, story_maxlen, query_maxlen)\n",
    "\n",
    "print('vocab = {}'.format(vocab))\n",
    "print('x.shape = {}'.format(x.shape))\n",
    "print('xq.shape = {}'.format(xq.shape))\n",
    "print('y.shape = {}'.format(y.shape))\n",
    "print('story_maxlen, query_maxlen = {}, {}'.format(story_maxlen, query_maxlen))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Build model...\n"
     ]
    }
   ],
   "source": [
    "print('Build model...')\n",
    "\n",
    "sentence = layers.Input(shape=(story_maxlen,), dtype='int32', name='sentence')\n",
    "encoded_sentence = layers.Embedding(vocab_size, EMBED_HIDDEN_SIZE)(sentence)\n",
    "encoded_sentence = layers.Dropout(0.3)(encoded_sentence)\n",
    "\n",
    "question = layers.Input(shape=(query_maxlen,), dtype='int32', name='question')\n",
    "encoded_question = layers.Embedding(vocab_size, EMBED_HIDDEN_SIZE)(question)\n",
    "encoded_question = layers.Dropout(0.3)(encoded_question)\n",
    "encoded_question = RNN(EMBED_HIDDEN_SIZE)(encoded_question)\n",
    "encoded_question = layers.RepeatVector(story_maxlen)(encoded_question)\n",
    "\n",
    "merged = layers.add([encoded_sentence, encoded_question])\n",
    "merged = RNN(EMBED_HIDDEN_SIZE)(merged)\n",
    "merged = layers.Dropout(0.3)(merged)\n",
    "preds = layers.Dense(vocab_size, activation='softmax')(merged)\n",
    "\n",
    "model = Model([sentence, question], preds)\n",
    "model.compile(optimizer='adam',\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<svg height=\"719pt\" viewBox=\"0.00 0.00 837.00 719.00\" width=\"837pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g class=\"graph\" id=\"graph0\" transform=\"scale(1 1) rotate(0) translate(4 715)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"white\" points=\"-4,4 -4,-715 833,-715 833,4 -4,4\" stroke=\"none\"/>\n",
       "<!-- 140201345524568 -->\n",
       "<g class=\"node\" id=\"node1\"><title>140201345524568</title>\n",
       "<polygon fill=\"none\" points=\"58.5,-664.5 58.5,-710.5 376.5,-710.5 376.5,-664.5 58.5,-664.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"142\" y=\"-683.8\">question: InputLayer</text>\n",
       "<polyline fill=\"none\" points=\"225.5,-664.5 225.5,-710.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"259.5\" y=\"-695.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"225.5,-687.5 293.5,-687.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"259.5\" y=\"-672.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"293.5,-664.5 293.5,-710.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"335\" y=\"-695.3\">(None, 5)</text>\n",
       "<polyline fill=\"none\" points=\"293.5,-687.5 376.5,-687.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"335\" y=\"-672.3\">(None, 5)</text>\n",
       "</g>\n",
       "<!-- 140201344863368 -->\n",
       "<g class=\"node\" id=\"node2\"><title>140201344863368</title>\n",
       "<polygon fill=\"none\" points=\"27,-581.5 27,-627.5 408,-627.5 408,-581.5 27,-581.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"128.5\" y=\"-600.8\">embedding_4: Embedding</text>\n",
       "<polyline fill=\"none\" points=\"230,-581.5 230,-627.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"264\" y=\"-612.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"230,-604.5 298,-604.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"264\" y=\"-589.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"298,-581.5 298,-627.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"353\" y=\"-612.3\">(None, 5)</text>\n",
       "<polyline fill=\"none\" points=\"298,-604.5 408,-604.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"353\" y=\"-589.3\">(None, 5, 50)</text>\n",
       "</g>\n",
       "<!-- 140201345524568&#45;&gt;140201344863368 -->\n",
       "<g class=\"edge\" id=\"edge1\"><title>140201345524568-&gt;140201344863368</title>\n",
       "<path d=\"M217.5,-664.366C217.5,-656.152 217.5,-646.658 217.5,-637.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"221,-637.607 217.5,-627.607 214,-637.607 221,-637.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 140201345126352 -->\n",
       "<g class=\"node\" id=\"node4\"><title>140201345126352</title>\n",
       "<polygon fill=\"none\" points=\"50,-498.5 50,-544.5 385,-544.5 385,-498.5 50,-498.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"128.5\" y=\"-517.8\">dropout_5: Dropout</text>\n",
       "<polyline fill=\"none\" points=\"207,-498.5 207,-544.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"241\" y=\"-529.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"207,-521.5 275,-521.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"241\" y=\"-506.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"275,-498.5 275,-544.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"330\" y=\"-529.3\">(None, 5, 50)</text>\n",
       "<polyline fill=\"none\" points=\"275,-521.5 385,-521.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"330\" y=\"-506.3\">(None, 5, 50)</text>\n",
       "</g>\n",
       "<!-- 140201344863368&#45;&gt;140201345126352 -->\n",
       "<g class=\"edge\" id=\"edge2\"><title>140201344863368-&gt;140201345126352</title>\n",
       "<path d=\"M217.5,-581.366C217.5,-573.152 217.5,-563.658 217.5,-554.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"221,-554.607 217.5,-544.607 214,-554.607 221,-554.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 140201345245592 -->\n",
       "<g class=\"node\" id=\"node3\"><title>140201345245592</title>\n",
       "<polygon fill=\"none\" points=\"460,-498.5 460,-544.5 799,-544.5 799,-498.5 460,-498.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"545\" y=\"-517.8\">sentence: InputLayer</text>\n",
       "<polyline fill=\"none\" points=\"630,-498.5 630,-544.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"664\" y=\"-529.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"630,-521.5 698,-521.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"664\" y=\"-506.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"698,-498.5 698,-544.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"748.5\" y=\"-529.3\">(None, 552)</text>\n",
       "<polyline fill=\"none\" points=\"698,-521.5 799,-521.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"748.5\" y=\"-506.3\">(None, 552)</text>\n",
       "</g>\n",
       "<!-- 140201345245928 -->\n",
       "<g class=\"node\" id=\"node5\"><title>140201345245928</title>\n",
       "<polygon fill=\"none\" points=\"430,-415.5 430,-461.5 829,-461.5 829,-415.5 430,-415.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"531.5\" y=\"-434.8\">embedding_3: Embedding</text>\n",
       "<polyline fill=\"none\" points=\"633,-415.5 633,-461.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"667\" y=\"-446.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"633,-438.5 701,-438.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"667\" y=\"-423.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"701,-415.5 701,-461.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"765\" y=\"-446.3\">(None, 552)</text>\n",
       "<polyline fill=\"none\" points=\"701,-438.5 829,-438.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"765\" y=\"-423.3\">(None, 552, 50)</text>\n",
       "</g>\n",
       "<!-- 140201345245592&#45;&gt;140201345245928 -->\n",
       "<g class=\"edge\" id=\"edge3\"><title>140201345245592-&gt;140201345245928</title>\n",
       "<path d=\"M629.5,-498.366C629.5,-490.152 629.5,-480.658 629.5,-471.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"633,-471.607 629.5,-461.607 626,-471.607 633,-471.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 140201345248224 -->\n",
       "<g class=\"node\" id=\"node6\"><title>140201345248224</title>\n",
       "<polygon fill=\"none\" points=\"70.5,-415.5 70.5,-461.5 364.5,-461.5 364.5,-415.5 70.5,-415.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"128.5\" y=\"-434.8\">lstm_3: LSTM</text>\n",
       "<polyline fill=\"none\" points=\"186.5,-415.5 186.5,-461.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"220.5\" y=\"-446.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"186.5,-438.5 254.5,-438.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"220.5\" y=\"-423.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"254.5,-415.5 254.5,-461.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"309.5\" y=\"-446.3\">(None, 5, 50)</text>\n",
       "<polyline fill=\"none\" points=\"254.5,-438.5 364.5,-438.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"309.5\" y=\"-423.3\">(None, 50)</text>\n",
       "</g>\n",
       "<!-- 140201345126352&#45;&gt;140201345248224 -->\n",
       "<g class=\"edge\" id=\"edge4\"><title>140201345126352-&gt;140201345248224</title>\n",
       "<path d=\"M217.5,-498.366C217.5,-490.152 217.5,-480.658 217.5,-471.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"221,-471.607 217.5,-461.607 214,-471.607 221,-471.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 140201345539544 -->\n",
       "<g class=\"node\" id=\"node7\"><title>140201345539544</title>\n",
       "<polygon fill=\"none\" points=\"453,-332.5 453,-378.5 806,-378.5 806,-332.5 453,-332.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"531.5\" y=\"-351.8\">dropout_4: Dropout</text>\n",
       "<polyline fill=\"none\" points=\"610,-332.5 610,-378.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"644\" y=\"-363.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"610,-355.5 678,-355.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"644\" y=\"-340.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"678,-332.5 678,-378.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"742\" y=\"-363.3\">(None, 552, 50)</text>\n",
       "<polyline fill=\"none\" points=\"678,-355.5 806,-355.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"742\" y=\"-340.3\">(None, 552, 50)</text>\n",
       "</g>\n",
       "<!-- 140201345245928&#45;&gt;140201345539544 -->\n",
       "<g class=\"edge\" id=\"edge5\"><title>140201345245928-&gt;140201345539544</title>\n",
       "<path d=\"M629.5,-415.366C629.5,-407.152 629.5,-397.658 629.5,-388.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"633,-388.607 629.5,-378.607 626,-388.607 633,-388.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 140201345247160 -->\n",
       "<g class=\"node\" id=\"node8\"><title>140201345247160</title>\n",
       "<polygon fill=\"none\" points=\"0,-332.5 0,-378.5 435,-378.5 435,-332.5 0,-332.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"119.5\" y=\"-351.8\">repeat_vector_2: RepeatVector</text>\n",
       "<polyline fill=\"none\" points=\"239,-332.5 239,-378.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"273\" y=\"-363.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"239,-355.5 307,-355.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"273\" y=\"-340.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"307,-332.5 307,-378.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"371\" y=\"-363.3\">(None, 50)</text>\n",
       "<polyline fill=\"none\" points=\"307,-355.5 435,-355.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"371\" y=\"-340.3\">(None, 552, 50)</text>\n",
       "</g>\n",
       "<!-- 140201345248224&#45;&gt;140201345247160 -->\n",
       "<g class=\"edge\" id=\"edge6\"><title>140201345248224-&gt;140201345247160</title>\n",
       "<path d=\"M217.5,-415.366C217.5,-407.152 217.5,-397.658 217.5,-388.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"221,-388.607 217.5,-378.607 214,-388.607 221,-388.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 140201345168272 -->\n",
       "<g class=\"node\" id=\"node9\"><title>140201345168272</title>\n",
       "<polygon fill=\"none\" points=\"211,-249.5 211,-295.5 636,-295.5 636,-249.5 211,-249.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"259.5\" y=\"-268.8\">add_2: Add</text>\n",
       "<polyline fill=\"none\" points=\"308,-249.5 308,-295.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"342\" y=\"-280.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"308,-272.5 376,-272.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"342\" y=\"-257.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"376,-249.5 376,-295.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"506\" y=\"-280.3\">[(None, 552, 50), (None, 552, 50)]</text>\n",
       "<polyline fill=\"none\" points=\"376,-272.5 636,-272.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"506\" y=\"-257.3\">(None, 552, 50)</text>\n",
       "</g>\n",
       "<!-- 140201345539544&#45;&gt;140201345168272 -->\n",
       "<g class=\"edge\" id=\"edge7\"><title>140201345539544-&gt;140201345168272</title>\n",
       "<path d=\"M573.482,-332.473C547.371,-322.206 516.133,-309.924 488.941,-299.232\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"490.094,-295.924 479.507,-295.522 487.533,-302.439 490.094,-295.924\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 140201345247160&#45;&gt;140201345168272 -->\n",
       "<g class=\"edge\" id=\"edge8\"><title>140201345247160-&gt;140201345168272</title>\n",
       "<path d=\"M273.518,-332.473C299.629,-322.206 330.867,-309.924 358.059,-299.232\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"359.467,-302.439 367.493,-295.522 356.906,-295.924 359.467,-302.439\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 140201345170120 -->\n",
       "<g class=\"node\" id=\"node10\"><title>140201345170120</title>\n",
       "<polygon fill=\"none\" points=\"267.5,-166.5 267.5,-212.5 579.5,-212.5 579.5,-166.5 267.5,-166.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"325.5\" y=\"-185.8\">lstm_4: LSTM</text>\n",
       "<polyline fill=\"none\" points=\"383.5,-166.5 383.5,-212.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"417.5\" y=\"-197.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"383.5,-189.5 451.5,-189.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"417.5\" y=\"-174.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"451.5,-166.5 451.5,-212.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"515.5\" y=\"-197.3\">(None, 552, 50)</text>\n",
       "<polyline fill=\"none\" points=\"451.5,-189.5 579.5,-189.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"515.5\" y=\"-174.3\">(None, 50)</text>\n",
       "</g>\n",
       "<!-- 140201345168272&#45;&gt;140201345170120 -->\n",
       "<g class=\"edge\" id=\"edge9\"><title>140201345168272-&gt;140201345170120</title>\n",
       "<path d=\"M423.5,-249.366C423.5,-241.152 423.5,-231.658 423.5,-222.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"427,-222.607 423.5,-212.607 420,-222.607 427,-222.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 140201341573944 -->\n",
       "<g class=\"node\" id=\"node11\"><title>140201341573944</title>\n",
       "<polygon fill=\"none\" points=\"265,-83.5 265,-129.5 582,-129.5 582,-83.5 265,-83.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"343.5\" y=\"-102.8\">dropout_6: Dropout</text>\n",
       "<polyline fill=\"none\" points=\"422,-83.5 422,-129.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"456\" y=\"-114.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"422,-106.5 490,-106.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"456\" y=\"-91.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"490,-83.5 490,-129.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"536\" y=\"-114.3\">(None, 50)</text>\n",
       "<polyline fill=\"none\" points=\"490,-106.5 582,-106.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"536\" y=\"-91.3\">(None, 50)</text>\n",
       "</g>\n",
       "<!-- 140201345170120&#45;&gt;140201341573944 -->\n",
       "<g class=\"edge\" id=\"edge10\"><title>140201345170120-&gt;140201341573944</title>\n",
       "<path d=\"M423.5,-166.366C423.5,-158.152 423.5,-148.658 423.5,-139.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"427,-139.607 423.5,-129.607 420,-139.607 427,-139.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 140201340980416 -->\n",
       "<g class=\"node\" id=\"node12\"><title>140201340980416</title>\n",
       "<polygon fill=\"none\" points=\"279.5,-0.5 279.5,-46.5 567.5,-46.5 567.5,-0.5 279.5,-0.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"343.5\" y=\"-19.8\">dense_2: Dense</text>\n",
       "<polyline fill=\"none\" points=\"407.5,-0.5 407.5,-46.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"441.5\" y=\"-31.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"407.5,-23.5 475.5,-23.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"441.5\" y=\"-8.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"475.5,-0.5 475.5,-46.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"521.5\" y=\"-31.3\">(None, 50)</text>\n",
       "<polyline fill=\"none\" points=\"475.5,-23.5 567.5,-23.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"521.5\" y=\"-8.3\">(None, 36)</text>\n",
       "</g>\n",
       "<!-- 140201341573944&#45;&gt;140201340980416 -->\n",
       "<g class=\"edge\" id=\"edge11\"><title>140201341573944-&gt;140201340980416</title>\n",
       "<path d=\"M423.5,-83.3664C423.5,-75.1516 423.5,-65.6579 423.5,-56.7252\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"427,-56.6068 423.5,-46.6068 420,-56.6069 427,-56.6068\" stroke=\"black\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>"
      ],
      "text/plain": [
       "<IPython.core.display.SVG object>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "plot_keras_model(model, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training\n",
      "Train on 950 samples, validate on 50 samples\n",
      "Epoch 1/40\n",
      "950/950 [==============================] - 18s 19ms/step - loss: 3.1594 - acc: 0.1989 - val_loss: 2.1766 - val_acc: 0.3000\n",
      "Epoch 2/40\n",
      "950/950 [==============================] - 15s 16ms/step - loss: 2.0218 - acc: 0.1958 - val_loss: 1.8874 - val_acc: 0.0600\n",
      "Epoch 3/40\n",
      "950/950 [==============================] - 15s 16ms/step - loss: 1.9160 - acc: 0.1916 - val_loss: 1.8227 - val_acc: 0.0600\n",
      "Epoch 4/40\n",
      "950/950 [==============================] - 15s 16ms/step - loss: 1.8975 - acc: 0.1895 - val_loss: 1.8226 - val_acc: 0.0600\n",
      "Epoch 5/40\n",
      "950/950 [==============================] - 15s 16ms/step - loss: 1.9133 - acc: 0.1789 - val_loss: 1.8391 - val_acc: 0.0600\n",
      "Epoch 6/40\n",
      "950/950 [==============================] - 15s 16ms/step - loss: 1.8880 - acc: 0.1832 - val_loss: 1.8406 - val_acc: 0.0600\n",
      "Epoch 7/40\n",
      "950/950 [==============================] - 15s 16ms/step - loss: 1.8681 - acc: 0.1968 - val_loss: 1.8478 - val_acc: 0.0600\n",
      "Epoch 8/40\n",
      "950/950 [==============================] - 15s 16ms/step - loss: 1.8635 - acc: 0.1737 - val_loss: 1.8019 - val_acc: 0.1200\n",
      "Epoch 9/40\n",
      "950/950 [==============================] - 15s 16ms/step - loss: 1.8579 - acc: 0.1758 - val_loss: 1.8251 - val_acc: 0.0600\n",
      "Epoch 10/40\n",
      "950/950 [==============================] - 15s 16ms/step - loss: 1.8309 - acc: 0.1968 - val_loss: 1.8203 - val_acc: 0.0600\n",
      "Epoch 11/40\n",
      "950/950 [==============================] - 15s 16ms/step - loss: 1.8350 - acc: 0.1832 - val_loss: 1.8453 - val_acc: 0.0600\n",
      "Epoch 12/40\n",
      "950/950 [==============================] - 15s 16ms/step - loss: 1.8358 - acc: 0.1968 - val_loss: 1.7832 - val_acc: 0.3000\n",
      "Epoch 13/40\n",
      "950/950 [==============================] - 15s 16ms/step - loss: 1.8384 - acc: 0.2084 - val_loss: 1.8349 - val_acc: 0.0600\n",
      "Epoch 14/40\n",
      "950/950 [==============================] - 15s 16ms/step - loss: 1.8297 - acc: 0.2000 - val_loss: 1.8327 - val_acc: 0.0600\n",
      "Epoch 15/40\n",
      "950/950 [==============================] - 15s 16ms/step - loss: 1.8200 - acc: 0.2126 - val_loss: 1.7879 - val_acc: 0.3200\n",
      "Epoch 16/40\n",
      "950/950 [==============================] - 15s 16ms/step - loss: 1.8189 - acc: 0.2168 - val_loss: 1.7780 - val_acc: 0.3000\n",
      "Epoch 17/40\n",
      "950/950 [==============================] - 15s 16ms/step - loss: 1.8230 - acc: 0.1979 - val_loss: 1.8588 - val_acc: 0.0600\n",
      "Epoch 18/40\n",
      "950/950 [==============================] - 15s 16ms/step - loss: 1.8105 - acc: 0.2042 - val_loss: 1.8010 - val_acc: 0.1000\n",
      "Epoch 19/40\n",
      "950/950 [==============================] - 15s 16ms/step - loss: 1.8028 - acc: 0.2158 - val_loss: 1.7936 - val_acc: 0.1400\n",
      "Epoch 20/40\n",
      "950/950 [==============================] - 15s 16ms/step - loss: 1.7927 - acc: 0.2116 - val_loss: 1.7832 - val_acc: 0.2000\n",
      "Epoch 21/40\n",
      "950/950 [==============================] - 15s 16ms/step - loss: 1.7936 - acc: 0.2242 - val_loss: 1.7743 - val_acc: 0.2200\n",
      "Epoch 22/40\n",
      "950/950 [==============================] - 15s 16ms/step - loss: 1.7715 - acc: 0.2568 - val_loss: 1.7653 - val_acc: 0.2200\n",
      "Epoch 23/40\n",
      "950/950 [==============================] - 15s 16ms/step - loss: 1.7565 - acc: 0.2421 - val_loss: 1.7181 - val_acc: 0.3400\n",
      "Epoch 24/40\n",
      "950/950 [==============================] - 15s 16ms/step - loss: 1.7369 - acc: 0.2789 - val_loss: 1.7522 - val_acc: 0.2800\n",
      "Epoch 25/40\n",
      "950/950 [==============================] - 15s 16ms/step - loss: 1.7281 - acc: 0.2947 - val_loss: 1.7882 - val_acc: 0.2400\n",
      "Epoch 26/40\n",
      "950/950 [==============================] - 15s 16ms/step - loss: 1.7154 - acc: 0.2979 - val_loss: 1.8233 - val_acc: 0.2200\n",
      "Epoch 27/40\n",
      "950/950 [==============================] - 15s 16ms/step - loss: 1.7326 - acc: 0.2821 - val_loss: 1.7028 - val_acc: 0.3200\n",
      "Epoch 28/40\n",
      "950/950 [==============================] - 15s 16ms/step - loss: 1.7061 - acc: 0.3116 - val_loss: 1.6667 - val_acc: 0.4200\n",
      "Epoch 29/40\n",
      "950/950 [==============================] - 15s 16ms/step - loss: 1.6886 - acc: 0.3221 - val_loss: 1.7159 - val_acc: 0.3600\n",
      "Epoch 30/40\n",
      "950/950 [==============================] - 15s 15ms/step - loss: 1.6918 - acc: 0.3232 - val_loss: 1.6698 - val_acc: 0.3600\n",
      "Epoch 31/40\n",
      "950/950 [==============================] - 15s 16ms/step - loss: 1.6708 - acc: 0.3305 - val_loss: 1.7028 - val_acc: 0.3400\n",
      "Epoch 32/40\n",
      "950/950 [==============================] - 15s 15ms/step - loss: 1.6482 - acc: 0.3453 - val_loss: 1.6295 - val_acc: 0.3800\n",
      "Epoch 33/40\n",
      "950/950 [==============================] - 15s 15ms/step - loss: 1.7156 - acc: 0.3442 - val_loss: 1.7175 - val_acc: 0.3000\n",
      "Epoch 34/40\n",
      "950/950 [==============================] - 15s 15ms/step - loss: 1.6796 - acc: 0.3232 - val_loss: 1.6607 - val_acc: 0.3400\n",
      "Epoch 35/40\n",
      "950/950 [==============================] - 15s 15ms/step - loss: 1.6624 - acc: 0.3400 - val_loss: 1.6848 - val_acc: 0.3200\n",
      "Epoch 36/40\n",
      "950/950 [==============================] - 14s 15ms/step - loss: 1.6434 - acc: 0.3463 - val_loss: 1.5814 - val_acc: 0.4200\n",
      "Epoch 37/40\n",
      "950/950 [==============================] - 14s 15ms/step - loss: 1.6436 - acc: 0.3400 - val_loss: 1.6272 - val_acc: 0.3400\n",
      "Epoch 38/40\n",
      "950/950 [==============================] - 15s 15ms/step - loss: 1.6083 - acc: 0.3653 - val_loss: 1.6147 - val_acc: 0.3800\n",
      "Epoch 39/40\n",
      "950/950 [==============================] - 15s 15ms/step - loss: 1.5794 - acc: 0.3737 - val_loss: 1.6384 - val_acc: 0.3800\n",
      "Epoch 40/40\n",
      "950/950 [==============================] - 14s 15ms/step - loss: 1.6078 - acc: 0.3663 - val_loss: 1.6592 - val_acc: 0.3400\n",
      "1000/1000 [==============================] - 3s 3ms/step\n",
      "Test loss / test accuracy = 1.6684 / 0.3340\n"
     ]
    }
   ],
   "source": [
    "print('Training')\n",
    "model.fit([x, xq], y,\n",
    "          batch_size=BATCH_SIZE,\n",
    "          epochs=EPOCHS,\n",
    "          validation_split=0.05)\n",
    "loss, acc = model.evaluate([tx, txq], ty,\n",
    "                           batch_size=BATCH_SIZE)\n",
    "print('Test loss / test accuracy = {:.4f} / {:.4f}'.format(loss, acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}