Encoder. Decoder. LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from tensorflow import keras\n",
    "from keras.models import Model\n",
    "from keras.layers import Input, Dense\n",
    "from keras.layers.recurrent import LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size = 64\n",
    "epochs = 100\n",
    "latent_dim = 256\n",
    "num_texts = 10000 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "metadata": {},
   "outputs": [],
   "source": [
    "inp_text, out_text = [], []\n",
    "with open('spa.txt') as f:\n",
    "    for line in f:\n",
    "        x, y = line.split('\\t')\n",
    "        inp_text.append(x)\n",
    "        out_text.append(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Ve.\\n'"
      ]
     },
     "execution_count": 264,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out_text[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "inp_chars = ['\\t'] + sorted(list(set(\" \".join(inp_text))))\n",
    "out_chars = ['\\t'] + sorted(list(set(\" \".join(out_text))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(91, 114)"
      ]
     },
     "execution_count": 266,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(inp_chars), len(out_chars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "inp_max_len, out_max_len = max([len(x) for x in inp_text[:num_texts]]), max([len(x) for x in out_text[:num_texts]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 268,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(17, 41)"
      ]
     },
     "execution_count": 268,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inp_max_len, out_max_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "inp_c_ind = { c : i for i,c in enumerate(inp_chars) }\n",
    "out_c_ind = { c : i for i,c in enumerate(out_chars) }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 270,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "inp_data = np.zeros((num_texts, inp_max_len, len(inp_chars)))\n",
    "out_data = np.zeros((num_texts, out_max_len+1, len(out_chars)))\n",
    "out_target = np.zeros((num_texts, out_max_len+1, len(out_chars)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, s in enumerate(inp_text[:num_texts]):\n",
    "    inp_shift = inp_max_len - len(s)\n",
    "    for j, c in enumerate(s):\n",
    "        inp_data[i, j, inp_c_ind[c]] = 1.\n",
    "    out_data[i, 0, 0] = 1.\n",
    "    for j, c in enumerate(out_text[i]):\n",
    "        out_data[i, j+1, out_c_ind[c]] = 1.\n",
    "        out_target[i, j, out_c_ind[c]] = 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Input, Dense\n",
    "from keras.layers.recurrent import LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "enc_input = Input(shape=(inp_max_len, len(inp_chars)))\n",
    "dec_input = Input(shape=(out_max_len+1, len(out_chars)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "encoder = LSTM(latent_dim, return_state=True)\n",
    "enc_output, enc_h, enc_c = encoder(enc_input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "decoder = LSTM(latent_dim, return_state=True, return_sequences=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 276,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dec_output, dec_h, dec_c = decoder(dec_input, initial_state=[enc_h, enc_c])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dec_dense = Dense(len(out_chars), activation='softmax')\n",
    "dec_chars = dec_dense(dec_output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 278,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.models import Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 279,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = Model([enc_input, dec_input], dec_chars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 280,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(optimizer='rmsprop', loss='categorical_crossentropy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 281,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "input_35 (InputLayer)            (None, 17, 91)        0                                            \n",
      "____________________________________________________________________________________________________\n",
      "input_36 (InputLayer)            (None, 42, 114)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_11 (LSTM)                   [(None, 256), (None,  356352      input_35[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "lstm_12 (LSTM)                   [(None, 42, 256), (No 379904      input_36[0][0]                   \n",
      "                                                                   lstm_11[0][1]                    \n",
      "                                                                   lstm_11[0][2]                    \n",
      "____________________________________________________________________________________________________\n",
      "dense_6 (Dense)                  (None, 42, 114)       29298       lstm_12[0][0]                    \n",
      "====================================================================================================\n",
      "Total params: 765,554\n",
      "Trainable params: 765,554\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.fit([inp_data, out_data], out_target, batch_size=batch_size, epochs=50, validation_split=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.fit([inp_data, out_data], out_target, batch_size=batch_size, epochs=50, validation_split=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "metadata": {},
   "outputs": [],
   "source": [
    "enc_model = Model(enc_input, (enc_h, enc_c))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "metadata": {},
   "outputs": [],
   "source": [
    "dec_sample_h = Input(shape=(latent_dim,))\n",
    "dec_sample_c = Input(shape=(latent_dim,))\n",
    "dec_sample_lstm, _, _ = decoder(dec_input, initial_state=(dec_sample_h, dec_sample_c))\n",
    "dec_sample_char = dec_dense(dec_sample_lstm)\n",
    "dec_sampler = Model([dec_input, dec_sample_h, dec_sample_c], dec_sample_char)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "s = 'Hello!'\n",
    "def convert_to_inp_data(s):\n",
    "    res = np.zeros((1, inp_max_len, len(inp_chars)))\n",
    "    inp_shift = inp_max_len - len(s)\n",
    "    for j, c in enumerate(s):\n",
    "        inp_data[i, inp_shift + j, inp_c_ind[c]] = 1.\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 17, 91)"
      ]
     },
     "execution_count": 286,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "convert_to_inp_data(s).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 287,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cur_h, cur_c = enc_model.predict(convert_to_inp_data('Hello!'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 288,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 256)"
      ]
     },
     "execution_count": 288,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cur_h.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 289,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def convert_to_out_data(s):\n",
    "    res = np.zeros((1, out_max_len+1, len(out_chars)))\n",
    "    res[0, 0, 0] = 1.\n",
    "    out_target[i, len(out_text[i]), 0] = 1.\n",
    "    for j, c in enumerate(s):\n",
    "        out_data[i, j+1, out_c_ind[c]] = 1.\n",
    "        out_target[i, j, out_c_ind[c]] = 1.\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 290,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 42, 114)"
      ]
     },
     "execution_count": 290,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "convert_to_out_data('').shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 291,
   "metadata": {},
   "outputs": [],
   "source": [
    "def translate_sentence(s):\n",
    "    cur_h, cur_c = enc_model.predict(convert_to_inp_data(s))\n",
    "    out_string = ''\n",
    "    while len(out_string) < out_max_len:\n",
    "        x = convert_to_out_data(out_string)\n",
    "        model_output = dec_sampler.predict([x, cur_h, cur_c])\n",
    "        y = out_chars[np.argmax(model_output[0,len(out_string),:])]\n",
    "        if y == '\\t':\n",
    "            break\n",
    "        out_string = out_string + y\n",
    "    return out_string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "translate_sentence(\"Hi!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = 'Hi!'\n",
    "cur_h, cur_c = enc_model.predict(convert_to_inp_data(s))\n",
    "print(cur_h, cur_c)\n",
    "out_string = ''\n",
    "while len(out_string) < out_max_len:\n",
    "    x = convert_to_out_data(out_string)\n",
    "    model_output = dec_sampler.predict([x, cur_h, cur_c])\n",
    "    y = out_chars[np.argmax(model_output[0,len(out_string),:])]\n",
    "    if y == '\\n':\n",
    "        break\n",
    "    out_string = out_string + y\n",
    "print(out_string)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}