Encoder-Decoder with Attention.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"../Pics/MLSb-T.png\" width=\"160\">\n",
    "<br><br>\n",
    "<center><u><H1>Encoder-Decoder with Attention</H1></u></center>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from keras.backend.tensorflow_backend import set_session\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "config.log_device_placement = True\n",
    "sess = tf.Session(config=config)\n",
    "set_session(sess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import CuDNNLSTM, LSTM\n",
    "from keras.layers import Dense\n",
    "from keras.layers import TimeDistributed\n",
    "from keras.layers import RepeatVector\n",
    "from random import randint\n",
    "from numpy import array\n",
    "from numpy import argmax\n",
    "from numpy import array_equal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to generate random integer values between 0 and n. The number of timestpes is: length\n",
    "def gen_seq(length, n):\n",
    "    return [randint(0, n-1) for _ in range(length)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate sequence:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sequence = gen_seq(6, 30)\n",
    "sequence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def onehot_encoder(seq, n):\n",
    "    encod = []\n",
    "    for s in seq:\n",
    "        v = [0 for _ in range(n)]\n",
    "        v[s] = 1\n",
    "        encod.append(v)\n",
    "    return array(encod)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def onehot_decoder(encod_seq):\n",
    "    return [argmax(idx) for idx in encod_seq]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "onehot = onehot_encoder(sequence, 30)\n",
    "print(onehot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "decoded = onehot_decoder(onehot)\n",
    "print(decoded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_pair(n_in, n_out, n_total):\n",
    "    # generating random sequences\n",
    "    seq_in = gen_seq(n_in, n_total)\n",
    "    seq_out = seq_in[:n_out] + [0 for _ in range(n_in-n_out)]\n",
    "    \n",
    "    X = onehot_encoder(seq_in, n_total)\n",
    "    y = onehot_encoder(seq_out, n_total)\n",
    "    \n",
    "    # reshaping as 3D tensor\n",
    "    X = X.reshape((1, X.shape[0], X.shape[1]))\n",
    "    y = y.reshape((1, y.shape[0], y.shape[1]))\n",
    "    return X,y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = generate_pair(6, 3, 30)\n",
    "print('X=%s, y=%s' % (onehot_decoder(X[0]), onehot_decoder(y[0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(X.shape, y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Encoder-Decoder without Attention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_features = 50\n",
    "n_timesteps_in = 5\n",
    "n_timesteps_out = 3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating the model: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.add(CuDNNLSTM(150, input_shape=(n_timesteps_in, n_features)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.add(RepeatVector(n_timesteps_in))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.add(CuDNNLSTM(150, return_sequences=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.add(TimeDistributed(Dense(n_features, activation='softmax')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training the model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for epoch in range(5000):\n",
    "    X,y = generate_pair(n_timesteps_in, n_timesteps_out, n_features)\n",
    "    model.fit(X, y, epochs=1, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing the model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 100\n",
    "correct = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Testing the model with new 100 new randomly generated integer sequences\n",
    "for _ in range(epochs):\n",
    "    X,y = generate_pair(n_timesteps_in, n_timesteps_out, n_features)\n",
    "    pred = model.predict(X)\n",
    "    if array_equal(onehot_decoder(y[0]), onehot_decoder(pred[0])):\n",
    "        correct += 1\n",
    "print('Accuracy: %.2f%%' % (float(correct)/float(epochs)*100.0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Checking samples:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#checking 20 examples of expected output sequences and predictions\n",
    "for _ in range(20):\n",
    "    X,y = generate_pair(n_timesteps_in, n_timesteps_out, n_features)\n",
    "    pred = model.predict(X)\n",
    "    print('Expected:', onehot_decoder(y[0]), 'Predicted', onehot_decoder(pred[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Encoder-Decoder with Attention Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from attention_decoder import AttentionDecoder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating the model with Attention:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_att = Sequential()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_att.add(CuDNNLSTM(150, input_shape=(n_timesteps_in, n_features), return_sequences=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_att.add(CuDNNLSTM(150, return_sequences=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_att.add(CuDNNLSTM(150, return_sequences=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_att.add(AttentionDecoder(150, n_features))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_att.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_att.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for epoch in range(5000):\n",
    "    X, y = generate_pair(n_timesteps_in, n_timesteps_out, n_features)\n",
    "    model_att.fit(X, y, epochs=1, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing the attention model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for _ in range(epochs):\n",
    "    X, y = generate_pair(n_timesteps_in, n_timesteps_out, n_features)\n",
    "    pred_att = model_att.predict(X)\n",
    "    if array_equal(onehot_decoder(y[0]), onehot_decoder(pred_att[0])):\n",
    "        correct += 1\n",
    "print('Accuracy: %.2f%%' % (float(correct)/float(epochs)*100.0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Checking samples:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for _ in range(20):\n",
    "    X,y = generate_pair(n_timesteps_in, n_timesteps_out, n_features)\n",
    "    pred = model_att.predict(X)\n",
    "    print('Expected:', onehot_decoder(y[0]), 'Predicted', onehot_decoder(pred[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf1.11_gpu",
   "language": "python",
   "name": "tf111_gpu"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}