Text Generation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"../Pics/MLSb-T.png\" width=\"160\">\n",
    "<br><br>\n",
    "<center><u><H1>Text Generation</H1></u></center>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from keras.backend.tensorflow_backend import set_session\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "config.log_device_placement = True\n",
    "sess = tf.Session(config=config)\n",
    "set_session(sess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from keras.models import Sequential, load_model\n",
    "from keras.layers import Dense, Embedding, CuDNNLSTM, Dropout\n",
    "from keras.utils import to_categorical\n",
    "from random import randint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = open('../data/Ancient_Modern_Physics.txt','r')\n",
    "text = file.read()\n",
    "file.close()\n",
    "text[:1000]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cleaning the text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens = text.lower()\n",
    "print(tokens[:500])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_chars = len(tokens)\n",
    "unique_vocab = len(set(tokens))\n",
    "print('Total Tokens: %d' % n_chars)\n",
    "print('Unique Tokens: %d' % unique_vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "characters = sorted(list(set(tokens)))\n",
    "n_vocab = len(characters)\n",
    "n_vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "int_to_char = {n:char for n, char in enumerate(characters)}\n",
    "char_to_int = {char:n for n, char in enumerate(characters)}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating datasets:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = []\n",
    "y = []\n",
    "seq_length = 100\n",
    "\n",
    "for i in range(0, n_chars - seq_length, 1):\n",
    "    seq_in = tokens[i:i + seq_length]\n",
    "    seq_out = tokens[i + seq_length]\n",
    "    X.append([char_to_int[char] for char in seq_in])\n",
    "    y.append(char_to_int[seq_out])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(X[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(y[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_new = np.reshape(X, (len(X), seq_length, 1)) #samples, time steps, features\n",
    "X_new = X_new / float(n_vocab) #normalizing the values\n",
    "\n",
    "y_new = to_categorical(y) #one hot encode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"X_new shape:\", X_new.shape)\n",
    "print(\"y_new shape:\", y_new.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_new[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating the model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(CuDNNLSTM(700, input_shape=(X_new.shape[1], X_new.shape[2]), return_sequences=True))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(CuDNNLSTM(700, return_sequences=True))\n",
    "model.add(Dropout(0.2)) \n",
    "model.add(CuDNNLSTM(700))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(y_new.shape[1], activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss='categorical_crossentropy', optimizer='adam')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training the model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "model.fit(X_new, y_new, batch_size=64, epochs=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save the model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('../data/text_generation/text_generation_model.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_ = load_model('../data/text_generation/text_generation_model.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing a random sample:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ini = np.random.randint(0, len(X)-1)\n",
    "token_string = X[ini]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "complete_string = [int_to_char[value] for value in token_string]\n",
    "\n",
    "print (\"\\\"\", ''.join(complete_string), \"\\\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(500):\n",
    "    x = np.reshape(token_string, (1, len(token_string), 1))\n",
    "    x = x / float(n_vocab)\n",
    "    \n",
    "    prediction = model_.predict(x, verbose=0)\n",
    "\n",
    "    id_pred = np.argmax(prediction)\n",
    "    seq_in = [int_to_char[value] for value in token_string]\n",
    "    \n",
    "    complete_string.append(int_to_char[id_pred])\n",
    "    \n",
    "    token_string.append(id_pred)\n",
    "    token_string = token_string[1:len(token_string)] "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Showing the generated text:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = \"\"\n",
    "for char in complete_string:\n",
    "    text = text + char\n",
    "print(text)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf1.11_gpu",
   "language": "python",
   "name": "tf111_gpu"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}