word2vec.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# `Word2Vec`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Vanilla PyLibraries\n",
    "import os\n",
    "import sys\n",
    "import datetime as dt\n",
    "\n",
    "# Third party Librarires\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "\n",
    "# Custom libraries\n",
    "from dataset import TextDataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## initializing `Word2Vec`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing 8,224 of 8,224 sentences.\tTime taken: 0:00:39.736668"
     ]
    }
   ],
   "source": [
    "data_dir = 'datasets/wiki.valid.raw'\n",
    "save_file = 'datasets/saved/data.pkl'\n",
    "\n",
    "w2v = TextDataset(data_dir=data_dir, logging=True)\n",
    "w2v.create()\n",
    "# w2v.save(save_file=save_file)\n",
    "# w2v = w2v.load(save_file=save_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Model/Network\n",
    "embedding_dim = 50\n",
    "learning_rate = 1e-3\n",
    "vocab_size = w2v.vocab_size\n",
    "\n",
    "# Training\n",
    "epochs = 5000\n",
    "save_interval = 100\n",
    "batch_size = 25"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Training with a `tensorflow` model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Model's placeholders\n",
    "X = tf.placeholder(tf.float32, shape=[None, vocab_size], name='X_palceholder')\n",
    "y = tf.placeholder(tf.float32, shape=[None, vocab_size], name='y_placeholder')\n",
    "y_true = tf.argmax(y, axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Building the Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Input -> Hidden\n",
    "W1 = tf.Variable(tf.truncated_normal(shape=[vocab_size, embedding_dim]))\n",
    "b1 = tf.Variable(tf.zeros(shape=[embedding_dim]))\n",
    "hidden = tf.matmul(X, W1) + b1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Hidden -> Output\n",
    "W2 = tf.Variable(tf.truncated_normal(shape=[embedding_dim, vocab_size]))\n",
    "b2 = tf.Variable(tf.zeros(shape=[vocab_size]))\n",
    "y_hat = tf.matmul(hidden, W2) + b2\n",
    "y_norm = tf.nn.softmax(y_hat)\n",
    "y_pred = tf.argmax(y_norm, axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Loss, training and accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "xentropy = tf.nn.softmax_cross_entropy_with_logits(logits=y_hat, labels=y, name='xentropy')\n",
    "loss = tf.reduce_mean(xentropy, name='loss')\n",
    "\n",
    "# optimizer\n",
    "global_step = tf.Variable(0, trainable=False, name='global_step')\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "train_step = optimizer.minimize(loss, global_step=global_step)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "correct = tf.equal(y_pred, y_true)\n",
    "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Initializing global variables and  `tf.Session()`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "sess = tf.Session()\n",
    "init = tf.global_variables_initializer()\n",
    "sess.run(init)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Files & directories\n",
    "saved_path = 'saved/'\n",
    "\n",
    "model_dir = os.path.join(saved_path, 'models/')  # Trained model's directory\n",
    "model_path = os.path.join(model_dir, 'model.ckpt')  # Pre-trained model\n",
    "tensorboard_dir = os.path.join(saved_path, 'tensorboard/')  # summary protobuf\n",
    "logdir = os.path.join(tensorboard_dir, 'log')   # summary's file writer\n",
    "\n",
    "# Summaries\n",
    "tf.summary.scalar('Loss', loss)\n",
    "tf.summary.scalar('Accuracy', accuracy)\n",
    "merged = tf.summary.merge_all()\n",
    "\n",
    "# saver & writer\n",
    "saver = tf.train.Saver()\n",
    "writer = tf.summary.FileWriter(logdir=logdir, graph=sess.graph)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Restore last checkpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO: Attempting to restore last checkpoint\n",
      "INFO:tensorflow:Restoring parameters from saved/models/model.ckpt-4901\n",
      "INFO: Successfully restored checkpoint - saved/models/model.ckpt-4901\n"
     ]
    }
   ],
   "source": [
    "# Restore or create\n",
    "if tf.gfile.Exists(model_dir):\n",
    "    try:\n",
    "        print('INFO: Attempting to restore last checkpoint')\n",
    "        last_ckpt = tf.train.latest_checkpoint(model_dir)\n",
    "        saver.restore(sess=sess, save_path=last_ckpt)\n",
    "        print(f'INFO: Successfully restored checkpoint - {last_ckpt}')\n",
    "    except Exception as e:\n",
    "        sys.stderr.write(f'ERR: Could not restore checkpoint. {e}')\n",
    "        sys.stderr.flush()\n",
    "else:\n",
    "    tf.gfile.MakeDirs(model_dir)\n",
    "    print(f'INFO: Checkpoint directory created - {model_dir}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter: 2,373\tGlobal steps: 7,274\tTime taken: 0:02:56.965823"
     ]
    }
   ],
   "source": [
    "train_start = dt.datetime.now()\n",
    "for i in range(epochs):\n",
    "    # Train\n",
    "    X_batch, y_batch = w2v.next_batch(batch_size=batch_size, shuffle=True)\n",
    "    _, i_global = sess.run([train_step, global_step], feed_dict={X: X_batch, y: y_batch})\n",
    "    # Save at interval\n",
    "    if i % save_interval == 0:\n",
    "        # Save model\n",
    "        saver.save(sess=sess, save_path=model_path, global_step=global_step)\n",
    "        # Tensorboard summary\n",
    "        summary = sess.run(merged, feed_dict={X: X_batch, y: y_batch})\n",
    "        writer.add_summary(summary=summary, global_step=i_global)\n",
    "    sys.stdout.write(f'\\rIter: {i+1:,}\\tGlobal steps: {i_global:,}'\n",
    "                     f'\\tTime taken: {dt.datetime.now() - train_start}')\n",
    "    sys.stdout.flush()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Word vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "word_vectors = sess.run(W1 + b1)\n",
    "print(word_vectors.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "word_vectors[w2v.word2id['you']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}