06.1 Question matching.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import re\n",
    "import os\n",
    "import keras.backend as K\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from keras import layers, models, utils\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def reset_everything():\n",
    "    import tensorflow as tf\n",
    "    %reset -f in out dhist\n",
    "    tf.reset_default_graph()\n",
    "    K.set_session(tf.InteractiveSession())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Constants for our networks.  We keep these deliberately small to reduce training time.\n",
    "\n",
    "VOCAB_SIZE = 250000\n",
    "EMBEDDING_SIZE = 100\n",
    "MAX_DOC_LEN = 128\n",
    "MIN_DOC_LEN = 12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "87000/1000000"
     ]
    }
   ],
   "source": [
    "def extract_stackexchange(filename, limit=1000000):\n",
    "    json_file = filename + 'limit=%s.json' % limit\n",
    "\n",
    "    rows = []\n",
    "    for i, line in enumerate(os.popen('7z x -so \"%s\" Posts.xml' % filename)):\n",
    "        line = str(line)\n",
    "        if not line.startswith('  <row'):\n",
    "            continue\n",
    "            \n",
    "        if i % 1000 == 0:\n",
    "            print('\\r%05d/%05d' % (i, limit), end='', flush=True)\n",
    "\n",
    "        parts = line[6:-5].split('\"')\n",
    "        record = {}\n",
    "        for i in range(0, len(parts), 2):\n",
    "            k = parts[i].replace('=', '').strip()\n",
    "            v = parts[i+1].strip()\n",
    "            record[k] = v\n",
    "        rows.append(record)\n",
    "        \n",
    "        if len(rows) > limit:\n",
    "            break\n",
    "    \n",
    "    with open(json_file, 'w') as fout:\n",
    "        json.dump(rows, fout)\n",
    "    \n",
    "    return rows\n",
    "\n",
    "\n",
    "xml_7z = utils.get_file(\n",
    "    fname='travel.stackexchange.com.7z',\n",
    "    origin='https://ia800107.us.archive.org/27/items/stackexchange/travel.stackexchange.com.7z',\n",
    ")\n",
    "print()\n",
    "\n",
    "rows = extract_stackexchange(xml_7z)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Exploration\n",
    "\n",
    "Now that we have extracted our data, let's clean it up and take a look at what we have to work with."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame.from_records(rows)    \n",
    "df = df.set_index('Id', drop=False)\n",
    "df['Title'] = df['Title'].fillna('').astype('str')\n",
    "df['Tags'] = df['Tags'].fillna('').astype('str')\n",
    "df['Body'] = df['Body'].fillna('').astype('str')\n",
    "df['Id'] = df['Id'].astype('int')\n",
    "df['PostTypeId'] = df['PostTypeId'].astype('int')\n",
    "df['ViewCount'] = df['ViewCount'].astype('float')\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "list(df[df['ViewCount'] > 250000]['Title'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "tokenizer = Tokenizer(num_words=VOCAB_SIZE)\n",
    "tokenizer.fit_on_texts(df['Body'] + df['Title'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Compute TF/IDF Values\n",
    "\n",
    "total_count = sum(tokenizer.word_counts.values())\n",
    "idf = { k: np.log(total_count/v) for (k,v) in tokenizer.word_counts.items() }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Download pre-trained word2vec embeddings\n",
    "\n",
    "import gensim\n",
    "\n",
    "glove_100d = utils.get_file(\n",
    "    fname='glove.6B.100d.txt',\n",
    "    origin='https://storage.googleapis.com/deep-learning-cookbook/glove.6B.100d.txt',\n",
    ")\n",
    "\n",
    "w2v_100d = glove_100d + '.w2v'\n",
    "from gensim.scripts.glove2word2vec import glove2word2vec\n",
    "glove2word2vec(glove_100d, w2v_100d)\n",
    "w2v_model = gensim.models.KeyedVectors.load_word2vec_format(w2v_100d)\n",
    "\n",
    "w2v_weights = np.zeros((VOCAB_SIZE, w2v_model.syn0.shape[1]))\n",
    "idf_weights = np.zeros((VOCAB_SIZE, 1))\n",
    "\n",
    "for k, v in tokenizer.word_index.items():\n",
    "    if v >= VOCAB_SIZE:\n",
    "        continue\n",
    "    \n",
    "    if k in w2v_model:\n",
    "        w2v_weights[v] = w2v_model[k]\n",
    "    \n",
    "    idf_weights[v] = idf[k]\n",
    "    \n",
    "del w2v_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['title_tokens'] = tokenizer.texts_to_sequences(df['Title'])\n",
    "df['body_tokens'] = tokenizer.texts_to_sequences(df['Body'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "# We can create a data generator that will randomly title and body tokens for questions.  We'll use random text\n",
    "# from other questions as a negative example when necessary.\n",
    "def data_generator(batch_size, negative_samples=1):\n",
    "    questions = df[df['PostTypeId'] == 1]\n",
    "    all_q_ids = list(questions.index)\n",
    "        \n",
    "    batch_x_a = []\n",
    "    batch_x_b = []\n",
    "    batch_y = []\n",
    "    \n",
    "    def _add(x_a, x_b, y):\n",
    "        batch_x_a.append(x_a[:MAX_DOC_LEN])\n",
    "        batch_x_b.append(x_b[:MAX_DOC_LEN])\n",
    "        batch_y.append(y)\n",
    "    \n",
    "    while True:\n",
    "        questions = questions.sample(frac=1.0)\n",
    "        \n",
    "        for i, q in questions.iterrows():\n",
    "            _add(q['title_tokens'], q['body_tokens'], 1)\n",
    "            \n",
    "            negative_q = random.sample(all_q_ids, negative_samples)\n",
    "            for nq_id in negative_q:\n",
    "                _add(q['title_tokens'], df.at[nq_id, 'body_tokens'], 0)            \n",
    "            \n",
    "            if len(batch_y) >= batch_size:\n",
    "                yield ({\n",
    "                    'title': pad_sequences(batch_x_a, maxlen=None),\n",
    "                    'body': pad_sequences(batch_x_b, maxlen=None),\n",
    "                }, np.asarray(batch_y))\n",
    "                \n",
    "                batch_x_a = []\n",
    "                batch_x_b = []\n",
    "                batch_y = []\n",
    "\n",
    "# dg = data_generator(1, 2)\n",
    "# next(dg)\n",
    "# next(dg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Embedding Lookups\n",
    "\n",
    "Let's define a helper class for looking up our embedding results.  We'll use it\n",
    "to verify our models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "questions = df[df['PostTypeId'] == 1]['Title'].reset_index(drop=True)\n",
    "question_tokens = pad_sequences(tokenizer.texts_to_sequences(questions))\n",
    "\n",
    "class EmbeddingWrapper(object):\n",
    "    def __init__(self, model):\n",
    "        self._r = questions\n",
    "        self._i = {i:s for (i, s) in enumerate(questions)}\n",
    "        self._w = model.predict({'title': question_tokens}, verbose=1, batch_size=1024)\n",
    "        self._model = model\n",
    "        self._norm = np.sqrt(np.sum(self._w * self._w + 1e-5, axis=1))\n",
    "\n",
    "    def nearest(self, sentence, n=10):\n",
    "        x = tokenizer.texts_to_sequences([sentence])\n",
    "        if len(x[0]) < MIN_DOC_LEN:\n",
    "            x[0] += [0] * (MIN_DOC_LEN - len(x))\n",
    "        e = self._model.predict(np.asarray(x))[0]\n",
    "        norm_e = np.sqrt(np.dot(e, e))\n",
    "        dist = np.dot(self._w, e) / (norm_e * self._norm)\n",
    "\n",
    "        top_idx = np.argsort(dist)[-n:]\n",
    "        return pd.DataFrame.from_records([\n",
    "            {'question': self._r[i], 'dist': float(dist[i])}\n",
    "            for i in top_idx\n",
    "        ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Our first model will just sum up the embeddings of each token.\n",
    "# The similarity between documents will be the dot product of the final embedding.\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "def sum_model(embedding_size, vocab_size, embedding_weights=None, idf_weights=None):\n",
    "    title = layers.Input(shape=(None,), dtype='int32', name='title')\n",
    "    body = layers.Input(shape=(None,), dtype='int32', name='body')\n",
    "\n",
    "    def make_embedding(name):\n",
    "        if embedding_weights is not None:\n",
    "            embedding = layers.Embedding(mask_zero=True, input_dim=vocab_size, output_dim=w2v_weights.shape[1], \n",
    "                                         weights=[w2v_weights], trainable=False, \n",
    "                                         name='%s/embedding' % name)\n",
    "        else:\n",
    "            embedding = layers.Embedding(mask_zero=True, input_dim=vocab_size, output_dim=embedding_size,\n",
    "                                        name='%s/embedding' % name)\n",
    "\n",
    "        if idf_weights is not None:\n",
    "            idf = layers.Embedding(mask_zero=True, input_dim=vocab_size, output_dim=1, \n",
    "                                   weights=[idf_weights], trainable=False,\n",
    "                                   name='%s/idf' % name)\n",
    "        else:\n",
    "            idf = layers.Embedding(mask_zero=True, input_dim=vocab_size, output_dim=1,\n",
    "                                   name='%s/idf' % name)\n",
    "            \n",
    "        return embedding, idf\n",
    "    \n",
    "    embedding_a, idf_a = make_embedding('a')\n",
    "    embedding_b, idf_b = embedding_a, idf_a\n",
    "#     embedding_b, idf_b = make_embedding('b')\n",
    "\n",
    "    mask = layers.Masking(mask_value=0)\n",
    "    def _combine_and_sum(args):\n",
    "        [embedding, idf] = args\n",
    "        return K.sum(embedding * K.abs(idf), axis=1)\n",
    "\n",
    "    sum_layer = layers.Lambda(_combine_and_sum, name='combine_and_sum')\n",
    "\n",
    "    sum_a = sum_layer([mask(embedding_a(title)), idf_a(title)])\n",
    "    sum_b = sum_layer([mask(embedding_b(body)), idf_b(body)])\n",
    "\n",
    "    sim = layers.dot([sum_a, sum_b], axes=1, normalize=True)\n",
    "    sim_model = models.Model(\n",
    "        inputs=[title, body],\n",
    "        outputs=[sim],\n",
    "    )\n",
    "    sim_model.compile(loss='binary_crossentropy', optimizer='nadam', metrics=['accuracy'])\n",
    "    sim_model.summary()\n",
    "\n",
    "    embedding_model = models.Model(\n",
    "        inputs=[title],\n",
    "        outputs=[sum_a]\n",
    "    )\n",
    "    return sim_model, embedding_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Try using our model with pretrained weights from word2vec\n",
    "\n",
    "sum_model_precomputed, sum_embedding_precomputed = sum_model(\n",
    "    embedding_size=EMBEDDING_SIZE, vocab_size=VOCAB_SIZE,\n",
    "    embedding_weights=w2v_weights, idf_weights=idf_weights\n",
    ")\n",
    "\n",
    "x, y = next(data_generator(batch_size=4096))\n",
    "sum_model_precomputed.evaluate(x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "SAMPLE_QUESTIONS = [\n",
    "    'Roundtrip ticket versus one way',\n",
    "    'Shinkansen from Kyoto to Hiroshima',\n",
    "    'Bus tour of Germany',\n",
    "]\n",
    "\n",
    "def evaluate_sample(lookup):\n",
    "    pd.set_option('display.max_colwidth', 100)\n",
    "    results = []\n",
    "    for q in SAMPLE_QUESTIONS:\n",
    "        print(q)\n",
    "        q_res = lookup.nearest(q, n=4)\n",
    "        q_res['result'] = q_res['question']\n",
    "        q_res['question'] = q\n",
    "        results.append(q_res)\n",
    "\n",
    "    return pd.concat(results)\n",
    "\n",
    "lookup = EmbeddingWrapper(model=sum_embedding_precomputed)\n",
    "evaluate_sample(lookup)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training our own network\n",
    "\n",
    "The results are okay but not great... instead of using the word2vec embeddings, what happens if we train our network end-to-end?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sum_model_trained, sum_embedding_trained = sum_model(\n",
    "    embedding_size=EMBEDDING_SIZE, vocab_size=VOCAB_SIZE, \n",
    "    embedding_weights=None,\n",
    "    idf_weights=None\n",
    ")\n",
    "sum_model_trained.fit_generator(\n",
    "    data_generator(batch_size=128),\n",
    "    epochs=10,\n",
    "    steps_per_epoch=1000\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lookup = EmbeddingWrapper(model=sum_embedding_trained)\n",
    "evaluate_sample(lookup)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CNN Model\n",
    "\n",
    "Using a sum-of-embeddings model works well. What happens if we try to make a simple CNN model?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cnn_model(embedding_size, vocab_size):\n",
    "    title = layers.Input(shape=(None,), dtype='int32', name='title')\n",
    "    body = layers.Input(shape=(None,), dtype='int32', name='body')\n",
    "\n",
    "    embedding = layers.Embedding(\n",
    "        mask_zero=False,\n",
    "        input_dim=vocab_size,\n",
    "        output_dim=embedding_size,\n",
    "    )\n",
    "\n",
    "\n",
    "    def _combine_sum(v):\n",
    "        return K.sum(v, axis=1)\n",
    "\n",
    "    cnn_1 = layers.Convolution1D(256, 3)\n",
    "    cnn_2 = layers.Convolution1D(256, 3)\n",
    "    cnn_3 = layers.Convolution1D(256, 3)\n",
    "    \n",
    "    global_pool = layers.GlobalMaxPooling1D()\n",
    "    local_pool = layers.MaxPooling1D(strides=2, pool_size=3)\n",
    "\n",
    "    def forward(input):\n",
    "        embed = embedding(input)\n",
    "        return global_pool(\n",
    "            cnn_2(local_pool(cnn_1(embed))))\n",
    "\n",
    "    sum_a = forward(title)\n",
    "    sum_b = forward(body)\n",
    "\n",
    "    sim = layers.dot([sum_a, sum_b], axes=1, normalize=False)\n",
    "    sim_model = models.Model(\n",
    "        inputs=[title, body],\n",
    "        outputs=[sim],\n",
    "    )\n",
    "    sim_model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['accuracy'])\n",
    "\n",
    "    embedding_model = models.Model(\n",
    "        inputs=[title],\n",
    "        outputs=[sum_a]\n",
    "    )\n",
    "    return sim_model, embedding_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cnn, cnn_embedding = cnn_model(embedding_size=25, vocab_size=VOCAB_SIZE)\n",
    "cnn.summary()\n",
    "cnn.fit_generator(\n",
    "    data_generator(batch_size=128),\n",
    "    epochs=10,\n",
    "    steps_per_epoch=1000,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lookup = EmbeddingWrapper(model=cnn_embedding)\n",
    "evaluate_sample(lookup)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSTM Model\n",
    "\n",
    "We can also make an LSTM model.  Warning, this will be very slow to train and evaluate unless you have a relatively fast GPU to run it on!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def lstm_model(embedding_size, vocab_size):\n",
    "    title = layers.Input(shape=(None,), dtype='int32', name='title')\n",
    "    body = layers.Input(shape=(None,), dtype='int32', name='body')\n",
    "\n",
    "    embedding = layers.Embedding(\n",
    "        mask_zero=True,\n",
    "        input_dim=vocab_size,\n",
    "        output_dim=embedding_size,\n",
    "#         weights=[w2v_weights],\n",
    "#         trainable=False\n",
    "    )\n",
    "\n",
    "    lstm_1 = layers.LSTM(units=512, return_sequences=True)\n",
    "    lstm_2 = layers.LSTM(units=512, return_sequences=False)\n",
    "    \n",
    "    sum_a = lstm_2(lstm_1(embedding(title)))\n",
    "    sum_b = lstm_2(lstm_1(embedding(body)))\n",
    "\n",
    "    sim = layers.dot([sum_a, sum_b], axes=1, normalize=True)\n",
    "#     sim = layers.Activation(activation='sigmoid')(sim)\n",
    "    sim_model = models.Model(\n",
    "        inputs=[title, body],\n",
    "        outputs=[sim],\n",
    "    )\n",
    "    sim_model.compile(loss='binary_crossentropy', optimizer='rmsprop')\n",
    "\n",
    "    embedding_model = models.Model(\n",
    "        inputs=[title],\n",
    "        outputs=[sum_a]\n",
    "    )\n",
    "    return sim_model, embedding_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lstm, lstm_embedding = lstm_model(embedding_size=EMBEDDING_SIZE, vocab_size=VOCAB_SIZE)\n",
    "lstm.summary()\n",
    "lstm.fit_generator(\n",
    "    data_generator(batch_size=128),\n",
    "    epochs=10,\n",
    "    steps_per_epoch=100,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lookup = EmbeddingWrapper(model=lstm_embedding)\n",
    "evaluate_sample(lookup)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}