NLP Model_Text multi-classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NLP Model_Text Multi-class Classification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 2: Build the NLP Model with Pre-trained GloVe word embeddings and LSTM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# Keras\n",
    "import keras\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Flatten, LSTM, Bidirectional,Conv1D, MaxPooling1D, Dropout, Activation\n",
    "from keras.layers.embeddings import Embedding\n",
    "\n",
    "# Other\n",
    "import re\n",
    "import string\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. GloVe word embeddings "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load pre-trained GloVe word embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 400001 word vectors.\n"
     ]
    }
   ],
   "source": [
    "glove_6B_100d_file_path_name = \"glove.6B/glove.6B.100d.txt\"\n",
    "\n",
    "embeddings_index = dict()\n",
    "\n",
    "f = open(glove_6B_100d_file_path_name)\n",
    "\n",
    "for line in f:\n",
    "    values = line.split()\n",
    "    word = values[0]\n",
    "    coefs = np.asarray(values[1:], dtype='float32')\n",
    "    embeddings_index[word] = coefs\n",
    "    \n",
    "f.close()\n",
    "print('Loaded %s word vectors.' % len(embeddings_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "## windows system version: need to use io.open\n",
    "# embeddings_index = dict()\n",
    "\n",
    "# with io.open(glove_6B_100d_file_path_name, \"r\", encoding=\"utf-8\") as f:\n",
    "\n",
    "#     for line in f:\n",
    "#         values = line.split()\n",
    "#         word = values[0]\n",
    "#         coefs = np.asarray(values[1:], dtype='float32')\n",
    "#         embeddings_index[word] = coefs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### List all GloVe words\n",
    "When fit the tokenizer, use all golove words to fit. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# list all golve words\n",
    "all_glove_words = list(embeddings_index.keys())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tokenizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tokenizer is from Keras.\n",
    "After fit on all GloVe words, tokenizer vocabulary size is 400k words."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tokenize\n",
    "vocabulary_size = len(all_glove_words)\n",
    "tokenizer = Tokenizer() #num_words= vocabulary_size\n",
    "tokenizer.fit_on_texts(all_glove_words) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pickle the tokenizer\n",
    "Using pickled tokenizer when deploy to the cloud web service."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import pickle\n",
    "\n",
    "# with open('tokenizer.pickle', 'wb') as handle:\n",
    "#     pickle.dump(tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creat Embedding Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a weight matrix for words in training docs\n",
    "embedding_matrix = np.zeros((vocabulary_size, 100)) \n",
    "for word, index in tokenizer.word_index.items():\n",
    "    if index > vocabulary_size - 1:\n",
    "        break\n",
    "    else:\n",
    "        embedding_vector = embeddings_index.get(word)\n",
    "        if embedding_vector is not None:\n",
    "            embedding_matrix[index] = embedding_vector\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tokenizer.word_index.items()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(400001, 100)\n"
     ]
    }
   ],
   "source": [
    "print(embedding_matrix.shape)\n",
    "# embedding_matrix[78]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. NLP Model for Multi-classification\n",
    "Categorize the given text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# training \n",
    "path_filename_for_training_data = \"train_data.csv\"\n",
    "data_df = pd.read_csv(path_filename_for_training_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define class labels and turn it to categorical value using keras function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = data_df['label']\n",
    "\n",
    "num_classes = 17\n",
    "labels_cat = keras.utils.to_categorical(np.array(labels), num_classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tokenize the text from training set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "maxlen = 10\n",
    "sequences = tokenizer.texts_to_sequences(data_df['phrase'])\n",
    "data = pad_sequences(sequences, maxlen=maxlen)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Building Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_glove = Sequential()\n",
    "model_glove.add(Embedding(vocabulary_size, output_dim=100, input_length=maxlen, weights=[embedding_matrix], trainable=False))\n",
    "#model_glove.add(Dropout(0.2))\n",
    "#model_glove.add(Conv1D(64, 2, activation='relu')) \n",
    "model_glove.add(LSTM(64))\n",
    "model_glove.add(Dense(17, activation='softmax'))\n",
    "\n",
    "model_glove.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 5383 samples, validate on 2307 samples\n",
      "Epoch 1/2\n",
      "5383/5383 [==============================] - 5s 857us/step - loss: 0.8288 - acc: 0.8209 - val_loss: 0.7325 - val_acc: 0.8960\n",
      "Epoch 2/2\n",
      "5383/5383 [==============================] - 3s 560us/step - loss: 0.0187 - acc: 1.0000 - val_loss: 0.7399 - val_acc: 0.8960\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0xb208f75c0>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# training the model\n",
    "model_glove.fit(data, labels_cat, validation_split=0.3, epochs = 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pickle the model\n",
    "Using pickled model when deploy to the cloud web service."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import pickle\n",
    "\n",
    "# pickle.dump(model_glove, open(\"model_600.pkl\",\"wb\"))\n",
    "# print(\"Successfully pickled.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prediction\n",
    "Before put the test text into model, several steps of preprocessing is needed. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = \"Here put the text you want to test\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nOutput is list of phrases \\nCut all possible phrases from the text with window size of minlen ~ maxlen. \\n'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Convert to lowercase\n",
    "text = text.lower()    \n",
    "# remove special characters\n",
    "text = re.sub(r'[?|!|\\'|\"|#|,|)|(|\\|/$%\\n\\t.:;\"\"‘’]',r'',text)\n",
    "\n",
    "# split text to word\n",
    "word_list = text.split(' ')\n",
    "len_text = len(word_list)\n",
    "\n",
    "# combine the words to make short phrases with 6~10 words (actually 6,8,10)\n",
    "minlen = 6\n",
    "maxlen = 10\n",
    "\n",
    "phrases = []   \n",
    "len_each_phrase = list(set([minlen,(maxlen + minlen)//2, maxlen])) \n",
    "\n",
    "i = 0\n",
    "while i <= len_text:\n",
    "    for nword in len_each_phrase:    \n",
    "        if i+nword >= len_text:\n",
    "            phrase = ' '.join(word_list[i:])\n",
    "            phrases.append(phrase)\n",
    "            i = len_text\n",
    "            break\n",
    "\n",
    "        else:\n",
    "            phrase = ' '.join(word_list[i:i+nword])\n",
    "            phrases.append(phrase)\n",
    "    i += 1\n",
    "'''\n",
    "Output is list of phrases \n",
    "Cut all possible phrases from the text with window size of minlen ~ maxlen. \n",
    "'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tokenize the text (list of phrases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tokenize and padding\n",
    "sequences = tokenizer.texts_to_sequences(phrases)\n",
    "test_data = pad_sequences(sequences, maxlen = maxlen)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predict with model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# predict \n",
    "preds = model_glove.predict(test_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Convert the predicted probability to category"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prob = 0.5\n",
    "\n",
    "preds_df = pd.DataFrame(preds)\n",
    "\n",
    "'''\n",
    "For each class, count the number of the phrases that the probability is higher than threshhold 'prob'.\n",
    "Choose the max as the predicted category. \n",
    "'''\n",
    "count = preds_df[preds_df > prob].count()\n",
    "\n",
    "if sum(count[1:] > 0) > 0:\n",
    "    result = count.idxmax()\n",
    "else:\n",
    "    result = 0\n",
    "\n",
    "# result is the category\n",
    "result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### (optional) Defined a function to predict class\n",
    "Easy to explore test result using function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(text):\n",
    "    # test data\n",
    "    prob = 0.5\n",
    "    minlen = 6\n",
    "    maxlen = 10\n",
    "\n",
    "    text = text.lower()     # Converting to lowercase\n",
    "    text = re.sub(r'[?|!|\\'|\"|#|,|)|(|\\|/$%\\n\\t.:;\"\"‘’]',r'',text)\n",
    "\n",
    "    # split text to word\n",
    "    word_list = text.split(' ')\n",
    "    len_text = len(word_list)\n",
    "    \n",
    "    phrases = []   \n",
    "    len_each_phrase = list(set([minlen,(maxlen + minlen)//2, maxlen]))    \n",
    "\n",
    "    i = 0\n",
    "    while i <= len_text:\n",
    "        for nword in len_each_phrase:    \n",
    "            if i+nword >= len_text:\n",
    "                phrase = ' '.join(word_list[i:])\n",
    "                phrases.append(phrase)\n",
    "                i = len_text\n",
    "                break\n",
    "                \n",
    "            else:\n",
    "                phrase = ' '.join(word_list[i:i+nword])\n",
    "                phrases.append(phrase)\n",
    "        i += 1\n",
    "\n",
    "    sequences = tokenizer.texts_to_sequences(phrases)\n",
    "    test_data = pad_sequences(sequences, maxlen = maxlen)\n",
    "   \n",
    "    # predict with model\n",
    "    preds = model_glove.predict(test_data)\n",
    "    \n",
    "    preds_df = pd.DataFrame(preds)\n",
    "\n",
    "    count = preds_df[preds_df > prob].count()\n",
    "    \n",
    "    if sum(count[1:] > 0) > 0:\n",
    "        result = count.idxmax()\n",
    "    else:\n",
    "        result = 0\n",
    "\n",
    "    return result,preds_df, count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "result,preds_df, count = test(text)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}