GloVe-Yelp-Comments-Classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"../Pics/MLSb-T.png\" width=\"160\">\n",
    "<br><br>\n",
    "<center><u><H1>GloVe-Yelp-Comments-Classification</H1></u></center>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from keras.backend.tensorflow_backend import set_session\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "config.log_device_placement = True\n",
    "sess = tf.Session(config=config)\n",
    "set_session(sess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Embedding, Flatten, Dense, Dropout\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "import numpy as np\n",
    "import os\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import re\n",
    "import nltk\n",
    "import string\n",
    "import plotly.offline as py\n",
    "import plotly.graph_objs as go\n",
    "py.init_notebook_mode(connected=True)\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import SnowballStemmer\n",
    "from sklearn.manifold import TSNE\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load the data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('../data/yelp.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df= df.dropna()\n",
    "df=df[['text','stars']]\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = df['stars'].map(lambda x : 1 if int(x) > 3 else 0)\n",
    "print(labels[10:20])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_text(text):\n",
    "    \n",
    "    ## Remove puncuation\n",
    "    text = text.translate(string.punctuation)\n",
    "    \n",
    "    ## Convert words to lower case and split them\n",
    "    text = text.lower().split()\n",
    "    \n",
    "    ## Remove stop words\n",
    "    stops = set(stopwords.words(\"english\"))\n",
    "    text = [w for w in text if not w in stops and len(w) >= 3]\n",
    "    \n",
    "    text = \" \".join(text)\n",
    "\n",
    "    # Clean the text\n",
    "    text = re.sub(r\"[^A-Za-z0-9^,!.\\/'+-=]\", \" \", text)\n",
    "    text = re.sub(r\"what's\", \"what is \", text)\n",
    "    text = re.sub(r\"\\'s\", \" \", text)\n",
    "    text = re.sub(r\"\\'ve\", \" have \", text)\n",
    "    text = re.sub(r\"n't\", \" not \", text)\n",
    "    text = re.sub(r\"i'm\", \"i am \", text)\n",
    "    text = re.sub(r\"\\'re\", \" are \", text)\n",
    "    text = re.sub(r\"\\'d\", \" would \", text)\n",
    "    text = re.sub(r\"\\'ll\", \" will \", text)\n",
    "    text = re.sub(r\",\", \" \", text)\n",
    "    text = re.sub(r\"\\.\", \" \", text)\n",
    "    text = re.sub(r\"!\", \" ! \", text)\n",
    "    text = re.sub(r\"\\/\", \" \", text)\n",
    "    text = re.sub(r\"\\^\", \" ^ \", text)\n",
    "    text = re.sub(r\"\\+\", \" + \", text)\n",
    "    text = re.sub(r\"\\-\", \" - \", text)\n",
    "    text = re.sub(r\"\\=\", \" = \", text)\n",
    "    text = re.sub(r\"'\", \" \", text)\n",
    "    text = re.sub(r\"(\\d+)(k)\", r\"\\g<1>000\", text)\n",
    "    text = re.sub(r\":\", \" : \", text)\n",
    "    text = re.sub(r\" e g \", \" eg \", text)\n",
    "    text = re.sub(r\" b g \", \" bg \", text)\n",
    "    text = re.sub(r\" u s \", \" american \", text)\n",
    "    text = re.sub(r\"\\0s\", \"0\", text)\n",
    "    text = re.sub(r\" 9 11 \", \"911\", text)\n",
    "    text = re.sub(r\"e - mail\", \"email\", text)\n",
    "    text = re.sub(r\"j k\", \"jk\", text)\n",
    "    text = re.sub(r\"\\s{2,}\", \" \", text)\n",
    "    \n",
    "    text = text.split()\n",
    "    stemmer = SnowballStemmer('english')\n",
    "    stemmed_words = [stemmer.stem(word) for word in text]\n",
    "    text = \" \".join(stemmed_words)\n",
    "\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['text'] = df['text'].map(lambda x: clean_text(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "maxlen = 50\n",
    "embed_dim = 100\n",
    "max_words = 20000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer(num_words=max_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer.fit_on_texts(df['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sequences = tokenizer.texts_to_sequences(df['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pad_sequences(sequences, maxlen=maxlen, padding='post')\n",
    "data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_size = len(tokenizer.word_index) + 1\n",
    "vocab_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = np.asarray(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Shape of data:', data.shape)\n",
    "print('Shape of label:', labels.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating datasets:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_split = .2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "indices = np.arange(data.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.shuffle(indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data[indices]\n",
    "labels = labels[indices]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_samples = int(validation_split * data.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = data[:-val_samples]\n",
    "y_train = labels[:-val_samples]\n",
    "x_val = data[-val_samples:]\n",
    "y_val = labels[-val_samples:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load the GloVe embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir = '../data/GloVe/glove.6B'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embed_index = dict()\n",
    "f = open(os.path.join(dir, 'glove.6B.100d.txt'), encoding=\"utf8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for line in f:\n",
    "    values = line.split()\n",
    "    word = values[0]\n",
    "    coefs = np.asarray(values[1:], dtype='float32')\n",
    "    embed_index[word] = coefs\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('%s Word vectors' % len(embed_index))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create a weight matrix:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embed_matrix = np.zeros((max_words, embed_dim))\n",
    "\n",
    "for word, i in tokenizer.word_index.items():\n",
    "    if i < max_words:\n",
    "        embed_vector = embed_index.get(word)\n",
    "        if embed_vector is not None:\n",
    "            embed_matrix[i] = embed_vector"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating the model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.add(Embedding(max_words,\n",
    "                    embed_dim,\n",
    "                    weights=[embed_matrix],\n",
    "                    input_length=maxlen))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.add(Flatten())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.add(Dropout(0.5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.add(Dense(32, activation='relu'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.add(Dropout(0.5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.add(Dense(1, activation='sigmoid'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "early_stopping = EarlyStopping(monitor='val_loss', patience=5, mode='min')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_best = ModelCheckpoint('../data/yelp_comments.hdf', save_best_only=True, \n",
    "                               monitor='val_loss', mode='min')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "model.fit(X_train, y_train,\n",
    "          epochs=20,\n",
    "          validation_data=(x_val, y_val),\n",
    "          batch_size=128,\n",
    "          verbose=1,\n",
    "          callbacks=[early_stopping, save_best])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Making predictions:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_weights(filepath = '../data/yelp_comments.hdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = model.predict(x_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Word embeddings visualization:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "glove_embds = model.layers[0].get_weights()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = []\n",
    "for word, i in tokenizer.word_index.items():\n",
    "    words.append(word)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualizing words:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_words(data, start, stop, step):\n",
    "    trace = go.Scatter(\n",
    "        x = data[start:stop:step,0], \n",
    "        y = data[start:stop:step, 1],\n",
    "        mode = 'markers',\n",
    "        text= words[start:stop:step]\n",
    "    )\n",
    "    layout = dict(title= 't-SNE_factor1 vs t-SNE_factor2',\n",
    "                  yaxis = dict(title='t-SNE_factor2'),\n",
    "                  xaxis = dict(title='t-SNE_factor1'),\n",
    "                  hovermode= 'closest')\n",
    "    fig = dict(data = [trace], layout= layout)\n",
    "    py.iplot(fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "glove_tsne_embds = TSNE(n_components=2).fit_transform(glove_embds)\n",
    "plot_words(glove_tsne_embds, 0, 100, 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reference:\n",
    "\n",
    "https://nlp.stanford.edu/projects/glove/"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf1.11_gpu",
   "language": "python",
   "name": "tf111_gpu"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}