3. TFIDF_features.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2> 3.6 Featurizing text data with tfidf weighted word-vectors </h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import re\n",
    "import time\n",
    "import warnings\n",
    "import numpy as np\n",
    "from nltk.corpus import stopwords\n",
    "from sklearn.preprocessing import normalize\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "import sys\n",
    "import os \n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import pickle\n",
    "\n",
    "# exctract word2vec vectors\n",
    "# https://github.com/explosion/spaCy/issues/1721\n",
    "# http://landinghub.visualstudio.com/visual-cpp-build-tools\n",
    "import spacy\n",
    "from nltk.stem import PorterStemmer\n",
    "from bs4 import BeautifulSoup\n",
    "from nltk import word_tokenize\n",
    "from scipy import sparse\n",
    "from sklearn.model_selection import train_test_split\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# avoid decoding problems\n",
    "df = pd.read_csv(\"train.csv\")\n",
    " \n",
    "# encode questions to unicode\n",
    "# https://stackoverflow.com/a/6812069\n",
    "# ----------------- python 2 ---------------------\n",
    "# df['question1'] = df['question1'].apply(lambda x: unicode(str(x),\"utf-8\"))\n",
    "# df['question2'] = df['question2'].apply(lambda x: unicode(str(x),\"utf-8\"))\n",
    "# ----------------- python 3 ---------------------\n",
    "df['question1'] = df['question1'].apply(lambda x: str(x))\n",
    "df['question2'] = df['question2'].apply(lambda x: str(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>What is the step by step guide to invest in sh...</td>\n",
       "      <td>What is the step by step guide to invest in sh...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>What is the story of Kohinoor (Koh-i-Noor) Dia...</td>\n",
       "      <td>What would happen if the Indian government sto...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>How can I increase the speed of my internet co...</td>\n",
       "      <td>How can Internet speed be increased by hacking...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>Why am I mentally very lonely? How can I solve...</td>\n",
       "      <td>Find the remainder when [math]23^{24}[/math] i...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>Which one dissolve in water quikly sugar, salt...</td>\n",
       "      <td>Which fish would survive in salt water?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  qid1  qid2                                          question1  \\\n",
       "0   0     1     2  What is the step by step guide to invest in sh...   \n",
       "1   1     3     4  What is the story of Kohinoor (Koh-i-Noor) Dia...   \n",
       "2   2     5     6  How can I increase the speed of my internet co...   \n",
       "3   3     7     8  Why am I mentally very lonely? How can I solve...   \n",
       "4   4     9    10  Which one dissolve in water quikly sugar, salt...   \n",
       "\n",
       "                                           question2  is_duplicate  \n",
       "0  What is the step by step guide to invest in sh...             0  \n",
       "1  What would happen if the Indian government sto...             0  \n",
       "2  How can Internet speed be increased by hacking...             0  \n",
       "3  Find the remainder when [math]23^{24}[/math] i...             0  \n",
       "4            Which fish would survive in salt water?             0  "
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To get the results in 4 decemal points\n",
    "SAFE_DIV = 0.0001 \n",
    "\n",
    "STOP_WORDS = stopwords.words(\"english\")\n",
    "\n",
    "\n",
    "def preprocess(x):\n",
    "    x = str(x).lower()\n",
    "    x = x.replace(\",000,000\", \"m\").replace(\",000\", \"k\").replace(\"′\", \"'\").replace(\"’\", \"'\")\\\n",
    "                           .replace(\"won't\", \"will not\").replace(\"cannot\", \"can not\").replace(\"can't\", \"can not\")\\\n",
    "                           .replace(\"n't\", \" not\").replace(\"what's\", \"what is\").replace(\"it's\", \"it is\")\\\n",
    "                           .replace(\"'ve\", \" have\").replace(\"i'm\", \"i am\").replace(\"'re\", \" are\")\\\n",
    "                           .replace(\"he's\", \"he is\").replace(\"she's\", \"she is\").replace(\"'s\", \" own\")\\\n",
    "                           .replace(\"%\", \" percent \").replace(\"₹\", \" rupee \").replace(\"$\", \" dollar \")\\\n",
    "                           .replace(\"€\", \" euro \").replace(\"'ll\", \" will\")\n",
    "    x = re.sub(r\"([0-9]+)000000\", r\"\\1m\", x)\n",
    "    x = re.sub(r\"([0-9]+)000\", r\"\\1k\", x)\n",
    "    \n",
    "    \n",
    "    porter = PorterStemmer()\n",
    "    pattern = re.compile('\\W')\n",
    "    \n",
    "    if type(x) == type(''):\n",
    "        x = re.sub(pattern, ' ', x)\n",
    "    \n",
    "    \n",
    "    if type(x) == type(''):\n",
    "        \n",
    "        example1 = BeautifulSoup(x)\n",
    "        x = example1.get_text()\n",
    "        word=word_tokenize(x)\n",
    "        lit=[]\n",
    "        for i in word:\n",
    "            lit.append(porter.stem(i))\n",
    "            u=' '.join(lit)\n",
    "            u=str(u)\n",
    "    \n",
    "    return x\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#prepro_features_train.csv (Simple Preprocessing Feartures)\n",
    "#nlp_features_train.csv (NLP Features)\n",
    "if os.path.isfile('nlp_features_train.csv'):\n",
    "    dfnlp = pd.read_csv(\"nlp_features_train.csv\",encoding='latin-1')\n",
    "else:\n",
    "    print(\"download nlp_features_train.csv from drive or run previous notebook\")\n",
    "\n",
    "if os.path.isfile('df_fe_without_preprocessing_train.csv'):\n",
    "    dfppro = pd.read_csv(\"df_fe_without_preprocessing_train.csv\",encoding='latin-1')\n",
    "else:\n",
    "    print(\"download df_fe_without_preprocessing_train.csv from drive or run previous notebook\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = dfnlp.drop(['qid1','qid2','question1','question2','is_duplicate'],axis=1)\n",
    "df2 = dfppro.drop(['qid1','qid2','question1','question2','is_duplicate'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_f=df1.merge(df2 , on='id', how= 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(404290, 27)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_f.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.merge(df_f , on='id', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(404290, 32)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df= df[0:100000]\n",
    "y_true = df['is_duplicate']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop('is_duplicate', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100000, 31)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"question1\"] = df[\"question1\"].fillna(\"\").apply(preprocess)\n",
    "df[\"question2\"] = df[\"question2\"].fillna(\"\").apply(preprocess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "que= np.dstack([df['question1'],df['question2']])\n",
    "\n",
    "ques=[]\n",
    "for i in que[0]:\n",
    "    i=''.join(i)\n",
    "    ques.append(i)\n",
    "\n",
    "questions = pd.DataFrame(ques , columns=['questions'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "questions['id']= df['id']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "df= df.merge(questions ,on='id', how ='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>cwc_min</th>\n",
       "      <th>cwc_max</th>\n",
       "      <th>csc_min</th>\n",
       "      <th>csc_max</th>\n",
       "      <th>ctc_min</th>\n",
       "      <th>...</th>\n",
       "      <th>q1len</th>\n",
       "      <th>q2len</th>\n",
       "      <th>q1_n_words</th>\n",
       "      <th>q2_n_words</th>\n",
       "      <th>word_Common</th>\n",
       "      <th>word_Total</th>\n",
       "      <th>word_share</th>\n",
       "      <th>freq_q1+q2</th>\n",
       "      <th>freq_q1-q2</th>\n",
       "      <th>questions</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>what is the step by step guide to invest in sh...</td>\n",
       "      <td>what is the step by step guide to invest in sh...</td>\n",
       "      <td>0.999980</td>\n",
       "      <td>0.833319</td>\n",
       "      <td>0.999983</td>\n",
       "      <td>0.999983</td>\n",
       "      <td>0.916659</td>\n",
       "      <td>...</td>\n",
       "      <td>66</td>\n",
       "      <td>57</td>\n",
       "      <td>14</td>\n",
       "      <td>12</td>\n",
       "      <td>10.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>0.434783</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>what is the step by step guide to invest in sh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>what is the story of kohinoor  koh i noor  dia...</td>\n",
       "      <td>what would happen if the indian government sto...</td>\n",
       "      <td>0.799984</td>\n",
       "      <td>0.399996</td>\n",
       "      <td>0.749981</td>\n",
       "      <td>0.599988</td>\n",
       "      <td>0.699993</td>\n",
       "      <td>...</td>\n",
       "      <td>51</td>\n",
       "      <td>88</td>\n",
       "      <td>8</td>\n",
       "      <td>13</td>\n",
       "      <td>4.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>what is the story of kohinoor  koh i noor  dia...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>how can i increase the speed of my internet co...</td>\n",
       "      <td>how can internet speed be increased by hacking...</td>\n",
       "      <td>0.399992</td>\n",
       "      <td>0.333328</td>\n",
       "      <td>0.399992</td>\n",
       "      <td>0.249997</td>\n",
       "      <td>0.399996</td>\n",
       "      <td>...</td>\n",
       "      <td>73</td>\n",
       "      <td>59</td>\n",
       "      <td>14</td>\n",
       "      <td>10</td>\n",
       "      <td>4.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>how can i increase the speed of my internet co...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>why am i mentally very lonely  how can i solve...</td>\n",
       "      <td>find the remainder when  math 23  24   math  i...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>50</td>\n",
       "      <td>65</td>\n",
       "      <td>11</td>\n",
       "      <td>9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>why am i mentally very lonely  how can i solve...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>which one dissolve in water quikly sugar  salt...</td>\n",
       "      <td>which fish would survive in salt water</td>\n",
       "      <td>0.399992</td>\n",
       "      <td>0.199998</td>\n",
       "      <td>0.999950</td>\n",
       "      <td>0.666644</td>\n",
       "      <td>0.571420</td>\n",
       "      <td>...</td>\n",
       "      <td>76</td>\n",
       "      <td>39</td>\n",
       "      <td>13</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>which one dissolve in water quikly sugar  salt...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 32 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  qid1  qid2                                          question1  \\\n",
       "0   0     1     2  what is the step by step guide to invest in sh...   \n",
       "1   1     3     4  what is the story of kohinoor  koh i noor  dia...   \n",
       "2   2     5     6  how can i increase the speed of my internet co...   \n",
       "3   3     7     8  why am i mentally very lonely  how can i solve...   \n",
       "4   4     9    10  which one dissolve in water quikly sugar  salt...   \n",
       "\n",
       "                                           question2   cwc_min   cwc_max  \\\n",
       "0  what is the step by step guide to invest in sh...  0.999980  0.833319   \n",
       "1  what would happen if the indian government sto...  0.799984  0.399996   \n",
       "2  how can internet speed be increased by hacking...  0.399992  0.333328   \n",
       "3  find the remainder when  math 23  24   math  i...  0.000000  0.000000   \n",
       "4            which fish would survive in salt water   0.399992  0.199998   \n",
       "\n",
       "    csc_min   csc_max   ctc_min  \\\n",
       "0  0.999983  0.999983  0.916659   \n",
       "1  0.749981  0.599988  0.699993   \n",
       "2  0.399992  0.249997  0.399996   \n",
       "3  0.000000  0.000000  0.000000   \n",
       "4  0.999950  0.666644  0.571420   \n",
       "\n",
       "                         ...                          q1len  q2len  \\\n",
       "0                        ...                             66     57   \n",
       "1                        ...                             51     88   \n",
       "2                        ...                             73     59   \n",
       "3                        ...                             50     65   \n",
       "4                        ...                             76     39   \n",
       "\n",
       "   q1_n_words  q2_n_words  word_Common  word_Total  word_share  freq_q1+q2  \\\n",
       "0          14          12         10.0        23.0    0.434783           2   \n",
       "1           8          13          4.0        20.0    0.200000           5   \n",
       "2          14          10          4.0        24.0    0.166667           2   \n",
       "3          11           9          0.0        19.0    0.000000           2   \n",
       "4          13           7          2.0        20.0    0.100000           4   \n",
       "\n",
       "   freq_q1-q2                                          questions  \n",
       "0           0  what is the step by step guide to invest in sh...  \n",
       "1           3  what is the story of kohinoor  koh i noor  dia...  \n",
       "2           0  how can i increase the speed of my internet co...  \n",
       "3           0  why am i mentally very lonely  how can i solve...  \n",
       "4           2  which one dissolve in water quikly sugar  salt...  \n",
       "\n",
       "[5 rows x 32 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train , X_test , y_train , y_test = train_test_split(df , y_true ,stratify =y_true , test_size=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train ,X_cv ,y_train ,y_cv = train_test_split(X_train , y_train , stratify=y_train ,test_size=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of data points in train data : (49000, 32)\n",
      "Number of data points in train data : (21000, 32)\n",
      "Number of data points in test data : (30000, 32)\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of data points in train data :\",X_train.shape)\n",
    "print(\"Number of data points in train data :\",X_cv.shape)\n",
    "print(\"Number of data points in test data :\",X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------- Distribution of output variable in train data ----------\n",
      "Class 0:  0.6274489795918368 Class 1:  0.3725510204081633\n",
      "---------- Distribution of output variable in test data ----------\n",
      "Class 0:  0.6274666666666666 Class 1:  0.3725333333333333\n",
      "---------- Distribution of output variable in cv data ----------\n",
      "Class 0:  0.6274761904761905 Class 1:  0.37252380952380953\n"
     ]
    }
   ],
   "source": [
    "print(\"-\"*10, \"Distribution of output variable in train data\", \"-\"*10)\n",
    "train_distr = Counter(y_train)\n",
    "train_len = len(y_train)\n",
    "print(\"Class 0: \",int(train_distr[0])/train_len,\"Class 1: \", int(train_distr[1])/train_len)\n",
    "\n",
    "print(\"-\"*10, \"Distribution of output variable in test data\", \"-\"*10)\n",
    "test_distr = Counter(y_test)\n",
    "test_len = len(y_test)\n",
    "print(\"Class 0: \",int(test_distr[0])/test_len, \"Class 1: \",int(test_distr[1])/test_len)\n",
    "\n",
    "print(\"-\"*10, \"Distribution of output variable in cv data\", \"-\"*10)\n",
    "cv_distr = Counter(y_cv)\n",
    "cv_len = len(y_cv)\n",
    "print(\"Class 0: \",int(cv_distr[0])/cv_len, \"Class 1: \",int(cv_distr[1])/cv_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The shape of train tfidf  (49000, 5000)\n",
      "The shape of cv tfidf  (21000, 5000)\n",
      "The shape of test tfidf  (30000, 5000)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "tfidf = TfidfVectorizer(lowercase=False, max_features= 5000 )\n",
    "\n",
    "train_tfidf= tfidf.fit_transform(X_train['questions'].values)\n",
    "cv_tfidf= tfidf.transform(X_cv['questions'].values)\n",
    "test_tfidf =tfidf.transform(X_test['questions'].values)\n",
    "\n",
    "tfidf_feats=tfidf.get_feature_names()\n",
    "\n",
    "print(\"The shape of train tfidf \",train_tfidf.shape)\n",
    "print(\"The shape of cv tfidf \",cv_tfidf.shape)\n",
    "print(\"The shape of test tfidf \",test_tfidf.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = X_train.drop(['id','qid1','qid2','question1','question2','questions'],axis=1)\n",
    "X_cv = X_cv.drop(['id','qid1','qid2','question1','question2','questions'],axis=1)\n",
    "X_test = X_test.drop(['id','qid1','qid2','question1','question2','questions'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "for colu in X_train:\n",
    "    column=np.array(X_train[colu][0:train_tfidf.shape[0]])\n",
    "    train_tfidf=sparse.hstack((train_tfidf,column[:,None]))\n",
    "    \n",
    "for colu in X_cv:\n",
    "    column=np.array(X_cv[colu][0:cv_tfidf.shape[0]])\n",
    "    cv_tfidf=sparse.hstack((cv_tfidf,column[:,None]))\n",
    "    \n",
    "for colu in X_test:\n",
    "    column=np.array(X_test[colu][0:test_tfidf.shape[0]])\n",
    "    test_tfidf=sparse.hstack((test_tfidf,column[:,None]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The shape of final train data is  (49000, 5026)\n",
      "The shape of final cv data is  (21000, 5026)\n",
      "The shape of final test data is  (30000, 5026)\n"
     ]
    }
   ],
   "source": [
    "print(\"The shape of final train data is \",train_tfidf.shape)\n",
    "print(\"The shape of final cv data is \",cv_tfidf.shape)\n",
    "print(\"The shape of final test data is \",test_tfidf.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "sparse.save_npz('train_tfidf',train_tfidf)               #storing the sparse matrix    \n",
    "sparse.save_npz('cv_tfidf',cv_tfidf) \n",
    "sparse.save_npz('test_tfidf',test_tfidf) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train.to_pickle('y_train')                                       #storing the true labels\n",
    "y_cv.to_pickle('y_cv') \n",
    "y_test.to_pickle('y_test') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}