LSTM Network.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using LSTM Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "pos_train_data = pd.read_csv('train_pos.tsv',sep = '\\t')\n",
    "neg_train_data = pd.read_csv('train_neg.tsv',sep = '\\t')\n",
    "pos_test_data = pd.read_csv('test_pos.tsv',sep = '\\t')\n",
    "neg_test_data = pd.read_csv('test_neg.tsv',sep = '\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pos_train_data = pos_train_data[['Text','Sentiment']]\n",
    "neg_train_data = neg_train_data[['Text','Sentiment']]\n",
    "pos_test_data = pos_test_data[['Text','Sentiment']]\n",
    "neg_test_data = neg_test_data[['Text','Sentiment']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>I just wanna say that amongst all the so-calle...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A friend once told me that an art-house indepe...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>I saw this movie a couple years back. I could'...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>But this movie was a bore. The history part wa...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>This guy has no idea of cinema. Okay, it seems...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Text  Sentiment\n",
       "0  I just wanna say that amongst all the so-calle...          1\n",
       "1  A friend once told me that an art-house indepe...          0\n",
       "2  I saw this movie a couple years back. I could'...          0\n",
       "3  But this movie was a bore. The history part wa...          0\n",
       "4  This guy has no idea of cinema. Okay, it seems...          0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_train = pd.concat([pos_train_data,neg_train_data],ignore_index = True)\n",
    "data_train = data_train.sample(frac=1).reset_index(drop=True)\n",
    "data_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25000"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>This isn't a dreadful film, merely insipid. Th...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>One of master director Alfred Hitchcock's fine...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The original Female Convict Scorpion is an all...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Out of boredom and vast curiosity, I decided t...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>the real plot...&lt;br /&gt;&lt;br /&gt;A group of post-Ci...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Text  Sentiment\n",
       "0  This isn't a dreadful film, merely insipid. Th...          0\n",
       "1  One of master director Alfred Hitchcock's fine...          1\n",
       "2  The original Female Convict Scorpion is an all...          1\n",
       "3  Out of boredom and vast curiosity, I decided t...          0\n",
       "4  the real plot...<br /><br />A group of post-Ci...          0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_test = pd.concat([pos_test_data,neg_test_data],ignore_index = True)\n",
    "data_test = data_test.sample(frac=1).reset_index(drop=True)\n",
    "data_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from string import punctuation\n",
    "\n",
    "stop_words = set(stopwords.words('english'))\n",
    "table = str.maketrans('', '', punctuation)\n",
    "\n",
    "def textclean(text):\n",
    "    tokens = word_tokenize(text)\n",
    "    tokens = [word for word in tokens if word.isalpha()]\n",
    "    tokens = [w.translate(table) for w in tokens]\n",
    "    tokens = [word for word in tokens if not word in stop_words]\n",
    "    tokens = [word for word in tokens if len(word) > 1]\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['What', \"'s\", 'the', 'matter', '?']"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_tokenize(\"What's the matter?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['wan',\n",
       " 'na',\n",
       " 'say',\n",
       " 'amongst',\n",
       " 'classic',\n",
       " 'hiphop',\n",
       " 'films',\n",
       " 'ive',\n",
       " 'seen',\n",
       " 'like',\n",
       " 'wild',\n",
       " 'style',\n",
       " 'krushgroove',\n",
       " 'breakin',\n",
       " 'style',\n",
       " 'wars',\n",
       " 'etc',\n",
       " 'imo',\n",
       " 'beat',\n",
       " 'street',\n",
       " 'best',\n",
       " 'amongst',\n",
       " 'others',\n",
       " 'whenever',\n",
       " 'ask',\n",
       " 'people',\n",
       " 'fave',\n",
       " 'seems',\n",
       " 'beat',\n",
       " 'street',\n",
       " 'pops',\n",
       " 'still',\n",
       " 'lowest',\n",
       " 'ranked',\n",
       " 'punch',\n",
       " 'belt',\n",
       " 'say',\n",
       " 'points',\n",
       " 'belt',\n",
       " 'love',\n",
       " 'music',\n",
       " 'performances',\n",
       " 'breakdancing',\n",
       " 'makes',\n",
       " 'wan',\n",
       " 'na',\n",
       " 'spin',\n",
       " 'ramo',\n",
       " 'makes',\n",
       " 'wan',\n",
       " 'na',\n",
       " 'throw',\n",
       " 'piece',\n",
       " 'classic']"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviews = []\n",
    "\n",
    "for index,row in data_train.iterrows():\n",
    "    text = (row['Text'].lower())    \n",
    "    reviews.append(textclean(text))\n",
    "reviews[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'na'"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import itertools\n",
    "linked_reviews = list(itertools.chain.from_iterable(reviews))\n",
    "\n",
    "vocab_freq = dict()\n",
    "\n",
    "linked_reviews[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for word in linked_reviews:\n",
    "    if word not in vocab_freq:\n",
    "        vocab_freq[word] = 1\n",
    "    else:\n",
    "        vocab_freq[word] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'sites': 38,\n",
       " 'drea': 2,\n",
       " 'neelix': 3,\n",
       " 'psychobilly': 1,\n",
       " 'sincere': 85,\n",
       " 'retentive': 2,\n",
       " 'siodmak': 18,\n",
       " 'concerning': 115,\n",
       " 'maître': 1,\n",
       " 'heller': 4,\n",
       " 'buttercream': 1,\n",
       " 'friel': 12,\n",
       " 'downgrades': 2,\n",
       " 'ehh': 3,\n",
       " 'oriental': 32,\n",
       " 'emery': 6,\n",
       " 'unreviewed': 1,\n",
       " 'afficionados': 2,\n",
       " 'latrine': 3,\n",
       " 'reins': 7,\n",
       " 'brahms': 1,\n",
       " 'touchings': 1,\n",
       " 'moffat': 4,\n",
       " 'anesthesia': 8,\n",
       " 'bombshell': 10,\n",
       " 'interesting': 3062,\n",
       " 'getz': 4,\n",
       " 'symmetric': 1,\n",
       " 'tykes': 3,\n",
       " 'favorite': 1221,\n",
       " 'mulcahy': 4,\n",
       " 'kravitz': 1,\n",
       " 'helfer': 1,\n",
       " 'midriff': 3,\n",
       " 'fumiya': 2,\n",
       " 'tenths': 1,\n",
       " 'perdu': 2,\n",
       " 'anderson': 218,\n",
       " 'uninhibited': 14,\n",
       " 'missions': 36,\n",
       " 'polnareff': 1,\n",
       " 'impulses': 14,\n",
       " 'cindi': 1,\n",
       " 'sensitivities': 1,\n",
       " 'campfire': 22,\n",
       " 'kanpur': 1,\n",
       " 'anamorphic': 14,\n",
       " 'bodily': 17,\n",
       " 'fooled': 92,\n",
       " 'leeli': 1,\n",
       " 'expresso': 1,\n",
       " 'deadhead': 1,\n",
       " 'tourist': 55,\n",
       " 'premium': 9,\n",
       " 'dreads': 4,\n",
       " 'immobilize': 1,\n",
       " 'beecham': 1,\n",
       " 'weighing': 4,\n",
       " 'assuage': 1,\n",
       " 'damningly': 1,\n",
       " 'chile': 21,\n",
       " 'silvio': 16,\n",
       " 'poet': 35,\n",
       " 'sugiyama': 12,\n",
       " 'illiterate': 34,\n",
       " 'sanctimoniousness': 1,\n",
       " 'darwell': 2,\n",
       " 'beseech': 1,\n",
       " 'gianfranco': 6,\n",
       " 'buoyant': 6,\n",
       " 'persepctive': 1,\n",
       " 'splatters': 3,\n",
       " 'shopped': 1,\n",
       " 'millan': 1,\n",
       " 'chanteuse': 3,\n",
       " 'macissac': 1,\n",
       " 'fantine': 5,\n",
       " 'affections': 32,\n",
       " 'sunnys': 1,\n",
       " 'disarms': 3,\n",
       " 'cinderella': 229,\n",
       " 'forgivable': 32,\n",
       " 'verbatim': 11,\n",
       " 'weitz': 6,\n",
       " 'hydra': 2,\n",
       " 'stefanson': 2,\n",
       " 'adriana': 5,\n",
       " 'elfriede': 1,\n",
       " 'playboys': 2,\n",
       " 'hearth': 4,\n",
       " 'jeanette': 24,\n",
       " 'grapefruit': 3,\n",
       " 'obtains': 6,\n",
       " 'sullies': 2,\n",
       " 'oops': 21,\n",
       " 'sansabelt': 1,\n",
       " 'athenly': 1,\n",
       " 'bikram': 1,\n",
       " 'uncomprehended': 1,\n",
       " 'nakamura': 3,\n",
       " 'prosy': 1,\n",
       " 'fairchild': 8,\n",
       " 'opposes': 4,\n",
       " 'horrendous': 132,\n",
       " 'washrooms': 1,\n",
       " 'figuratively': 16,\n",
       " 'tutti': 2,\n",
       " 'amatuer': 3,\n",
       " 'dumitru': 2,\n",
       " 'rpg': 16,\n",
       " 'oo': 1,\n",
       " 'espy': 1,\n",
       " 'danced': 29,\n",
       " 'therapy': 46,\n",
       " 'quickliy': 1,\n",
       " 'scotched': 1,\n",
       " 'lends': 52,\n",
       " 'communities': 27,\n",
       " 'chomps': 2,\n",
       " 'anarchist': 3,\n",
       " 'rutkay': 3,\n",
       " 'langdon': 3,\n",
       " 'disregarded': 9,\n",
       " 'philosophy': 99,\n",
       " 'gek': 2,\n",
       " 'hogs': 3,\n",
       " 'dangly': 3,\n",
       " 'allergies': 1,\n",
       " 'effectually': 1,\n",
       " 'turner': 120,\n",
       " 'ange': 1,\n",
       " 'unawareness': 1,\n",
       " 'jupiter': 6,\n",
       " 'careers': 101,\n",
       " 'convents': 1,\n",
       " 'landau': 17,\n",
       " 'partook': 4,\n",
       " 'britney': 41,\n",
       " 'chang': 30,\n",
       " 'kamal': 33,\n",
       " 'preferences': 6,\n",
       " 'edifying': 2,\n",
       " 'aquaman': 6,\n",
       " 'jerking': 7,\n",
       " 'logician': 1,\n",
       " 'sportsmen': 1,\n",
       " 'hoyos': 3,\n",
       " 'extracurricular': 2,\n",
       " 'debunk': 6,\n",
       " 'steryotypes': 1,\n",
       " 'safeguarding': 3,\n",
       " 'isd': 1,\n",
       " 'blige': 1,\n",
       " 'patchett': 1,\n",
       " 'hollywoodian': 3,\n",
       " 'wharton': 1,\n",
       " 'leroux': 2,\n",
       " 'notably': 119,\n",
       " 'dacascos': 2,\n",
       " 'imploring': 1,\n",
       " 'pecos': 3,\n",
       " 'sooooooooo': 1,\n",
       " 'jada': 18,\n",
       " 'clément': 3,\n",
       " 'fleet': 49,\n",
       " 'newcomers': 23,\n",
       " 'huntsville': 2,\n",
       " 'closeness': 12,\n",
       " 'silent': 427,\n",
       " 'unsubstantial': 2,\n",
       " 'dipper': 1,\n",
       " 'boer': 1,\n",
       " 'chaotic': 45,\n",
       " 'rebel': 101,\n",
       " 'feces': 10,\n",
       " 'fearlessly': 3,\n",
       " 'promisingly': 13,\n",
       " 'yodelling': 1,\n",
       " 'argonne': 4,\n",
       " 'collides': 7,\n",
       " 'kairo': 2,\n",
       " 'yasoumi': 1,\n",
       " 'fugly': 1,\n",
       " 'flying': 344,\n",
       " 'raintree': 1,\n",
       " 'lakeside': 4,\n",
       " 'brujas': 1,\n",
       " 'eppes': 5,\n",
       " 'hurls': 7,\n",
       " 'global': 91,\n",
       " 'strides': 12,\n",
       " 'misfortune': 59,\n",
       " 'slaughtering': 14,\n",
       " 'unclouded': 1,\n",
       " 'discouraged': 9,\n",
       " 'prohibit': 1,\n",
       " 'unidentifiable': 1,\n",
       " 'unmemorable': 12,\n",
       " 'franic': 1,\n",
       " 'padme': 2,\n",
       " 'genii': 3,\n",
       " 'translate': 50,\n",
       " 'center': 222,\n",
       " 'mindf': 1,\n",
       " 'worshiped': 3,\n",
       " 'incited': 1,\n",
       " 'binded': 1,\n",
       " 'intersection': 6,\n",
       " 'bolting': 1,\n",
       " 'obscuring': 3,\n",
       " 'meurent': 1,\n",
       " 'conjuring': 7,\n",
       " 'bamatabois': 1,\n",
       " 'ahahhahahaha': 1,\n",
       " 'powerglove': 3,\n",
       " 'deana': 4,\n",
       " 'mercury': 24,\n",
       " 'junior': 88,\n",
       " 'grouch': 7,\n",
       " 'avni': 1,\n",
       " 'alda': 10,\n",
       " 'uhhhh': 1,\n",
       " 'braids': 2,\n",
       " 'iu': 1,\n",
       " 'unhand': 1,\n",
       " 'desperados': 1,\n",
       " 'gojira': 5,\n",
       " 'spiritualized': 1,\n",
       " 'prunella': 3,\n",
       " 'style': 1424,\n",
       " 'headfirst': 2,\n",
       " 'journeying': 1,\n",
       " 'nyland': 1,\n",
       " 'carson': 34,\n",
       " 'laud': 1,\n",
       " 'perused': 2,\n",
       " 'polanski': 104,\n",
       " 'pawing': 3,\n",
       " 'sullavan': 29,\n",
       " 'underclass': 5,\n",
       " 'saleswoman': 2,\n",
       " 'doiiing': 1,\n",
       " 'energy': 307,\n",
       " 'sexagenarians': 1,\n",
       " 'unsavory': 19,\n",
       " 'hostile': 44,\n",
       " 'callum': 1,\n",
       " 'tanna': 1,\n",
       " 'euthanized': 2,\n",
       " 'humbled': 1,\n",
       " 'criticzed': 1,\n",
       " 'spokesman': 5,\n",
       " 'publication': 9,\n",
       " 'doubles': 20,\n",
       " 'sacchi': 2,\n",
       " 'krusty': 1,\n",
       " 'nasal': 11,\n",
       " 'kucch': 1,\n",
       " 'sweepstakes': 2,\n",
       " 'deathstalker': 34,\n",
       " 'bent': 63,\n",
       " 'yielded': 3,\n",
       " 'marcella': 7,\n",
       " 'ainsworth': 1,\n",
       " 'salvages': 4,\n",
       " 'jawbreaker': 2,\n",
       " 'woolrich': 7,\n",
       " 'cya': 3,\n",
       " 'patches': 11,\n",
       " 'pickpockets': 4,\n",
       " 'gin': 7,\n",
       " 'draub': 1,\n",
       " 'atamana': 1,\n",
       " 'morose': 17,\n",
       " 'detectable': 3,\n",
       " 'proposed': 14,\n",
       " 'sith': 14,\n",
       " 'blacklist': 3,\n",
       " 'kerwin': 1,\n",
       " 'topkapi': 4,\n",
       " 'archs': 1,\n",
       " 'revolving': 48,\n",
       " 'knauf': 1,\n",
       " 'reconsiders': 1,\n",
       " 'servicemen': 8,\n",
       " 'asylums': 2,\n",
       " 'obviosly': 1,\n",
       " 'precept': 1,\n",
       " 'wentworth': 19,\n",
       " 'torment': 41,\n",
       " 'caseman': 2,\n",
       " 'whacks': 4,\n",
       " 'mappo': 1,\n",
       " 'legalities': 1,\n",
       " 'exercising': 3,\n",
       " 'locationed': 1,\n",
       " 'flamboyance': 6,\n",
       " 'ephron': 4,\n",
       " 'graphic': 235,\n",
       " 'wacked': 3,\n",
       " 'bleachers': 1,\n",
       " 'willingly': 35,\n",
       " 'centrality': 1,\n",
       " 'hustles': 4,\n",
       " 'participation': 31,\n",
       " 'larner': 3,\n",
       " 'tyro': 1,\n",
       " 'california': 181,\n",
       " 'jurgen': 5,\n",
       " 'wurman': 2,\n",
       " 'headtripping': 1,\n",
       " 'inspire': 62,\n",
       " 'zaitung': 1,\n",
       " 'sand': 55,\n",
       " 'zebbedy': 1,\n",
       " 'bunged': 1,\n",
       " 'cohesively': 2,\n",
       " 'encompass': 5,\n",
       " 'dabrova': 1,\n",
       " 'meru': 1,\n",
       " 'bludhorn': 1,\n",
       " 'chewbaka': 1,\n",
       " 'rekha': 4,\n",
       " 'motorized': 6,\n",
       " 'juke': 4,\n",
       " 'buggy': 12,\n",
       " 'airspace': 2,\n",
       " 'dun': 8,\n",
       " 'transitory': 4,\n",
       " 'evacuated': 16,\n",
       " 'vilest': 3,\n",
       " 'cheezie': 1,\n",
       " 'provoke': 26,\n",
       " 'hennessy': 8,\n",
       " 'lube': 6,\n",
       " 'driller': 3,\n",
       " 'uncinematic': 2,\n",
       " 'shines': 146,\n",
       " 'trekkies': 4,\n",
       " 'dreamy': 48,\n",
       " 'installing': 1,\n",
       " 'oik': 1,\n",
       " 'wireless': 5,\n",
       " 'animators': 36,\n",
       " 'willi': 2,\n",
       " 'surreptitiously': 1,\n",
       " 'byproduct': 6,\n",
       " 'startling': 61,\n",
       " 'bonestell': 1,\n",
       " 'wardo': 1,\n",
       " 'swigged': 1,\n",
       " 'ishtar': 13,\n",
       " 'lustig': 10,\n",
       " 'turds': 8,\n",
       " 'shoved': 39,\n",
       " 'rocketry': 4,\n",
       " 'parlaying': 1,\n",
       " 'sequal': 6,\n",
       " 'lazers': 1,\n",
       " 'sparring': 17,\n",
       " 'hatsumo': 2,\n",
       " 'phalke': 1,\n",
       " 'calder': 1,\n",
       " 'tedesco': 1,\n",
       " 'cowl': 4,\n",
       " 'jip': 1,\n",
       " 'scrip': 3,\n",
       " 'bruckner': 3,\n",
       " 'procedurals': 1,\n",
       " 'bensen': 1,\n",
       " 'deitrich': 1,\n",
       " 'argyle': 2,\n",
       " 'networked': 1,\n",
       " 'piling': 10,\n",
       " 'brennecke': 1,\n",
       " 'monitored': 6,\n",
       " 'qua': 1,\n",
       " 'yuma': 19,\n",
       " 'jenny': 92,\n",
       " 'handedly': 7,\n",
       " 'faring': 2,\n",
       " 'mindgaming': 1,\n",
       " 'pessimism': 5,\n",
       " 'invalids': 2,\n",
       " 'impounding': 1,\n",
       " 'dennings': 1,\n",
       " 'martin': 361,\n",
       " 'petwee': 1,\n",
       " 'woosh': 1,\n",
       " 'anu': 6,\n",
       " 'humanist': 8,\n",
       " 'usable': 2,\n",
       " 'replaying': 12,\n",
       " 'hyodo': 1,\n",
       " 'fidani': 6,\n",
       " 'using': 792,\n",
       " 'geddis': 1,\n",
       " 'evilest': 1,\n",
       " 'stolid': 9,\n",
       " 'barwood': 1,\n",
       " 'perabo': 9,\n",
       " 'för': 1,\n",
       " 'unplayable': 2,\n",
       " 'freer': 1,\n",
       " 'infighting': 5,\n",
       " 'zomezing': 1,\n",
       " 'oooooh': 1,\n",
       " 'gyllenhaal': 28,\n",
       " 'harmful': 11,\n",
       " 'lcc': 1,\n",
       " 'mercurial': 6,\n",
       " 'helpless': 63,\n",
       " 'feeds': 22,\n",
       " 'deification': 2,\n",
       " 'pepe': 4,\n",
       " 'dennison': 1,\n",
       " 'abruptness': 2,\n",
       " 'mendel': 5,\n",
       " 'bhangra': 1,\n",
       " 'imbd': 2,\n",
       " 'rationalized': 1,\n",
       " 'excelled': 6,\n",
       " 'impervious': 6,\n",
       " 'thanatos': 1,\n",
       " 'heaves': 1,\n",
       " 'slowenian': 1,\n",
       " 'soundscape': 1,\n",
       " 'comcast': 1,\n",
       " 'helpful': 62,\n",
       " 'yale': 5,\n",
       " 'etude': 1,\n",
       " 'argentine': 15,\n",
       " 'rossilini': 1,\n",
       " 'teenkill': 1,\n",
       " 'id': 56,\n",
       " 'download': 29,\n",
       " 'satisfactory': 36,\n",
       " 'gosselaar': 2,\n",
       " 'hopalong': 16,\n",
       " 'ossessione': 39,\n",
       " 'hardwicke': 20,\n",
       " 'mestressat': 1,\n",
       " 'doctrine': 10,\n",
       " 'hesitating': 1,\n",
       " 'chauvinist': 5,\n",
       " 'overeating': 2,\n",
       " 'evenhandedness': 1,\n",
       " 'orchids': 5,\n",
       " 'khakee': 4,\n",
       " 'muddy': 28,\n",
       " 'derrick': 5,\n",
       " 'reverb': 1,\n",
       " 'deploy': 3,\n",
       " 'carnelutti': 1,\n",
       " 'huddling': 2,\n",
       " 'bangers': 3,\n",
       " 'gauges': 1,\n",
       " 'boddhisatva': 1,\n",
       " 'pimple': 1,\n",
       " 'incomprehendably': 1,\n",
       " 'dreary': 87,\n",
       " 'pecs': 2,\n",
       " 'kidnaps': 38,\n",
       " 'berkhoff': 2,\n",
       " 'slimiest': 2,\n",
       " 'diagonally': 1,\n",
       " 'bergstrom': 1,\n",
       " 'screenings': 17,\n",
       " 'meatier': 1,\n",
       " 'rather': 2728,\n",
       " 'photogenic': 12,\n",
       " 'hairband': 1,\n",
       " 'nonsensical': 80,\n",
       " 'swarming': 10,\n",
       " 'mmb': 1,\n",
       " 'broody': 3,\n",
       " 'fires': 43,\n",
       " 'takita': 1,\n",
       " 'especially': 2521,\n",
       " 'traveler': 10,\n",
       " 'overproduced': 3,\n",
       " 'indians': 114,\n",
       " 'rolffes': 1,\n",
       " 'jannick': 2,\n",
       " 'tracked': 15,\n",
       " 'hickey': 6,\n",
       " 'werecat': 1,\n",
       " 'rexs': 1,\n",
       " 'fixates': 2,\n",
       " 'goop': 5,\n",
       " 'sway': 21,\n",
       " 'rockford': 5,\n",
       " 'optimum': 2,\n",
       " 'bakesfield': 1,\n",
       " 'currents': 8,\n",
       " 'marmite': 3,\n",
       " 'almeria': 2,\n",
       " 'congeniality': 5,\n",
       " 'committed': 191,\n",
       " 'watt': 1,\n",
       " 'zefferelli': 6,\n",
       " 'nivens': 2,\n",
       " 'intermediary': 1,\n",
       " 'extender': 1,\n",
       " 'metropolis': 34,\n",
       " 'lonliness': 1,\n",
       " 'misportrayed': 1,\n",
       " 'fall': 756,\n",
       " 'amused': 76,\n",
       " 'sellers': 97,\n",
       " 'amorós': 2,\n",
       " 'tuttle': 1,\n",
       " 'sexual': 690,\n",
       " 'exist': 282,\n",
       " 'exhuberance': 1,\n",
       " 'pointeblank': 1,\n",
       " 'jacob': 26,\n",
       " 'todean': 1,\n",
       " 'unredeeming': 1,\n",
       " 'swd': 1,\n",
       " 'larva': 1,\n",
       " 'diatribe': 9,\n",
       " 'aids': 87,\n",
       " 'wrightly': 1,\n",
       " 'willett': 3,\n",
       " 'preformance': 1,\n",
       " 'carolinas': 1,\n",
       " 'bureacracy': 2,\n",
       " 'hap': 6,\n",
       " 'orbs': 2,\n",
       " 'overconfident': 4,\n",
       " 'rockumentaries': 1,\n",
       " 'summing': 18,\n",
       " 'backup': 13,\n",
       " 'eod': 11,\n",
       " 'explicitly': 24,\n",
       " 'cosby': 14,\n",
       " 'stis': 2,\n",
       " 'capitalised': 2,\n",
       " 'pasa': 1,\n",
       " 'clyve': 1,\n",
       " 'disapprove': 3,\n",
       " 'walkees': 1,\n",
       " 'dratch': 8,\n",
       " 'aced': 1,\n",
       " 'creators': 121,\n",
       " 'japanse': 1,\n",
       " 'skewering': 3,\n",
       " 'frighting': 1,\n",
       " 'reasserted': 1,\n",
       " 'differentiate': 12,\n",
       " 'implausibilities': 7,\n",
       " 'imaging': 7,\n",
       " 'soze': 3,\n",
       " 'slayride': 1,\n",
       " 'waterlogged': 2,\n",
       " 'bronston': 3,\n",
       " 'saner': 1,\n",
       " 'undependable': 1,\n",
       " 'shittier': 1,\n",
       " 'taraporevala': 1,\n",
       " 'suburb': 17,\n",
       " 'inadmissible': 1,\n",
       " 'carefully': 128,\n",
       " 'mohave': 1,\n",
       " 'runaways': 5,\n",
       " 'woolworths': 2,\n",
       " 'manuscripts': 3,\n",
       " 'armand': 16,\n",
       " 'engrossment': 1,\n",
       " 'wayans': 49,\n",
       " 'wale': 2,\n",
       " 'adjutant': 1,\n",
       " 'plutocrats': 1,\n",
       " 'stoumen': 1,\n",
       " 'unsightly': 3,\n",
       " 'pr': 16,\n",
       " 'comprising': 7,\n",
       " 'peeble': 1,\n",
       " 'surprises': 198,\n",
       " 'hosts': 29,\n",
       " 'netting': 1,\n",
       " 'syria': 3,\n",
       " 'taekwondo': 6,\n",
       " 'commodore': 8,\n",
       " 'muncey': 1,\n",
       " 'lulu': 24,\n",
       " 'fn': 3,\n",
       " 'sinatra': 236,\n",
       " 'buffaloes': 6,\n",
       " 'tashlin': 3,\n",
       " 'emigration': 2,\n",
       " 'gael': 7,\n",
       " 'krs': 1,\n",
       " 'murderously': 1,\n",
       " 'instills': 5,\n",
       " 'paley': 3,\n",
       " 'oafish': 1,\n",
       " 'ahhhhhh': 2,\n",
       " 'scouted': 3,\n",
       " 'pseudocomedies': 1,\n",
       " 'asbury': 1,\n",
       " 'upbringing': 36,\n",
       " 'solemnity': 4,\n",
       " 'heroo': 1,\n",
       " 'galaxies': 2,\n",
       " 'misfortunes': 5,\n",
       " 'hokier': 1,\n",
       " 'doozys': 1,\n",
       " 'cellulose': 1,\n",
       " 'magobei': 2,\n",
       " 'bane': 16,\n",
       " 'hairstyles': 19,\n",
       " 'sijan': 2,\n",
       " 'hucksters': 2,\n",
       " 'herve': 2,\n",
       " 'madhura': 1,\n",
       " 'droves': 8,\n",
       " 'disadvantageous': 1,\n",
       " 'squire': 19,\n",
       " 'fdny': 4,\n",
       " 'electrocuted': 17,\n",
       " 'sawed': 4,\n",
       " 'pittsburg': 1,\n",
       " 'sadomania': 1,\n",
       " 'kass': 1,\n",
       " 'lightner': 2,\n",
       " 'backseat': 17,\n",
       " 'punctuation': 3,\n",
       " 'weeped': 1,\n",
       " 'harland': 1,\n",
       " 'tarrytown': 1,\n",
       " 'whalers': 2,\n",
       " 'thanatopsis': 1,\n",
       " 'condemnation': 10,\n",
       " 'culp': 27,\n",
       " 'needful': 1,\n",
       " 'fossils': 1,\n",
       " 'lavender': 9,\n",
       " 'beauté': 3,\n",
       " 'relief': 234,\n",
       " 'ozu': 16,\n",
       " 'unrest': 13,\n",
       " 'snicker': 12,\n",
       " 'generalization': 4,\n",
       " 'irritant': 3,\n",
       " 'tura': 1,\n",
       " 'planning': 124,\n",
       " 'unforgivably': 7,\n",
       " 'genji': 3,\n",
       " 'gambas': 1,\n",
       " 'hier': 1,\n",
       " 'mortgan': 1,\n",
       " 'eric': 265,\n",
       " 'clementine': 8,\n",
       " 'deficiencies': 9,\n",
       " 'consigned': 6,\n",
       " 'polution': 1,\n",
       " 'gitan': 1,\n",
       " 'numberless': 1,\n",
       " 'simpathetic': 2,\n",
       " 'statute': 4,\n",
       " 'trippy': 20,\n",
       " 'posers': 3,\n",
       " 'procrastinator': 2,\n",
       " 'pots': 2,\n",
       " 'helluva': 10,\n",
       " 'pathology': 5,\n",
       " 'illona': 2,\n",
       " 'commonwealth': 2,\n",
       " 'drivas': 9,\n",
       " 'hein': 1,\n",
       " 'saskatchewan': 5,\n",
       " 'maltex': 1,\n",
       " 'knocks': 55,\n",
       " 'preexisting': 1,\n",
       " 'sampled': 3,\n",
       " 'corsair': 1,\n",
       " 'gainsborough': 1,\n",
       " 'nietzschean': 4,\n",
       " 'sid': 82,\n",
       " 'jaliyl': 1,\n",
       " 'supertank': 2,\n",
       " 'hip': 129,\n",
       " 'lobs': 1,\n",
       " 'infatuations': 1,\n",
       " 'babaganoosh': 1,\n",
       " 'pipsqueak': 2,\n",
       " 'northeastern': 1,\n",
       " 'judges': 38,\n",
       " 'horiible': 1,\n",
       " 'valga': 1,\n",
       " 'suzuka': 1,\n",
       " 'putain': 14,\n",
       " 'embezzlement': 5,\n",
       " 'loc': 11,\n",
       " 'macedonian': 3,\n",
       " 'denial': 36,\n",
       " 'regis': 4,\n",
       " 'mew': 1,\n",
       " 'intersting': 1,\n",
       " 'schwarzeneggar': 3,\n",
       " 'thoughtlessness': 4,\n",
       " 'frazer': 3,\n",
       " 'mj': 35,\n",
       " 'blacksnake': 10,\n",
       " 'langford': 3,\n",
       " 'excon': 1,\n",
       " 'boles': 1,\n",
       " 'apperciate': 1,\n",
       " 'admonition': 1,\n",
       " 'blank': 114,\n",
       " 'conor': 1,\n",
       " 'awfull': 1,\n",
       " 'entities': 4,\n",
       " 'rosanna': 26,\n",
       " 'gendered': 1,\n",
       " 'wacky': 83,\n",
       " 'tightening': 9,\n",
       " 'impressively': 30,\n",
       " 'sketched': 5,\n",
       " 'envoled': 1,\n",
       " 'linoleum': 3,\n",
       " 'céline': 3,\n",
       " 'defininitive': 1,\n",
       " 'blane': 2,\n",
       " 'attractions': 16,\n",
       " 'eleanora': 2,\n",
       " 'cain': 62,\n",
       " 'charachter': 2,\n",
       " 'latrina': 2,\n",
       " 'quench': 4,\n",
       " 'subsist': 1,\n",
       " 'pullitzer': 2,\n",
       " 'kidman': 72,\n",
       " 'sleeker': 3,\n",
       " 'albino': 6,\n",
       " 'jessica': 169,\n",
       " 'aislinn': 1,\n",
       " 'volcano': 13,\n",
       " 'misconceptions': 7,\n",
       " 'gispsy': 1,\n",
       " 'grandmasters': 1,\n",
       " 'diwali': 1,\n",
       " 'speeds': 17,\n",
       " 'colbet': 1,\n",
       " 'accelerant': 1,\n",
       " 'chaplins': 2,\n",
       " 'jawbones': 1,\n",
       " 'emotionalism': 4,\n",
       " 'spirogolou': 1,\n",
       " 'clownhouse': 1,\n",
       " 'dollops': 3,\n",
       " 'snapper': 1,\n",
       " 'ends': 954,\n",
       " 'deux': 11,\n",
       " 'doqui': 1,\n",
       " 'hatta': 2,\n",
       " 'sudow': 1,\n",
       " 'montreux': 1,\n",
       " 'dracht': 2,\n",
       " 'trending': 1,\n",
       " 'opened': 149,\n",
       " 'well': 9445,\n",
       " 'sidenote': 4,\n",
       " 'litteraly': 1,\n",
       " 'lumbly': 4,\n",
       " 'captures': 216,\n",
       " 'fundamentally': 19,\n",
       " 'cridits': 1,\n",
       " 'caroll': 1,\n",
       " 'mackichan': 1,\n",
       " 'jams': 5,\n",
       " 'bhagat': 1,\n",
       " 'robbed': 41,\n",
       " 'landmines': 1,\n",
       " 'entrapping': 1,\n",
       " 'scribbled': 2,\n",
       " 'reunuin': 1,\n",
       " 'defer': 1,\n",
       " 'feint': 1,\n",
       " 'wackier': 2,\n",
       " 'dereliction': 1,\n",
       " 'contemporary': 205,\n",
       " 'celie': 31,\n",
       " 'rte': 2,\n",
       " 'shabana': 4,\n",
       " 'petites': 1,\n",
       " 'traumatisingly': 1,\n",
       " 'waxing': 7,\n",
       " 'jacquin': 1,\n",
       " 'reform': 22,\n",
       " 'walid': 1,\n",
       " 'feats': 11,\n",
       " 'discriminating': 6,\n",
       " 'cutitta': 1,\n",
       " 'paths': 48,\n",
       " 'yell': 44,\n",
       " 'taller': 13,\n",
       " 'offshore': 2,\n",
       " 'laudably': 1,\n",
       " 'sorrier': 1,\n",
       " 'daines': 3,\n",
       " 'bleached': 13,\n",
       " 'lattanzi': 1,\n",
       " 'meier': 1,\n",
       " 'mahoganoy': 1,\n",
       " 'herzog': 17,\n",
       " 'roared': 4,\n",
       " 'caterina': 1,\n",
       " 'balikbayan': 1,\n",
       " 'acteurs': 1,\n",
       " 'juxtaposing': 9,\n",
       " 'porfirio': 1,\n",
       " 'veering': 3,\n",
       " 'cheerleaders': 9,\n",
       " 'immigrating': 1,\n",
       " 'centurians': 1,\n",
       " 'goldstone': 1,\n",
       " 'youji': 1,\n",
       " 'mtv': 76,\n",
       " 'loughlin': 2,\n",
       " 'musta': 3,\n",
       " 'eggs': 30,\n",
       " 'overrate': 1,\n",
       " 'thlema': 1,\n",
       " 'whodunnits': 2,\n",
       " 'elainor': 1,\n",
       " 'expounded': 2,\n",
       " 'ribs': 15,\n",
       " 'minglun': 1,\n",
       " 'wheedon': 1,\n",
       " 'draught': 1,\n",
       " 'tree': 160,\n",
       " 'cutiest': 1,\n",
       " 'veal': 1,\n",
       " 'bruce': 381,\n",
       " 'mild': 122,\n",
       " 'islanders': 21,\n",
       " 'templi': 1,\n",
       " 'idia': 1,\n",
       " 'woeful': 27,\n",
       " 'feebly': 4,\n",
       " 'selby': 6,\n",
       " 'unreasoned': 1,\n",
       " 'sachar': 9,\n",
       " 'arie': 1,\n",
       " 'mercurially': 1,\n",
       " 'elbows': 1,\n",
       " 'davitelja': 1,\n",
       " 'inviting': 26,\n",
       " 'hildreth': 1,\n",
       " 'unfindable': 1,\n",
       " 'wrongheaded': 2,\n",
       " 'quartet': 22,\n",
       " 'vinny': 18,\n",
       " 'salli': 1,\n",
       " 'implicates': 1,\n",
       " 'couldrelate': 1,\n",
       " 'chokes': 7,\n",
       " 'perce': 1,\n",
       " 'unattached': 2,\n",
       " 'apartment': 320,\n",
       " 'kringle': 1,\n",
       " 'geranium': 1,\n",
       " 'platform': 45,\n",
       " 'norm': 65,\n",
       " 'kendra': 1,\n",
       " 'lohde': 4,\n",
       " 'schema': 1,\n",
       " 'dupery': 1,\n",
       " 'supreme': 47,\n",
       " 'harman': 4,\n",
       " 'robards': 8,\n",
       " 'myrtle': 49,\n",
       " 'voter': 4,\n",
       " 'demoralising': 1,\n",
       " 'trespassed': 1,\n",
       " 'devoted': 109,\n",
       " 'fauke': 1,\n",
       " 'kellaway': 4,\n",
       " 'pepin': 3,\n",
       " 'sortee': 1,\n",
       " 'disgusting': 217,\n",
       " 'vh': 1,\n",
       " 'unleash': 16,\n",
       " 'cdn': 1,\n",
       " 'jewellery': 2,\n",
       " 'dirth': 1,\n",
       " 'suderland': 1,\n",
       " 'rumore': 1,\n",
       " 'dislikeable': 4,\n",
       " 'meditations': 1,\n",
       " 'mendanassos': 1,\n",
       " 'coma': 43,\n",
       " 'keener': 18,\n",
       " 'nursery': 13,\n",
       " 'vacuously': 1,\n",
       " 'schwartzeneggar': 1,\n",
       " 'fervent': 9,\n",
       " 'baur': 1,\n",
       " 'embodied': 15,\n",
       " 'kiddoes': 1,\n",
       " 'tobei': 1,\n",
       " 'unanimousness': 1,\n",
       " 'tunnel': 62,\n",
       " 'manica': 1,\n",
       " 'batman': 414,\n",
       " 'sufferer': 6,\n",
       " 'plotless': 5,\n",
       " 'fcker': 1,\n",
       " 'falsities': 1,\n",
       " 'slutty': 22,\n",
       " 'grahm': 1,\n",
       " 'manhunt': 7,\n",
       " 'qute': 1,\n",
       " 'achievers': 1,\n",
       " 'ejaculating': 1,\n",
       " 'phisique': 1,\n",
       " 'lug': 4,\n",
       " 'insubordination': 2,\n",
       " 'candidate': 61,\n",
       " 'bodys': 1,\n",
       " 'echt': 2,\n",
       " 'fitness': 5,\n",
       " 'ojos': 4,\n",
       " 'barabarian': 1,\n",
       " 'coaches': 7,\n",
       " 'revive': 33,\n",
       " 'accessibility': 8,\n",
       " 'lederer': 1,\n",
       " 'revenues': 5,\n",
       " 'sentimentally': 2,\n",
       " 'flic': 12,\n",
       " 'specificity': 2,\n",
       " 'manuccie': 2,\n",
       " 'atenborough': 3,\n",
       " 'suresh': 1,\n",
       " 'puposelessly': 1,\n",
       " 'sera': 2,\n",
       " 'cartman': 2,\n",
       " 'lyoko': 1,\n",
       " 'hampers': 4,\n",
       " 'superposition': 1,\n",
       " 'kohut': 1,\n",
       " 'demote': 1,\n",
       " 'graduation': 23,\n",
       " 'washed': 44,\n",
       " 'pachelbel': 1,\n",
       " 'sceptical': 17,\n",
       " 'retaliate': 4,\n",
       " 'capones': 1,\n",
       " 'spreader': 1,\n",
       " 'bargained': 17,\n",
       " 'sawant': 2,\n",
       " 'conditioned': 19,\n",
       " 'silvano': 2,\n",
       " 'inoue': 4,\n",
       " 'avantguard': 1,\n",
       " 'cliffhanging': 3,\n",
       " 'ddl': 2,\n",
       " 'undeserving': 9,\n",
       " 'borrough': 1,\n",
       " 'refreshments': 1,\n",
       " 'gangster': 245,\n",
       " 'senelick': 1,\n",
       " 'hefty': 16,\n",
       " 'talkier': 1,\n",
       " 'hoky': 1,\n",
       " 'sneakers': 3,\n",
       " 'emancipated': 3,\n",
       " 'cossimo': 1,\n",
       " 'hardboiled': 2,\n",
       " 'masterton': 1,\n",
       " 'modernism': 3,\n",
       " 'sjunde': 1,\n",
       " 'clarinet': 2,\n",
       " 'seminara': 1,\n",
       " 'accessability': 1,\n",
       " 'farlan': 6,\n",
       " 'musashibo': 1,\n",
       " 'goth': 24,\n",
       " 'abandoning': 20,\n",
       " 'suprise': 6,\n",
       " 'devonsville': 1,\n",
       " 'raimi': 24,\n",
       " 'nabakov': 2,\n",
       " 'bypass': 9,\n",
       " 'hipocresy': 1,\n",
       " 'mxyzptlk': 1,\n",
       " 'lewis': 218,\n",
       " 'familiarizing': 1,\n",
       " 'observe': 46,\n",
       " 'pachebel': 1,\n",
       " 'noose': 11,\n",
       " 'risky': 27,\n",
       " 'tykwer': 11,\n",
       " 'murvyn': 4,\n",
       " 'lavishness': 1,\n",
       " 'panico': 1,\n",
       " ...}"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab_freq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import operator\n",
    "\n",
    "sorted_vocab_freq = list(reversed(sorted(vocab_freq.items(), key=operator.itemgetter(1))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "71238"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sorted_vocab_freq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "TOTAL_VOCAB = 5000\n",
    "\n",
    "word_to_id = dict()\n",
    "id_to_word = dict()\n",
    "for i in range(TOTAL_VOCAB):\n",
    "    word_to_id[sorted_vocab_freq[i][0]] = i\n",
    "    id_to_word[i] = sorted_vocab_freq[i][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'br'"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "id_to_word[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>218</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>166</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>121</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24970</th>\n",
       "      <td>79</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24971</th>\n",
       "      <td>81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24972</th>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24973</th>\n",
       "      <td>66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24974</th>\n",
       "      <td>56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24975</th>\n",
       "      <td>123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24976</th>\n",
       "      <td>134</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24977</th>\n",
       "      <td>219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24978</th>\n",
       "      <td>170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24979</th>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24980</th>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24981</th>\n",
       "      <td>74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24982</th>\n",
       "      <td>87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24983</th>\n",
       "      <td>62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24984</th>\n",
       "      <td>130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24985</th>\n",
       "      <td>102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24986</th>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24987</th>\n",
       "      <td>137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24988</th>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24989</th>\n",
       "      <td>102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24990</th>\n",
       "      <td>80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24991</th>\n",
       "      <td>61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24992</th>\n",
       "      <td>84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24993</th>\n",
       "      <td>93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24994</th>\n",
       "      <td>59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24995</th>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24996</th>\n",
       "      <td>124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24997</th>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24998</th>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24999</th>\n",
       "      <td>108</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>25000 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Len\n",
       "0       54\n",
       "1      224\n",
       "2       74\n",
       "3       62\n",
       "4       55\n",
       "5       66\n",
       "6       78\n",
       "7       70\n",
       "8       72\n",
       "9       60\n",
       "10     229\n",
       "11      72\n",
       "12      37\n",
       "13     218\n",
       "14     166\n",
       "15      60\n",
       "16     145\n",
       "17      60\n",
       "18      64\n",
       "19     165\n",
       "20      68\n",
       "21     119\n",
       "22      69\n",
       "23      75\n",
       "24      42\n",
       "25      62\n",
       "26      59\n",
       "27     210\n",
       "28      83\n",
       "29     121\n",
       "...    ...\n",
       "24970   79\n",
       "24971   81\n",
       "24972   31\n",
       "24973   66\n",
       "24974   56\n",
       "24975  123\n",
       "24976  134\n",
       "24977  219\n",
       "24978  170\n",
       "24979   58\n",
       "24980  100\n",
       "24981   74\n",
       "24982   87\n",
       "24983   62\n",
       "24984  130\n",
       "24985  102\n",
       "24986   26\n",
       "24987  137\n",
       "24988   31\n",
       "24989  102\n",
       "24990   80\n",
       "24991   61\n",
       "24992   84\n",
       "24993   93\n",
       "24994   59\n",
       "24995   57\n",
       "24996  124\n",
       "24997   39\n",
       "24998   52\n",
       "24999  108\n",
       "\n",
       "[25000 rows x 1 columns]"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "review_lengths = pd.DataFrame([len(review) for review in reviews])\n",
    "review_lengths.columns = ['Len']\n",
    "\n",
    "review_lengths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>25000.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>118.36848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>89.42677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>4.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>63.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>88.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>144.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1409.00000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               Len\n",
       "count  25000.00000\n",
       "mean     118.36848\n",
       "std       89.42677\n",
       "min        4.00000\n",
       "25%       63.00000\n",
       "50%       88.00000\n",
       "75%      144.00000\n",
       "max     1409.00000"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "review_lengths.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(265.5, -58.5)"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Removal of outliers using Tukey's Method\n",
    "first_q = review_lengths.Len.quantile([0.25])[0.25]\n",
    "third_q = review_lengths.Len.quantile([0.75])[0.75]\n",
    "\n",
    "upper_threshold = third_q + 1.5*(third_q-first_q)\n",
    "lower_threshold = first_q - 1.5*(third_q-first_q)\n",
    "\n",
    "upper_threshold,lower_threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def convert(l):\n",
    "    new_l = []\n",
    "    for word in l:\n",
    "        if word in word_to_id:\n",
    "            new_l.append(word_to_id[word])\n",
    "    return new_l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25000"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(reviews)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25000"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data_train['Sentiment'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train = []\n",
    "y_train = []\n",
    "\n",
    "for i in range(len(data_train)):\n",
    "    converted_review = convert(reviews[i])\n",
    "    if len(converted_review) <= upper_threshold:\n",
    "        X_train.append(converted_review)\n",
    "        y_train.append(data_train['Sentiment'][i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = np.array(X_train)\n",
    "y_train = np.array(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras\n",
    "from keras.preprocessing import sequence\n",
    "\n",
    "X_train = sequence.pad_sequences(X_train, maxlen=int(upper_threshold),value = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((24010, 265), (24010,))"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape,y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_test = pd.concat([pos_test_data,neg_test_data],ignore_index = True)\n",
    "data_test = data_test.sample(frac=0.3).reset_index(drop=True)\n",
    "\n",
    "validation_reviews = []\n",
    "\n",
    "for index,row in data_test.iterrows():\n",
    "    text = (row['Text'].lower())\n",
    "    validation_reviews.append(textclean(text))\n",
    "    \n",
    "X_val = []\n",
    "y_val = []\n",
    "\n",
    "for i in range(len(data_test)):\n",
    "    converted_review = convert(validation_reviews[i])\n",
    "    if len(converted_review) <= upper_threshold:\n",
    "        X_val.append(converted_review)\n",
    "        y_val.append(data_test['Sentiment'][i])\n",
    "        \n",
    "X_val = np.array(X_val)\n",
    "X_val = sequence.pad_sequences(X_val, maxlen=int(upper_threshold),value = 0)\n",
    "y_val = np.array(y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7235, 265)"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_val.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense,LSTM,Dropout,Activation\n",
    "from keras.layers import Embedding\n",
    "\n",
    "EMBEDDING_LEN = 32\n",
    "\n",
    "model = Sequential()\n",
    "\n",
    "model.add(Embedding(TOTAL_VOCAB,EMBEDDING_LEN,input_length = int(upper_threshold)))\n",
    "model.add(LSTM(100,dropout=0.3,recurrent_dropout=0.2))\n",
    "\n",
    "model.add(Dense(1,activation = 'sigmoid'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_14 (Embedding)     (None, 265, 32)           160000    \n",
      "_________________________________________________________________\n",
      "lstm_14 (LSTM)               (None, 100)               53200     \n",
      "_________________________________________________________________\n",
      "dense_14 (Dense)             (None, 1)                 101       \n",
      "=================================================================\n",
      "Total params: 213,301\n",
      "Trainable params: 213,301\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(loss = 'binary_crossentropy',optimizer = 'adam',metrics = ['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 24010 samples, validate on 7235 samples\n",
      "Epoch 1/3\n",
      "24010/24010 [==============================] - 152s 6ms/step - loss: 0.4726 - acc: 0.7760 - val_loss: 0.3496 - val_acc: 0.8517\n",
      "Epoch 2/3\n",
      "24010/24010 [==============================] - 156s 6ms/step - loss: 0.3207 - acc: 0.8688 - val_loss: 0.3372 - val_acc: 0.8576\n",
      "Epoch 3/3\n",
      "24010/24010 [==============================] - 161s 7ms/step - loss: 0.2886 - acc: 0.8864 - val_loss: 0.3407 - val_acc: 0.8612\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f988ee05f98>"
      ]
     },
     "execution_count": 175,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train,y_train,validation_data = (X_val,y_val),epochs = 3,batch_size = 64)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}