Opinion Mining.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Opinion Mining Using POS Tagging and Grammar Association"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "from xml.dom import minidom"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parse an XML File and extract data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "doc = minidom.parse('reviews.review')\n",
    "\n",
    "review_data = []\n",
    "\n",
    "reviews = doc.getElementsByTagName(\"review\")\n",
    "for review in reviews:\n",
    "        review_text = review.getElementsByTagName(\"review_text\")[0]\n",
    "        review_data.append(review_text.firstChild.data.replace(\"\\n\",\"\"))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exploratory Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "986"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(review_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'Fabulous product --- now I can store over 700 photos in the 8 mega pixel range.    Happy I purchased this one -- very trouble free'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "review_data[45]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "showing info http://www.nltk.org/nltk_data/\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download()\n",
    "# Download Averaged Perceptron Tagger in Models\n",
    "# Help on Tagsets in Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Word Tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['And', 'now', 'for', 'something', 'completely', 'different']\n"
     ]
    }
   ],
   "source": [
    "text = nltk.word_tokenize(\"And now for something completely different\")\n",
    "#print text\n",
    "nltk.pos_tag(text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table border=\"2\" width=\"100%\" cellspacing=\"2\">\n",
    "\t<tbody>\n",
    "\t\t<tr>\n",
    "\t\t\t<th>POS Tag</th>\n",
    "\t\t\t<th>Description</th>\n",
    "\t\t\t<th>Example</th>\n",
    "\t\t</tr>\n",
    "\t\t<tr>\n",
    "\t\t\t<td>CC</td>\n",
    "\t\t\t<td>coordinating conjunction</td>\n",
    "\t\t\t<td>and</td>\n",
    "\t\t</tr>\n",
    "\t\t<tr>\n",
    "\t\t\t<td>CD</td>\n",
    "\t\t\t<td>cardinal number</td>\n",
    "\t\t\t<td>1, third</td>\n",
    "\t\t</tr>\n",
    "\t\t<tr>\n",
    "\t\t\t<td>DT</td>\n",
    "\t\t\t<td>determiner</td>\n",
    "\t\t\t<td>the</td>\n",
    "\t\t</tr>\n",
    "\t\t<tr>\n",
    "\t\t\t<td>EX</td>\n",
    "\t\t\t<td>existential there</td>\n",
    "\t\t\t<td>\n",
    "\t\t\t\t<em>there</em> is</td>\n",
    "\t\t</tr>\n",
    "\t\t<tr>\n",
    "\t\t\t<td>FW</td>\n",
    "\t\t\t<td>foreign word</td>\n",
    "\t\t\t<td>d&#8217;hoevre</td>\n",
    "\t\t</tr>\n",
    "        <tr>\n",
    "\t\t\t<td>IN</td>\n",
    "\t\t\t<td>preposition/subordinating conjunction</td>\n",
    "\t\t\t<td>in, of, like</td>\n",
    "\t\t</tr>\n",
    "\t\t<tr>\n",
    "\t\t\t<td>JJ</td>\n",
    "\t\t\t<td>adjective</td>\n",
    "\t\t\t<td>big</td>\n",
    "\t\t</tr>\n",
    "\t\t<tr>\n",
    "\t\t\t<td>JJR</td>\n",
    "\t\t\t<td>adjective, comparative</td>\n",
    "\t\t\t<td>bigger</td>\n",
    "\t\t</tr>\n",
    "\t\t<tr>\n",
    "\t\t\t<td>JJS</td>\n",
    "\t\t\t<td>adjective, superlative</td>\n",
    "\t\t\t<td>biggest</td>\n",
    "\t\t</tr>\n",
    "\t\t<tr>\n",
    "\t\t\t<td>LS</td>\n",
    "\t\t\t<td>list marker</td>\n",
    "\t\t\t<td>1)</td>\n",
    "\t\t</tr>\n",
    "\t\t<tr>\n",
    "\t\t\t<td>MD</td>\n",
    "\t\t\t<td>modal</td>\n",
    "\t\t\t<td>could, will</td>\n",
    "\t\t</tr>\n",
    "        <tr>\n",
    "\t\t\t<td>NN</td>\n",
    "\t\t\t<td>noun, singular or mass</td>\n",
    "\t\t\t<td>door</td>\n",
    "\t\t</tr>\n",
    "\t\t<tr>\n",
    "\t\t\t<td>NNS</td>\n",
    "\t\t\t<td>noun plural</td>\n",
    "\t\t\t<td>doors</td>\n",
    "\t\t</tr>\n",
    "\t\t<tr>\n",
    "\t\t\t<td>NNP</td>\n",
    "\t\t\t<td>proper noun, singular</td>\n",
    "\t\t\t<td>John</td>\n",
    "\t\t</tr>\n",
    "\t\t<tr>\n",
    "\t\t\t<td>NNPS</td>\n",
    "\t\t\t<td>proper noun, plural</td>\n",
    "\t\t\t<td>Vikings</td>\n",
    "\t\t</tr>\n",
    "\t\t<tr>\n",
    "\t\t\t<td>PDT</td>\n",
    "\t\t\t<td>predeterminer</td>\n",
    "\t\t\t<td>\n",
    "\t\t\t\t<em>both</em> the boys</td>\n",
    "\t\t</tr>\n",
    "\t\t<tr>\n",
    "\t\t\t<td>POS</td>\n",
    "\t\t\t<td>possessive ending</td>\n",
    "\t\t\t<td>friend<em>&#8216;s</em>\n",
    "\t\t\t</td>\n",
    "\t\t</tr>\n",
    "\t\t<tr>\n",
    "\t\t\t<td>PRP</td>\n",
    "\t\t\t<td>personal pronoun</td>\n",
    "\t\t\t<td>I, he, it</td>\n",
    "\t\t</tr>\n",
    "\t\t<tr>\n",
    "\t\t\t<td>PRP</td>\n",
    "\t\t\t<td>possessive pronoun</td>\n",
    "\t\t\t<td>my, his</td>\n",
    "\t\t</tr>\n",
    "\t\t<tr>\n",
    "\t\t\t<td>RB</td>\n",
    "\t\t\t<td>adverb</td>\n",
    "\t\t\t<td>however, usually, naturally, here, good</td>\n",
    "\t\t</tr>\n",
    "\t\t<tr>\n",
    "\t\t\t<td>RBR</td>\n",
    "\t\t\t<td>adverb, comparative</td>\n",
    "\t\t\t<td>better</td>\n",
    "\t\t</tr>\n",
    "\t\t<tr>\n",
    "\t\t\t<td>RBS</td>\n",
    "\t\t\t<td>adverb, superlative</td>\n",
    "\t\t\t<td>best</td>\n",
    "\t\t</tr>\n",
    "        <tr>\n",
    "\t\t\t<td>RP</td>\n",
    "\t\t\t<td>particle</td>\n",
    "\t\t\t<td>give <em> up </em>\n",
    "\t\t\t</td>\n",
    "\t\t</tr>\n",
    "\t\t<tr>\n",
    "\t\t\t<td>TO</td>\n",
    "\t\t\t<td>to</td>\n",
    "\t\t\t<td>\n",
    "\t\t\t\t<em>to</em> go, <em>to</em> him</td>\n",
    "\t\t</tr>\n",
    "\t\t<tr>\n",
    "\t\t\t<td>UH</td>\n",
    "\t\t\t<td>interjection</td>\n",
    "\t\t\t<td>uhhuhhuhh</td>\n",
    "\t\t</tr>\n",
    "\t\t<tr>\n",
    "\t\t\t<td>VB</td>\n",
    "\t\t\t<td>verb, base form</td>\n",
    "\t\t\t<td>take</td>\n",
    "\t\t</tr>\n",
    "\t\t<tr>\n",
    "\t\t\t<td>VBD</td>\n",
    "\t\t\t<td>verb, past tense</td>\n",
    "\t\t\t<td>took</td>\n",
    "\t\t</tr>\n",
    "\t\t<tr>\n",
    "\t\t\t<td>VBG</td>\n",
    "\t\t\t<td>verb, gerund/present participle</td>\n",
    "\t\t\t<td>taking</td>\n",
    "\t\t</tr>\n",
    "\t\t<tr>\n",
    "\t\t\t<td>VBN</td>\n",
    "\t\t\t<td>verb, past participle</td>\n",
    "\t\t\t<td>taken</td>\n",
    "\t\t</tr>\n",
    "\t\t<tr>\n",
    "\t\t\t<td>VBP</td>\n",
    "\t\t\t<td>verb, sing. present, non-3d</td>\n",
    "\t\t\t<td>take</td>\n",
    "\t\t</tr>\n",
    "\t\t<tr>\n",
    "\t\t\t<td>VBZ</td>\n",
    "\t\t\t<td>verb, 3rd person sing. present</td>\n",
    "\t\t\t<td>takes</td>\n",
    "\t\t</tr>\n",
    "\t\t<tr>\n",
    "\t\t\t<td>WDT</td>\n",
    "\t\t\t<td>wh-determiner</td>\n",
    "\t\t\t<td>which</td>\n",
    "\t\t</tr>\n",
    "\t\t<tr>\n",
    "\t\t\t<td>WP</td>\n",
    "\t\t\t<td>wh-pronoun</td>\n",
    "\t\t\t<td>who, what</td>\n",
    "\t\t</tr>\n",
    "\t\t<tr>\n",
    "\t\t\t<td>WP</td>\n",
    "\t\t\t<td>possessive wh-pronoun</td>\n",
    "\t\t\t<td>whose</td>\n",
    "\t\t</tr>\n",
    "\t\t<tr>\n",
    "\t\t\t<td>WRB</td>\n",
    "\t\t\t<td>wh-abverb</td>\n",
    "\t\t\t<td>where, when</td>\n",
    "\t\t</tr>\n",
    "\t</tbody>\n",
    "</table>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## POS Tagging of  Words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'I', 'PRP'),\n",
       " (u'purchased', 'VBD'),\n",
       " (u'this', 'DT'),\n",
       " (u'unit', 'NN'),\n",
       " (u'due', 'JJ'),\n",
       " (u'to', 'TO'),\n",
       " (u'frequent', 'JJ'),\n",
       " (u'blackouts', 'NNS'),\n",
       " (u'in', 'IN'),\n",
       " (u'my', 'PRP$'),\n",
       " (u'area', 'NN'),\n",
       " (u'and', 'CC'),\n",
       " (u'2', 'CD'),\n",
       " (u'power', 'NN'),\n",
       " (u'supplies', 'NNS'),\n",
       " (u'going', 'VBG'),\n",
       " (u'bad', 'JJ'),\n",
       " (u'.', '.'),\n",
       " (u'It', 'PRP'),\n",
       " (u'will', 'MD'),\n",
       " (u'run', 'VB'),\n",
       " (u'my', 'PRP$'),\n",
       " (u'cable', 'NN'),\n",
       " (u'modem', 'NN'),\n",
       " (u',', ','),\n",
       " (u'router', 'NN'),\n",
       " (u',', ','),\n",
       " (u'PC', 'NN'),\n",
       " (u',', ','),\n",
       " (u'and', 'CC'),\n",
       " (u'LCD', 'NNP'),\n",
       " (u'monitor', 'NN'),\n",
       " (u'for', 'IN'),\n",
       " (u'5', 'CD'),\n",
       " (u'minutes', 'NNS'),\n",
       " (u'.', '.'),\n",
       " (u'This', 'DT'),\n",
       " (u'is', 'VBZ'),\n",
       " (u'more', 'JJR'),\n",
       " (u'than', 'IN'),\n",
       " (u'enough', 'JJ'),\n",
       " (u'time', 'NN'),\n",
       " (u'to', 'TO'),\n",
       " (u'save', 'VB'),\n",
       " (u'work', 'NN'),\n",
       " (u'and', 'CC'),\n",
       " (u'shut', 'VB'),\n",
       " (u'down', 'RP'),\n",
       " (u'.', '.'),\n",
       " (u'Equally', 'RB'),\n",
       " (u'important', 'JJ'),\n",
       " (u',', ','),\n",
       " (u'I', 'PRP'),\n",
       " (u'know', 'VBP'),\n",
       " (u'that', 'IN'),\n",
       " (u'my', 'PRP$'),\n",
       " (u'electronics', 'NNS'),\n",
       " (u'are', 'VBP'),\n",
       " (u'receiving', 'VBG'),\n",
       " (u'clean', 'JJ'),\n",
       " (u'power.I', 'NN'),\n",
       " (u'feel', 'NN'),\n",
       " (u'that', 'IN'),\n",
       " (u'this', 'DT'),\n",
       " (u'investment', 'NN'),\n",
       " (u'is', 'VBZ'),\n",
       " (u'minor', 'JJ'),\n",
       " (u'compared', 'VBN'),\n",
       " (u'to', 'TO'),\n",
       " (u'the', 'DT'),\n",
       " (u'loss', 'NN'),\n",
       " (u'of', 'IN'),\n",
       " (u'valuable', 'JJ'),\n",
       " (u'data', 'NNS'),\n",
       " (u'or', 'CC'),\n",
       " (u'the', 'DT'),\n",
       " (u'failure', 'NN'),\n",
       " (u'of', 'IN'),\n",
       " (u'equipment', 'NN'),\n",
       " (u'due', 'JJ'),\n",
       " (u'to', 'TO'),\n",
       " (u'a', 'DT'),\n",
       " (u'power', 'NN'),\n",
       " (u'spike', 'NN'),\n",
       " (u'or', 'CC'),\n",
       " (u'an', 'DT'),\n",
       " (u'irregular', 'JJ'),\n",
       " (u'power', 'NN'),\n",
       " (u'supply.As', 'NN'),\n",
       " (u'always', 'RB'),\n",
       " (u',', ','),\n",
       " (u'Amazon', 'NNP'),\n",
       " (u'had', 'VBD'),\n",
       " (u'it', 'PRP'),\n",
       " (u'to', 'TO'),\n",
       " (u'me', 'PRP'),\n",
       " (u'in', 'IN'),\n",
       " (u'2', 'CD'),\n",
       " (u'business', 'NN'),\n",
       " (u'days', 'NNS')]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tagged_reviews = []\n",
    "for each_review_text in review_data[0:10]:\n",
    "    text = nltk.word_tokenize(each_review_text)\n",
    "    tagged_reviews.append(nltk.pos_tag(text))\n",
    "tagged_reviews[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Opinion Mining/Association"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#nltk.help.upenn_tagset()\n",
    "grammar = \"NP: {<DT|PP|CD>?<JJ||JJR|JJS>*<NN|NNS|PRP|NNP|IN|PRP\\$>+<VBD|VBZ|VBN|VBP|IN>*<JJ|RB>*<PRP|NN|NNS>*}\"\n",
    "cp = nltk.RegexpParser(grammar)\n",
    "results = cp.parse(tagged_reviews[9])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot the parse tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "results.draw()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "75"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tagged_reviewsiews_reviews[9])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Explore results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[u'this', u'CF', u'card', u'for', u'my', u'Canon', u'Digital', u'Rebel']\n",
      "[u'it', u'has', u'worked', u'fine']\n",
      "[u'though', u'I', u'do', u\"n't\"]\n",
      "[u'an', u'expert', u'digital', u'photographer']\n",
      "[u'the', u'card', u'is', u'empty']\n",
      "[u'127', u'shots', u'available']\n",
      "[u'though', u'it', u'takes']\n",
      "[u'longer', u'than', u'I', u'was', u'used']\n",
      "[u'with', u'my', u'point-and-shoot']\n",
      "[u'its', u'SmartMedia', u'card']\n"
     ]
    }
   ],
   "source": [
    "for result in results:\n",
    "    if type(result) == nltk.tree.Tree:\n",
    "        assoc=[]\n",
    "        for res in result:\n",
    "            assoc.append(res[0])\n",
    "        if len(assoc) > 2:\n",
    "            print assoc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u\"After going through the reviews, I bought this CF card for my Canon Digital Rebel. So far it has worked fine, though I don't pretend to be an expert digital photographer. When the card is empty, it shows 127 shots available. It seems to read reasonably fast, though it takes a bit longer than I was used to with my point-and-shoot Olympus and its SmartMedia card.\""
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "review_data[9]\n",
    "#tagged_reviews[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}