POS_from_scratch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1 align=\"center\">  Built my  statistical POS tagger form scratch </h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__Assumption__:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* We are considering the __tagged dataset / corpus as input__ and apply __supervised Machine Learning algorithm.__\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__Fact__:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* This is __multi-class classification__ task."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Import dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk import word_tokenize\n",
    "from nltk.corpus import brown as cb\n",
    "from nltk.corpus import treebank as tb\n",
    "import pprint\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.feature_extraction import DictVectorizer\n",
    "from sklearn.pipeline import Pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###   Explore dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####  Explore Brown Corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<Text: The Fulton County Grand Jury said Friday an...>\n"
     ]
    }
   ],
   "source": [
    "raw_text = nltk.Text(cb.words('ca01'))\n",
    "print (raw_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['The', 'Fulton', 'County', 'Grand', 'Jury', 'said', 'Friday', 'an', 'investigation', 'of', \"Atlanta's\", 'recent', 'primary', 'election', 'produced', '``', 'no', 'evidence', \"''\", 'that']\n"
     ]
    }
   ],
   "source": [
    "print (cb.words()[0:20])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('The', 'AT'), ('Fulton', 'NP-TL'), ('County', 'NN-TL'), ('Grand', 'JJ-TL'), ('Jury', 'NN-TL'), ('said', 'VBD'), ('Friday', 'NR'), ('an', 'AT'), ('investigation', 'NN'), ('of', 'IN')]\n"
     ]
    }
   ],
   "source": [
    "print (cb.tagged_words()[0:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('The', 'AT'),\n",
      " ('Fulton', 'NP-TL'),\n",
      " ('County', 'NN-TL'),\n",
      " ('Grand', 'JJ-TL'),\n",
      " ('Jury', 'NN-TL'),\n",
      " ('said', 'VBD'),\n",
      " ('Friday', 'NR'),\n",
      " ('an', 'AT'),\n",
      " ('investigation', 'NN'),\n",
      " ('of', 'IN'),\n",
      " (\"Atlanta's\", 'NP$'),\n",
      " ('recent', 'JJ'),\n",
      " ('primary', 'NN'),\n",
      " ('election', 'NN'),\n",
      " ('produced', 'VBD'),\n",
      " ('``', '``'),\n",
      " ('no', 'AT'),\n",
      " ('evidence', 'NN'),\n",
      " (\"''\", \"''\"),\n",
      " ('that', 'CS'),\n",
      " ('any', 'DTI'),\n",
      " ('irregularities', 'NNS'),\n",
      " ('took', 'VBD'),\n",
      " ('place', 'NN'),\n",
      " ('.', '.')]\n"
     ]
    }
   ],
   "source": [
    "tagged_sentences_brown_corpus = nltk.corpus.brown.tagged_sents()\n",
    "pprint.pprint(tagged_sentences_brown_corpus[0])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####  Explore Penn-Treebank Corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<Text: Pierre Vinken , 61 years old , will...>\n"
     ]
    }
   ],
   "source": [
    "raw_text = nltk.Text(tb.words()[0:10])\n",
    "print (raw_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Pierre', 'Vinken', ',', '61', 'years', 'old', ',', 'will', 'join', 'the']\n"
     ]
    }
   ],
   "source": [
    "print (tb.words()[0:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('Pierre', 'NNP'),\n",
      " ('Vinken', 'NNP'),\n",
      " (',', ','),\n",
      " ('61', 'CD'),\n",
      " ('years', 'NNS'),\n",
      " ('old', 'JJ'),\n",
      " (',', ','),\n",
      " ('will', 'MD'),\n",
      " ('join', 'VB'),\n",
      " ('the', 'DT'),\n",
      " ('board', 'NN'),\n",
      " ('as', 'IN'),\n",
      " ('a', 'DT'),\n",
      " ('nonexecutive', 'JJ'),\n",
      " ('director', 'NN'),\n",
      " ('Nov.', 'NNP'),\n",
      " ('29', 'CD'),\n",
      " ('.', '.')]\n"
     ]
    }
   ],
   "source": [
    "tagged_sentences_treebank_corpus = nltk.corpus.treebank.tagged_sents()\n",
    "pprint.pprint (tagged_sentences_treebank_corpus[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h4 align=\"center\"> We will be using Treebank corpus to build our own POS tagger </h4>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tagged sentences:  3914\n"
     ]
    }
   ],
   "source": [
    "print (\"Tagged sentences: \", len(tagged_sentences_treebank_corpus))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tagged words: 100676\n"
     ]
    }
   ],
   "source": [
    "print (\"Tagged words:\", len(nltk.corpus.treebank.tagged_words()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Generate features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Function for generating features form tagged corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def features(sentence, index):\n",
    "    # \"sentence: [w1, w2, ...], index: the index of the word\"\n",
    "    return {\n",
    "    'word': sentence[index],\n",
    "    'is_first': index == 0,\n",
    "    'is_last': index == len(sentence) - 1,\n",
    "    'is_capitalized': sentence[index][0].upper() == sentence[index][0],\n",
    "    'is_all_caps': sentence[index].upper() == sentence[index],\n",
    "    'is_all_lower': sentence[index].lower() == sentence[index],\n",
    "    'prefix-1': sentence[index][0],\n",
    "    'prefix-2': sentence[index][:2],\n",
    "    'prefix-3': sentence[index][:3],\n",
    "    'suffix-1': sentence[index][-1],\n",
    "    'suffix-2': sentence[index][-2:],\n",
    "    'suffix-3': sentence[index][-3:],\n",
    "    'prev_word': '' if index == 0 else sentence[index - 1],\n",
    "    'next_word': '' if index == len(sentence) - 1 else sentence[index + 1],\n",
    "    'has_hyphen': '-' in sentence[index],\n",
    "    'is_numeric': sentence[index].isdigit(),\n",
    "    'capitals_inside': sentence[index][1:].lower() != sentence[index][1:]\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'capitals_inside': False,\n",
      " 'has_hyphen': False,\n",
      " 'is_all_caps': False,\n",
      " 'is_all_lower': False,\n",
      " 'is_capitalized': True,\n",
      " 'is_first': True,\n",
      " 'is_last': False,\n",
      " 'is_numeric': False,\n",
      " 'next_word': 'is',\n",
      " 'prefix-1': 'T',\n",
      " 'prefix-2': 'Th',\n",
      " 'prefix-3': 'Thi',\n",
      " 'prev_word': '',\n",
      " 'suffix-1': 's',\n",
      " 'suffix-2': 'is',\n",
      " 'suffix-3': 'his',\n",
      " 'word': 'This'}\n"
     ]
    }
   ],
   "source": [
    "pprint.pprint(features(['This', 'is', 'a', 'sentence'], 0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Transform Dataset\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extract words form tagged sentences using 'untag' function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def untag(tagged_sentence):\n",
    "    return [w for w, t in tagged_sentence]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Transform dataset into X, y pairs where X = Features Y = POS lables¶"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def transform_to_dataset(tagged_sentences):\n",
    "    X, y = [], []\n",
    "    for tagged in tagged_sentences:\n",
    "        for index in range(len(tagged)):\n",
    "            X.append(features(untag(tagged), index))\n",
    "            y.append(tagged[index][1])\n",
    "            #pprint.pprint(\" original word: \"+ str(tagged) + \" Word: \"+ str(untag(tagged))+ \"Y: \" + y[index])\n",
    "    return X, y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Build training and testing dataset   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "cutoff = int(.75 * len(tagged_sentences_treebank_corpus))\n",
    "training_sentences = tagged_sentences_treebank_corpus[:cutoff]\n",
    "test_sentences = tagged_sentences_treebank_corpus[cutoff:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2935\n",
      "979\n"
     ]
    }
   ],
   "source": [
    "print (len(training_sentences))\n",
    "print (len(test_sentences))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = transform_to_dataset(training_sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "75784\n",
      "75784\n"
     ]
    }
   ],
   "source": [
    "print(len(X)) \n",
    "print(len(y))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Train model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialize the classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = Pipeline([\n",
    "    ('vectorizer', DictVectorizer(sparse=False)),\n",
    "    ('classifier', DecisionTreeClassifier(criterion='entropy'))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('vectorizer', DictVectorizer(dtype=<class 'numpy.float64'>, separator='=', sort=True,\n",
       "        sparse=False)), ('classifier', DecisionTreeClassifier(class_weight=None, criterion='entropy', max_depth=None,\n",
       "            max_features=None, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, presort=False, random_state=None,\n",
       "            splitter='best'))])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Use only the first 10K samples if you're running it multiple times. It takes a fair bit :)\n",
    "clf.fit(X[:20000],y[:20000]) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Measure Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test, y_test = transform_to_dataset(test_sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:91.387%\n"
     ]
    }
   ],
   "source": [
    "print (\"Accuracy:{:.3%}\".format(clf.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Generate POS tags for given sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pos_tag(sentence):\n",
    "    tagged_sentence = []\n",
    "    tags = clf.predict([features(sentence, index) for index in range(len(sentence))])\n",
    "    return zip(sentence, tags)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This            DT\n",
      "is              VBZ\n",
      "my              PRP$\n",
      "friend          NN\n",
      ",                ,\n",
      "John            NNP\n",
      ".                .\n"
     ]
    }
   ],
   "source": [
    "POS_list = list(pos_tag(word_tokenize('This is my friend, John.')))\n",
    "for t in POS_list:\n",
    "    print(u\"{:<16}{:>2}\".format(str(t[0]),str(t[1])))   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We              IN\n",
      "will            MD\n",
      "meet            VB\n",
      "at              IN\n",
      "eight           NN\n",
      "o'clock         NN\n",
      "on              IN\n",
      "Thursday        DT\n",
      "morning         NN\n",
      ".                .\n"
     ]
    }
   ],
   "source": [
    "POS_list = list(pos_tag(word_tokenize(\"We will meet at eight o'clock on Thursday morning.\")))\n",
    "for t in POS_list:\n",
    "    print(u\"{:<16}{:>2}\".format(str(t[0]),str(t[1])))   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Alexander       IN\n",
      ",                ,\n",
      "the             DT\n",
      "great           NN\n",
      "...              :\n",
      "!               CD\n"
     ]
    }
   ],
   "source": [
    "POS_list = list(pos_tag(word_tokenize('Alexander, the great...!')))\n",
    "for t in POS_list:\n",
    "    print(u\"{:<16}{:>2}\".format(str(t[0]),str(t[1])))   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Alexander       IN\n",
      "the             DT\n",
      "Great           NNP\n",
      ",                ,\n",
      "was             VBD\n",
      "a               DT\n",
      "king            NN\n",
      "of              IN\n",
      "the             DT\n",
      "ancient         DT\n",
      "Greek           NNP\n",
      "kingdom         NN\n",
      "of              IN\n",
      "Macedon         NNP\n",
      ".                .\n"
     ]
    }
   ],
   "source": [
    "POS_list = list(pos_tag(word_tokenize('Alexander the Great, was a king of the ancient Greek kingdom of Macedon.')))\n",
    "for t in POS_list:\n",
    "    print(u\"{:<16}{:>2}\".format(str(t[0]),str(t[1])))   \n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}