pycrfsuite-training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# py-CRFsuite NER model\n",
    "\n",
    "this version does not consider capitalization (all words are lowered with `.lower()`), in hopes of creating a case-independent model, for use in situations where case information is not available, such as when using the output of an automatic speech recognition (speech-to-text) system.\n",
    "\n",
    "UPDATE: we use the train/test indices from `data_preprocessing` notebook so that we can make a direct comparison to the `keras` results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "some code adapted from [scrapinghub's python-crfsuite](https://github.com/scrapinghub/python-crfsuite/blob/master/examples/CoNLL%202002.ipynb) under the MIT license:\n",
    "\n",
    "MIT License\n",
    "\n",
    "Copyright (c) 2014-2017 ScrapingHub Inc. and contributors.\n",
    "\n",
    "Permission is hereby granted, free of charge, to any person obtaining a copy\n",
    "of this software and associated documentation files (the \"Software\"), to deal\n",
    "in the Software without restriction, including without limitation the rights\n",
    "to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n",
    "copies of the Software, and to permit persons to whom the Software is\n",
    "furnished to do so, subject to the following conditions:\n",
    "\n",
    "The above copyright notice and this permission notice shall be included in all\n",
    "copies or substantial portions of the Software.\n",
    "\n",
    "THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n",
    "IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n",
    "FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n",
    "AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n",
    "LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n",
    "OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n",
    "SOFTWARE."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pycrfsuite\n",
    "from nltk import pos_tag\n",
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelBinarizer\n",
    "from itertools import chain\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## read ConLL data\n",
    "\n",
    "read the data from `data_preprocessing` and split the data with the saved indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load binary files\n",
    "train_indices = list(np.load('encoded/train_idx.npy'))\n",
    "test_indices  = list(np.load('encoded/test_idx.npy'))\n",
    "sentence_text = list(np.load('encoded/sentence_text.npy'))\n",
    "sentence_post = list(np.load('encoded/sentence_post.npy'))\n",
    "sentence_ners = list(np.load('encoded/sentence_ners.npy'))\n",
    "sentence_zips = [list(zip(sentence_text[i], sentence_post[i])) for i in range(len(sentence_text))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_sents = [sentence_zips[i] for i in train_indices]\n",
    "test_sents  = [sentence_zips[i] for i in test_indices]\n",
    "y_train = [sentence_ners[i] for i in train_indices]\n",
    "y_test  = [sentence_ners[i] for i in test_indices]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['thousands', 'of', 'demonstrators', 'have', 'marched', 'through', 'london', 'to', 'protest', 'the', 'war', 'in', 'iraq', 'and', 'demand', 'the', 'withdrawal', 'of', 'british', 'troops', 'from', 'that', 'country']\n",
      "['NNS', 'IN', 'NNS', 'VBP', 'VBN', 'IN', 'NNP', 'TO', 'VB', 'DT', 'NN', 'IN', 'NNP', 'CC', 'VB', 'DT', 'NN', 'IN', 'JJ', 'NNS', 'IN', 'DT', 'NN']\n",
      "['O', 'O', 'O', 'O', 'O', 'O', 'B-geo', 'O', 'O', 'O', 'O', 'O', 'B-geo', 'O', 'O', 'O', 'O', 'O', 'B-gpe', 'O', 'O', 'O', 'O']\n",
      "[('thousands', 'NNS'), ('of', 'IN'), ('demonstrators', 'NNS'), ('have', 'VBP'), ('marched', 'VBN'), ('through', 'IN'), ('london', 'NNP'), ('to', 'TO'), ('protest', 'VB'), ('the', 'DT'), ('war', 'NN'), ('in', 'IN'), ('iraq', 'NNP'), ('and', 'CC'), ('demand', 'VB'), ('the', 'DT'), ('withdrawal', 'NN'), ('of', 'IN'), ('british', 'JJ'), ('troops', 'NNS'), ('from', 'IN'), ('that', 'DT'), ('country', 'NN')]\n",
      "\n",
      "['they', 'marched', 'from', 'the', 'houses', 'of', 'parliament', 'to', 'a', 'rally', 'in', 'hyde', 'park']\n",
      "['PRP', 'VBD', 'IN', 'DT', 'NNS', 'IN', 'NN', 'TO', 'DT', 'NN', 'IN', 'NNP', 'NNP']\n",
      "['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-geo', 'I-geo']\n",
      "[('they', 'PRP'), ('marched', 'VBD'), ('from', 'IN'), ('the', 'DT'), ('houses', 'NNS'), ('of', 'IN'), ('parliament', 'NN'), ('to', 'TO'), ('a', 'DT'), ('rally', 'NN'), ('in', 'IN'), ('hyde', 'NNP'), ('park', 'NNP')]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for idx, sent in enumerate(sentence_text[:2]):\n",
    "    print(sent)\n",
    "    print(sentence_post[idx])\n",
    "    print(sentence_ners[idx])\n",
    "    print(sentence_zips[idx])\n",
    "    print('')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## gazetteers\n",
    "\n",
    "precompiled lists for feature extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def file2list(filepath):\n",
    "    with open(filepath, 'r') as f:\n",
    "        data = f.readlines()\n",
    "    results = []\n",
    "    for d in data:\n",
    "        results.append(d.lower().replace('\\n', ''))\n",
    "    return list(set(results))\n",
    "\n",
    "gaz_countries = file2list('data/gazetteer_countries.txt')\n",
    "gaz_names = file2list('data/gazetteer_names.txt')\n",
    "gaz_cities = file2list('data/gazetteer_cities.txt')\n",
    "gaz_times = file2list('data/gazetteer_datetime.txt')\n",
    "gaz_demonyms = [s.split()[1] for s in file2list('data/gazetteer_demonyms.txt')]\n",
    "vowels = ['a', 'e', 'i', 'o', 'u']\n",
    "# http://academic.regis.edu/jseibert/Crypto/Frequency.pdf\n",
    "rareletters = ['w', 'k', 'v', 'x', 'z', 'j', 'q']\n",
    "commontrigrams = ['the', 'ing', 'and', 'her', 'ere', 'ent', 'tha', 'nth', 'was', 'eth', 'for', 'dth']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## feature extractors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def word2features(sent, i):\n",
    "    word = sent[i][0]\n",
    "    postag = sent[i][1]\n",
    "    features = [\n",
    "        'bias',\n",
    "        'wordlength='+str(len(word)),\n",
    "        'wordending[-3:]=' + word[-3:],\n",
    "        'wordending[-2:]=' + word[-2:],\n",
    "        'wordending[-1:]=' + word[-1:],\n",
    "        'word.isdigit=%s' % word.isdigit(),\n",
    "        'postag=' + postag,\n",
    "        'posclass=' + postag[:2],\n",
    "        'word.isname=%s' % (word in gaz_names),\n",
    "        'word.iscountry=%s' % (word in gaz_countries),\n",
    "        'word.iscity=%s' % (word in gaz_cities),\n",
    "        'word.isdatetime=%s' % (word in gaz_times),\n",
    "        'word.isdemonym=%s' % (word in gaz_demonyms),\n",
    "        'word.startsvowel=%s' % (word[0] in vowels),\n",
    "        'word.endsvowel=%s' % (word[-1] in vowels),\n",
    "        'word.rareletter=%s' % (len([l for l in list(word) if l in rareletters]) > 0),\n",
    "        'word.commontrigram=%s' % (len([t for t in commontrigrams if t in word]) > 0),\n",
    "        \n",
    "    ]\n",
    "    if i > 0:\n",
    "        word1 = sent[i-1][0]\n",
    "        postag1 = sent[i-1][1]\n",
    "        features.extend([\n",
    "            '-1:wordending[-3:]=' + word1[-3:],\n",
    "            '-1:wordending[-2:]=' + word1[-2:],\n",
    "            '-1:wordending[-1:]=' + word1[-1:],\n",
    "            '-1:word.isdigit=%s' % word1.isdigit(),\n",
    "            '-1:postag=' + postag1,\n",
    "            '-1:posclass=' + postag1[:2],\n",
    "            '-1:word.isname=%s' % (word1 in gaz_names),\n",
    "            '-1:word.iscountry=%s' % (word1 in gaz_countries),\n",
    "            '-1:word.iscity=%s' % (word1 in gaz_cities),\n",
    "            '-1:word.isdatetime=%s' % (word1 in gaz_times),\n",
    "            '-1:word.isdemonym=%s' % (word1 in gaz_demonyms),\n",
    "        ])\n",
    "    else:\n",
    "        features.append('BOS')\n",
    "        \n",
    "    if i < len(sent)-1:\n",
    "        word1 = sent[i+1][0]\n",
    "        postag1 = sent[i+1][1]\n",
    "        features.extend([\n",
    "            '+1:wordending[-3:]=' + word1[-3:],\n",
    "            '+1:wordending[-2:]=' + word1[-2:],\n",
    "            '+1:wordending[-1:]=' + word1[-1:],\n",
    "            '+1:word.isdigit=%s' % word1.isdigit(),\n",
    "            '+1:postag=' + postag1,\n",
    "            '+1:posclass=' + postag1[:2],\n",
    "            '+1:word.isname=%s' % (word1 in gaz_names),\n",
    "            '+1:word.iscountry=%s' % (word1 in gaz_countries),\n",
    "            '+1:word.iscity=%s' % (word1 in gaz_cities),\n",
    "            '+1:word.isdatetime=%s' % (word1 in gaz_times),\n",
    "            '+1:word.isdemonym=%s' % (word1 in gaz_demonyms),\n",
    "        ])\n",
    "    else:\n",
    "        features.append('EOS')\n",
    "                \n",
    "    return features\n",
    "\n",
    "\n",
    "def sent2features(sent):\n",
    "    return [word2features(sent, i) for i in range(len(sent))]\n",
    "\n",
    "def sent2tokens(sent):\n",
    "    return [token for token, postag in sent]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 24 s, sys: 793 ms, total: 24.8 s\n",
      "Wall time: 24.8 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "X_train = [sent2features(s) for s in train_sents]\n",
    "X_test = [sent2features(s) for s in test_sents]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['bias',\n",
       " 'wordlength=2',\n",
       " 'wordending[-3:]=he',\n",
       " 'wordending[-2:]=he',\n",
       " 'wordending[-1:]=e',\n",
       " 'word.isdigit=False',\n",
       " 'postag=PRP',\n",
       " 'posclass=PR',\n",
       " 'word.isname=False',\n",
       " 'word.iscountry=False',\n",
       " 'word.iscity=False',\n",
       " 'word.isdatetime=False',\n",
       " 'word.isdemonym=False',\n",
       " 'word.startsvowel=False',\n",
       " 'word.endsvowel=True',\n",
       " 'word.rareletter=False',\n",
       " 'word.commontrigram=False',\n",
       " 'BOS',\n",
       " '+1:wordending[-3:]=led',\n",
       " '+1:wordending[-2:]=ed',\n",
       " '+1:wordending[-1:]=d',\n",
       " '+1:word.isdigit=False',\n",
       " '+1:postag=VBD',\n",
       " '+1:posclass=VB',\n",
       " '+1:word.isname=False',\n",
       " '+1:word.iscountry=False',\n",
       " '+1:word.iscity=False',\n",
       " '+1:word.isdatetime=False',\n",
       " '+1:word.isdemonym=False']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sent2features(train_sents[0])[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## create model and train\n",
    "\n",
    "we create a pycrfsuite model `Trainer` and train on the train data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 14.7 s, sys: 109 ms, total: 14.8 s\n",
      "Wall time: 14.8 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "trainer = pycrfsuite.Trainer(verbose=False)\n",
    "for xseq, yseq in zip(X_train, y_train):\n",
    "    trainer.append(xseq, yseq)\n",
    "    \n",
    "trainer.set_params({\n",
    "    'c1': 1.0,   # coefficient for L1 penalty\n",
    "    'c2': 1e-3,  # coefficient for L2 penalty\n",
    "    'max_iterations': 200,  # stop earlier\n",
    "\n",
    "    # include transitions that are possible, but not observed\n",
    "    'feature.possible_transitions': True\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 8min 5s, sys: 513 ms, total: 8min 5s\n",
      "Wall time: 8min 4s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "trainer.train('model/conll2002-test.crfsuite')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## create tagger, evaluation\n",
    "\n",
    "we can use the trained model to label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<contextlib.closing at 0x7f7029c28160>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tagger = pycrfsuite.Tagger()\n",
    "tagger.open('model/conll2002-test.crfsuite')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the trip by holmes is taking place in the aftermath of heavy fighting between allied somali-ethiopian forces and islamists in mogadishu that killed hundreds of somalis and forced up to 4,00,000 others to escape to makeshift camps on the city 's outskirts\n",
      "\n",
      "Predicted: O O O B-org O O O O O O O O O O O O O O O O B-geo O O O O B-gpe O O O O O O O O O O O O O O O O\n",
      "Correct:   O O O B-per O O O O O O O O O O O O O O B-org O B-geo O O O O B-gpe O O O O O O O O O O O O O O O O\n"
     ]
    }
   ],
   "source": [
    "idx = 4\n",
    "example_sent = test_sents[idx]\n",
    "print(' '.join(sent2tokens(example_sent)), end='\\n\\n')\n",
    "\n",
    "print(\"Predicted:\", ' '.join(tagger.tag(sent2features(example_sent))))\n",
    "print(\"Correct:  \", ' '.join(y_test[idx]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def bio_classification_report(y_true, y_pred):\n",
    "    \"\"\"\n",
    "    from scrapinghub's python-crfsuite example\n",
    "    \n",
    "    Classification report for a list of BIO-encoded sequences.\n",
    "    It computes token-level metrics and discards \"O\" labels.\n",
    "    \n",
    "    Note that it requires scikit-learn 0.15+ (or a version from github master)\n",
    "    to calculate averages properly!\n",
    "    \"\"\"\n",
    "    lb = LabelBinarizer()\n",
    "    y_true_combined = lb.fit_transform(list(chain.from_iterable(y_true)))\n",
    "    y_pred_combined = lb.transform(list(chain.from_iterable(y_pred)))\n",
    "        \n",
    "    tagset = set(lb.classes_) - {'O'}\n",
    "    tagset = sorted(tagset, key=lambda tag: tag.split('-', 1)[::-1])\n",
    "    class_indices = {cls: idx for idx, cls in enumerate(lb.classes_)}\n",
    "    \n",
    "    return classification_report(\n",
    "        y_true_combined,\n",
    "        y_pred_combined,\n",
    "        labels = [class_indices[cls] for cls in tagset],\n",
    "        target_names = tagset,\n",
    "    )\n",
    "    lb = LabelBinarizer()\n",
    "    y_true_combined = lb.fit_transform(list(chain.from_iterable(y_true)))\n",
    "    y_pred_combined = lb.transform(list(chain.from_iterable(y_pred)))\n",
    "        \n",
    "    tagset = set(lb.classes_) - {'O'}\n",
    "    tagset = sorted(tagset, key=lambda tag: tag.split('-', 1)[::-1])\n",
    "    class_indices = {cls: idx for idx, cls in enumerate(lb.classes_)}\n",
    "    \n",
    "    return classification_report(\n",
    "        y_true_combined,\n",
    "        y_pred_combined,\n",
    "        labels = [class_indices[cls] for cls in tagset],\n",
    "        target_names = tagset,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "      B-art       0.31      0.06      0.10        69\n",
      "      I-art       0.00      0.00      0.00        54\n",
      "      B-eve       0.52      0.35      0.42        46\n",
      "      I-eve       0.35      0.22      0.27        36\n",
      "      B-geo       0.85      0.90      0.87      5629\n",
      "      I-geo       0.81      0.74      0.77      1120\n",
      "      B-gpe       0.94      0.92      0.93      2316\n",
      "      I-gpe       0.89      0.65      0.76        26\n",
      "      B-nat       0.73      0.46      0.56        24\n",
      "      I-nat       0.60      0.60      0.60         5\n",
      "      B-org       0.78      0.69      0.73      2984\n",
      "      I-org       0.77      0.76      0.76      2377\n",
      "      B-per       0.81      0.81      0.81      2424\n",
      "      I-per       0.81      0.90      0.85      2493\n",
      "      B-tim       0.92      0.83      0.87      2989\n",
      "      I-tim       0.82      0.70      0.75      1017\n",
      "\n",
      "avg / total       0.83      0.82      0.82     23609\n",
      "\n",
      "CPU times: user 3.02 s, sys: 24 ms, total: 3.04 s\n",
      "Wall time: 3.04 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "y_pred = [tagger.tag(xseq) for xseq in X_test]\n",
    "print(bio_classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top positive:\n",
      "10.672160 B-tim  word.isdatetime=True\n",
      "8.881390 I-gpe  +1:wordending[-3:]=yor\n",
      "7.795847 B-org  wordending[-3:]=213\n",
      "7.558754 B-gpe  wordending[-3:]=hen\n",
      "7.199009 I-tim  word.isdatetime=True\n",
      "6.781945 B-gpe  wordending[-3:]=pal\n",
      "6.772639 B-tim  BOS\n",
      "6.482320 O      BOS\n",
      "6.140998 B-art  wordending[-3:]=gdp\n",
      "6.081018 B-eve  wordending[-3:]=pic\n",
      "5.880988 B-gpe  wordending[-3:]=ger\n",
      "5.863983 B-org  wordending[-3:]=fm\n",
      "5.607637 B-per  wordending[-3:]=ms.\n",
      "5.596039 B-gpe  wordending[-3:]=khs\n",
      "5.493595 B-per  -1:wordending[-3:]=sab\n",
      "5.437427 B-tim  wordending[-2:]=0s\n",
      "5.428340 B-org  -1:wordending[-3:]=ngh\n",
      "5.424400 I-org  wordending[-3:]=rp.\n",
      "5.336027 I-per  -1:wordending[-3:]=ofi\n",
      "5.173353 B-gpe  wordending[-3:]=qis\n",
      "\n",
      "Top negative:\n",
      "-3.155254 I-tim  wordending[-3:]=his\n",
      "-3.197098 I-tim  wordending[-3:]=ear\n",
      "-3.198708 O      wordending[-3:]=iri\n",
      "-3.245888 O      wordending[-3:]=6th\n",
      "-3.304280 I-org  +1:wordending[-3:]=yat\n",
      "-3.314575 O      wordending[-3:]=3th\n",
      "-3.378080 O      wordending[-3:]=erb\n",
      "-3.522689 I-org  wordending[-3:]=day\n",
      "-3.619587 O      +1:wordending[-3:]=rs.\n",
      "-3.623329 O      wordending[-3:]=2nd\n",
      "-3.830555 O      wordending[-3:]=lis\n",
      "-3.886678 O      wordending[-3:]=5th\n",
      "-4.002612 O      +1:wordending[-3:]=nid\n",
      "-4.202672 O      wordending[-3:]=fox\n",
      "-4.304913 O      postag=NNP\n",
      "-4.416892 O      wordending[-2:]=0s\n",
      "-4.647622 I-tim  wordending[-1:]=k\n",
      "-4.988933 O      wordending[-3:]=1st\n",
      "-5.163742 I-gpe  word.iscountry=False\n",
      "-6.768453 O      wordending[-3:]=de\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "info = tagger.info()\n",
    "\n",
    "def print_state_features(state_features):\n",
    "    for (attr, label), weight in state_features:\n",
    "        print(\"%0.6f %-6s %s\" % (weight, label, attr))    \n",
    "\n",
    "print(\"Top positive:\")\n",
    "print_state_features(Counter(info.state_features).most_common(20))\n",
    "\n",
    "print(\"\\nTop negative:\")\n",
    "print_state_features(Counter(info.state_features).most_common()[-20:])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## decode some results and save to csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def decode(s):\n",
    "    toks = s.split()\n",
    "    toks = [w.lower() for w in toks]\n",
    "    post = pos_tag(toks)\n",
    "    tags = tagger.tag(sent2features(post))\n",
    "        \n",
    "    return tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "test_strings = [[w for w, t in s] for s in test_sents]\n",
    "test_postags = [[t for w, t in s] for s in test_sents]\n",
    "# test_strings[:3], test_postags[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "decoded = []\n",
    "for idx, sentlist in enumerate(test_strings[:500]):\n",
    "    \n",
    "    # join tokens into string and get preds\n",
    "    preds = decode(' '.join(sentlist))\n",
    "    \n",
    "    # print(len(sentlist), len(y_test[idx]), len(preds))\n",
    "    \n",
    "    word, pos, tru, prd = [], [], [], []\n",
    "\n",
    "    # for each word in the sentence...\n",
    "    for jdx, wrd in enumerate(sentlist):\n",
    "\n",
    "        # word\n",
    "        word.append(wrd)\n",
    "        # pos\n",
    "        pos.append(test_postags[idx][jdx])\n",
    "        # decode true NER tag\n",
    "        tru.append(y_test[idx][jdx])\n",
    "        # decode prediction\n",
    "        prd.append(preds[jdx])\n",
    "\n",
    "    answ = pd.DataFrame(\n",
    "    {\n",
    "        'word': word,\n",
    "        'pos': pos,\n",
    "        'true': tru,\n",
    "        'pred': prd,\n",
    "        'skip' : [' ' for s in word]\n",
    "    })\n",
    "    answ = answ[['word', 'pos', 'true', 'pred', 'skip']]\n",
    "    answ = answ.T\n",
    "    decoded.append(answ)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>55</th>\n",
       "      <th>56</th>\n",
       "      <th>57</th>\n",
       "      <th>58</th>\n",
       "      <th>59</th>\n",
       "      <th>60</th>\n",
       "      <th>61</th>\n",
       "      <th>62</th>\n",
       "      <th>63</th>\n",
       "      <th>64</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>word</th>\n",
       "      <td>the</td>\n",
       "      <td>post</td>\n",
       "      <td>became</td>\n",
       "      <td>vacant</td>\n",
       "      <td>in</td>\n",
       "      <td>october</td>\n",
       "      <td>,</td>\n",
       "      <td>when</td>\n",
       "      <td>former</td>\n",
       "      <td>costa</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pos</th>\n",
       "      <td>DT</td>\n",
       "      <td>NN</td>\n",
       "      <td>VBD</td>\n",
       "      <td>JJ</td>\n",
       "      <td>IN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>,</td>\n",
       "      <td>WRB</td>\n",
       "      <td>JJ</td>\n",
       "      <td>NNP</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>true</th>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>B-tim</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pred</th>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>B-tim</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>skip</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 65 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       0     1       2       3   4        5  6     7       8      9  ...   \\\n",
       "word  the  post  became  vacant  in  october  ,  when  former  costa ...    \n",
       "pos    DT    NN     VBD      JJ  IN      NNP  ,   WRB      JJ    NNP ...    \n",
       "true    O     O       O       O   O    B-tim  O     O       O      O ...    \n",
       "pred    O     O       O       O   O    B-tim  O     O       O      O ...    \n",
       "skip                                                                 ...    \n",
       "\n",
       "       55   56   57   58   59   60   61   62   63   64  \n",
       "word  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  \n",
       "pos   NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  \n",
       "true  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  \n",
       "pred  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  \n",
       "skip  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  \n",
       "\n",
       "[5 rows x 65 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result = pd.concat(decoded)\n",
    "result.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "result.to_csv('results/pyCRF_sample.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "atlas",
   "language": "python",
   "name": "atlas"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}