01_create_train_test_attn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CLASSPATH'] = \"H:/Relation-Classification/stanford/stanford-postagger-2017-06-09\"\n",
    "\n",
    "from nltk.tokenize.stanford import StanfordTokenizer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "trainFile = './corpus/SemEval2010_task8_training/TRAIN_FILE.TXT'\n",
    "testFile = './corpus/SemEval2010_task8_testing_keys/TEST_FILE_FULL.TXT'\n",
    "\n",
    "op_trainFile = \"./files/train_attn.txt\"\n",
    "op_testFile = \"./files/test_attn.txt\"\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "def createFile(filepath, outputpath):    \n",
    "    \n",
    "    def clean_tokens(sent_num, tokens):\n",
    "        ret = []\n",
    "        for t in tokens:\n",
    "            t = t.strip().split()\n",
    "            if len(t) > 1: \n",
    "                print(sent_num, t)\n",
    "            t = \"_\".join(t)\n",
    "            ret.append(t)\n",
    "        return ret \n",
    "    \n",
    "    fOut = open(outputpath, 'w')\n",
    "    lines = [line.strip() for line in open(filepath)]\n",
    "    for idx in range(0, len(lines), 4):\n",
    "        sentence_num = lines[idx].split(\"\\t\")[0]\n",
    "        sentence = lines[idx].split(\"\\t\")[1][1:-1]\n",
    "        label = lines[idx+1]\n",
    "        \n",
    "        sentence = sentence.replace(\"<e1>\", \" E1_START \").replace(\"</e1>\", \" E1_END \")\n",
    "        sentence = sentence.replace(\"<e2>\", \" E2_START \").replace(\"</e2>\", \" E2_END \")\n",
    "        #sentence = sentence.replace(\"<e1>\", \" _e1_ \").replace(\"</e1>\", \" _/e1_ \")\n",
    "        #sentence = sentence.replace(\"<e2>\", \" _e2_ \").replace(\"</e2>\", \" _/e2_ \")\n",
    "\n",
    "        tokens = StanfordTokenizer().tokenize(sentence)        \n",
    "        tokens = clean_tokens(sentence_num, tokens)\n",
    "        \n",
    "        fOut.write(\" \".join([ label, \" \".join(tokens) ]))\n",
    "        fOut.write(\"\\n\")\n",
    "    fOut.close()\n",
    "        \n",
    "    print(outputpath, \"created\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2609 ['1', '1/2']\n",
      "7589 ['1', '1/2']\n",
      "./files/train_attn.txt created\n",
      "10591 ['1', '1/4']\n",
      "10665 ['6', '1/2']\n",
      "./files/test_attn.txt created\n",
      "Train / Test file created\n"
     ]
    }
   ],
   "source": [
    "createFile(trainFile, op_trainFile)\n",
    "createFile(testFile, op_testFile)\n",
    "\n",
    "print(\"Train / Test file created\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 2609 ['1', '1/2']\n",
    "# 7589 ['1', '1/2']\n",
    "# 10591 ['1', '1/4']\n",
    "# 10665 ['6', '1/2']\n",
    "# Train / Test file created"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}