Semantic Labeling with TextRank.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import OrderedDict\n",
    "import numpy as np\n",
    "import spacy\n",
    "from spacy.lang.en.stop_words import STOP_WORDS\n",
    "import neuralcoref\n",
    "import nltk\n",
    "import json\n",
    "from collections import Counter\n",
    "from allennlp.predictors.predictor import Predictor\n",
    "import csv\n",
    "import re\n",
    "from nltk.corpus import stopwords \n",
    "\n",
    "stop_words = stopwords.words('english')\n",
    "stop_words.extend(\",\")\n",
    "inputList = []\n",
    "nlp = spacy.load('en_core_web_sm')\n",
    "neuralcoref.add_to_pipe(nlp)\n",
    "predictor = Predictor.from_path(\"/home/raj/UB_Stuff/CSE_635/Akshaya_code/srl-model-2018.05.25.tar.gz\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "class OrderedCounter(Counter, OrderedDict):\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TextRank4Keyword():\n",
    "    \"\"\"Extract keywords from text\"\"\"\n",
    "    \n",
    "    def __init__(self):\n",
    "        self.d = 0.85 # damping coefficient, usually is .85\n",
    "        self.min_diff = 1e-5 # convergence threshold\n",
    "        self.steps = 10 # iteration steps\n",
    "        self.node_weight = None # save keywords and its weight\n",
    "\n",
    "    \n",
    "    def set_stopwords(self, stopwords):  \n",
    "        \"\"\"Set stop words\"\"\"\n",
    "        for word in STOP_WORDS.union(set(stopwords)):\n",
    "            lexeme = nlp.vocab[word]\n",
    "            lexeme.is_stop = True\n",
    "    \n",
    "    def sentence_segment(self, doc, candidate_pos, lower):\n",
    "        \"\"\"Store those words only in cadidate_pos\"\"\"\n",
    "        sentences = []\n",
    "        candidate_pos.extend(\"party\")\n",
    "        for sent in doc.sents:\n",
    "            selected_words = []\n",
    "            for token in sent:\n",
    "                # Store words only with cadidate POS tag\n",
    "                if token.pos_ in candidate_pos and token.is_stop is False:\n",
    "                    if lower is True:\n",
    "                        selected_words.append(token.text.lower())\n",
    "                    else:\n",
    "                        selected_words.append(token.text)\n",
    "            sentences.append(selected_words)\n",
    "        return sentences\n",
    "        \n",
    "    def get_vocab(self, sentences):\n",
    "        \"\"\"Get all tokens\"\"\"\n",
    "        vocab = OrderedDict()\n",
    "        i = 0\n",
    "        for sentence in sentences:\n",
    "            for word in sentence:\n",
    "                if word not in vocab:\n",
    "                    vocab[word] = i\n",
    "                    i += 1\n",
    "        return vocab\n",
    "    \n",
    "    def get_token_pairs(self, window_size, sentences):\n",
    "        \"\"\"Build token_pairs from windows in sentences\"\"\"\n",
    "        token_pairs = list()\n",
    "        for sentence in sentences:\n",
    "            for i, word in enumerate(sentence):\n",
    "                for j in range(i+1, i+window_size):\n",
    "                    if j >= len(sentence):\n",
    "                        break\n",
    "                    pair = (word, sentence[j])\n",
    "                    if pair not in token_pairs:\n",
    "                        token_pairs.append(pair)\n",
    "        return token_pairs\n",
    "        \n",
    "    def symmetrize(self, a):\n",
    "        return a + a.T - np.diag(a.diagonal())\n",
    "    \n",
    "    def get_matrix(self, vocab, token_pairs):\n",
    "        \"\"\"Get normalized matrix\"\"\"\n",
    "        # Build matrix\n",
    "        vocab_size = len(vocab)\n",
    "        g = np.zeros((vocab_size, vocab_size), dtype='float')\n",
    "        for word1, word2 in token_pairs:\n",
    "            i, j = vocab[word1], vocab[word2]\n",
    "            g[i][j] = 1\n",
    "            \n",
    "        # Get Symmeric matrix\n",
    "        g = self.symmetrize(g)\n",
    "        \n",
    "        # Normalize matrix by column\n",
    "        norm = np.sum(g, axis=0)\n",
    "        g_norm = np.divide(g, norm, where=norm!=0) # this is ignore the 0 element in norm\n",
    "        \n",
    "        return g_norm\n",
    "\n",
    "    \n",
    "    def get_keywords(self, number=10):\n",
    "        \"\"\"Print top number keywords\"\"\"\n",
    "        retList = []\n",
    "        node_weight = OrderedDict(sorted(self.node_weight.items(), key=lambda t: t[1], reverse=True))\n",
    "        for i, (key, value) in enumerate(node_weight.items()):\n",
    "            #print(key + ' - ' + str(value))\n",
    "            retList.append(key)\n",
    "            if i > number:\n",
    "                break     \n",
    "        return retList\n",
    "        \n",
    "    def analyze(self, text, \n",
    "                candidate_pos=['NOUN', 'PROPN'], \n",
    "                window_size=4, lower=False, stopwords=list()):\n",
    "        \"\"\"Main function to analyze text\"\"\"\n",
    "        \n",
    "        # Set stop words\n",
    "        self.set_stopwords(stopwords)\n",
    "        \n",
    "        # Pare text by spaCy\n",
    "        doc = nlp(text)\n",
    "        \n",
    "        # Filter sentences\n",
    "        sentences = self.sentence_segment(doc, candidate_pos, lower) # list of list of words\n",
    "        \n",
    "        # Build vocabulary\n",
    "        vocab = self.get_vocab(sentences)\n",
    "        \n",
    "        # Get token_pairs from windows\n",
    "        token_pairs = self.get_token_pairs(window_size, sentences)\n",
    "        \n",
    "        # Get normalized matrix\n",
    "        g = self.get_matrix(vocab, token_pairs)\n",
    "        \n",
    "        # Initionlization for weight(pagerank value)\n",
    "        pr = np.array([1] * len(vocab))\n",
    "        \n",
    "        # Iteration\n",
    "        previous_pr = 0\n",
    "        for epoch in range(self.steps):\n",
    "            pr = (1-self.d) + self.d * np.dot(g, pr)\n",
    "            if abs(previous_pr - sum(pr))  < self.min_diff:\n",
    "                break\n",
    "            else:\n",
    "                previous_pr = sum(pr)\n",
    "\n",
    "        # Get weight for each node\n",
    "        node_weight = dict()\n",
    "        for word, index in vocab.items():\n",
    "            node_weight[word] = pr[index]\n",
    "        \n",
    "        self.node_weight = node_weight\n",
    "        \n",
    "    def getCoref(self, text):\n",
    "        doc = nlp(text)\n",
    "        text = doc._.coref_resolved \n",
    "        return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getData(text): \n",
    "      \n",
    "    retVal = {}\n",
    "    #Text Rank KeyWords\n",
    "    tr4w = TextRank4Keyword()\n",
    "    text = tr4w.getCoref(text)\n",
    "    tr4w.analyze(text, candidate_pos = ['NOUN', 'PROPN'], window_size=4, lower=False)\n",
    "    keyWords = tr4w.get_keywords(7)\n",
    "    retVal.update({'textrank':keyWords})\n",
    "    #retVal['textrank']\n",
    "    print('textrank done')\n",
    "    #Split into sentences\n",
    "    sent_text = nltk.sent_tokenize(text)\n",
    "    for sentence in sent_text:\n",
    "        data = {}\n",
    "        data[\"sentence\"] = sentence\n",
    "        inputList.append(data)\n",
    "        \n",
    "    #Semantic Labeling Predector\n",
    "    pred = predictor.predict_batch_json(inputList)\n",
    "    print('prediction done')\n",
    "    \n",
    "    arg0list = []\n",
    "    arg1list = []\n",
    "    argloclist = []\n",
    "    argtmplist = []\n",
    "    #Iterate over each sentence\n",
    "    for x in range(len(pred)):\n",
    "        #Iterate over each verb in each sentence\n",
    "        verbList = pred[x]['verbs']\n",
    "        for verb in verbList:\n",
    "            tags = verb['tags']\n",
    "            #if ('B-ARG1' in tags or 'I-ARG1'in tags) and ('B-ARG0' in tags or 'I-ARG0' in tags):\n",
    "            if ('B-ARG1' in tags or 'I-ARG1'in tags) and ('B-ARG0' in tags or 'I-ARG0' in tags):\n",
    "                descr = verb['description']\n",
    "                if 'ARG0' in descr:\n",
    "                    s = descr.find('ARG0')+5\n",
    "                    d = descr.find(']')\n",
    "                    arg0list.append(descr[s:d])\n",
    "                if 'ARG1' in descr:\n",
    "                    s = descr.find('ARG1')+5\n",
    "                    d = descr.find(']')\n",
    "                    arg1list.append(descr[s:d]) \n",
    "\n",
    "            if 'B-ARGM-LOC' in tags or 'I-ARGM-LOC' in tags:\n",
    "                descr = verb['description']\n",
    "                if 'ARGM-LOC' in descr:\n",
    "                    s = descr.find('ARGM-LOC')+9\n",
    "                    d = descr.find(']')\n",
    "                    argloclist.append(descr[s:d])\n",
    "\n",
    "\n",
    "    print('semantic parsing done')\n",
    "    arg0 = []\n",
    "    for word in arg0list:\n",
    "        w = \"\"\n",
    "        wordlist = word.split()\n",
    "        for x in wordlist:\n",
    "            if x in keyWords:\n",
    "                w += \" \"+x\n",
    "        #print(word)\n",
    "        arg0.extend(w.split())\n",
    "        #print(arg0)\n",
    "        counts = Counter(arg0)\n",
    "    #print(counts)\n",
    "\n",
    "\n",
    "    arg1 = []\n",
    "    for word in arg1list:\n",
    "        w = \"\"\n",
    "        wordlist = word.split()\n",
    "        for x in wordlist:\n",
    "            if x in keyWords:\n",
    "                w += \" \"+x\n",
    "        #print(word)\n",
    "        arg1.extend(w.split())\n",
    "        #print(arg1)\n",
    "        counts.update(arg1)\n",
    "    #print(counts)\n",
    "\n",
    "    argloc = []\n",
    "    for word in argloclist:\n",
    "        word = word.lower()\n",
    "        w = \"\"\n",
    "        wordlist = word.split()\n",
    "        for x in wordlist:\n",
    "            if x not in stop_words:\n",
    "                w += \" \"+x\n",
    "        #print(word)\n",
    "        argloc.extend(w.split())\n",
    "        #print(arg1)\n",
    "        countsLoc = Counter(argloc)\n",
    "    #print(counts)\n",
    "    \n",
    "#     sorted_words = sorted(counts, key: lambda x:-counterlist[x])\n",
    "#     distinct_words_from_list = set(list_to_be_sorted)\n",
    "#     sorted_distinct_list = sorted(distinct_words_from_list, key: lambda x:-counterlist[x])\n",
    "#     sorted_distinct_list = sorted_distinct_list[:10]\n",
    "    print('Trying to add to counter')\n",
    "    print(counts)\n",
    "    counterlist = OrderedCounter(counts)\n",
    "    counterlist = counterlist.keys()\n",
    "    counterlist = list(counterlist)\n",
    "    counterlist = counterlist[:5]\n",
    "    retVal.update({'semanticLabeling':counterlist})\n",
    "    \n",
    "    counterlist = OrderedCounter(countsLoc)\n",
    "    counterlist = counterlist.keys()\n",
    "    counterlist = list(counterlist)\n",
    "    counterlist = counterlist[:5]\n",
    "    retVal.update({'semanticLoc':counterlist})\n",
    "    print('Added successfully to counter')\n",
    "    return retVal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The announcement of tickets by the BJP has led to discontent among a section of its workers who burnt effigies of former MP Sangita Singh Deo and her husband and BJP Legislature Party Leader KV Singh Deo. After the party preferred a turncoat, Ashok Pujari, who recently came from the Congress to the BJP and who is uncle of BJD candidate Niranjan Pujari for the Sonepur Assembly seat, and Raghunath Jagdala, an officer of the Merchant Navy, who also came to BJP 20 days ago for the Birmaharajpur Seat, supporters of BJP leaders Baldev Bedbak and Anand Barik, who were seeking tickets for the two seats, respectively, held an emergency meeting and charged the party high command for ignoring dedicated workers and giving preference to relatives of rival candidates. More than 450 workers burnt effigies of Singh Deo couple at Sonepur bus stand on Sunday, alleging that KV had taken over Rs 2 crore from opposition parties. The auudio has been also viral in the social media in which someone is giving information of this money deal with another BJP leader. They also shouted slogans such as KV Singh Deo Murdabad, Sangita Singh Deo Murdabad . A party worker said Bedbak had prepared to be a BJP candidate for Sonepur Assembly seat for the last two years.\n",
      "BJP - 5.666063134592135\n",
      "Singh - 1.81190087736876\n",
      "Deo - 1.7980073756463106\n",
      "Sonepur - 1.768185538260088\n",
      "KV - 1.6763561925431483\n",
      "workers - 1.661925703052765\n",
      "Bedbak - 1.4498155997146132\n",
      "Party - 1.4381481256200104\n",
      "Pujari - 1.4203767897124804\n",
      "[' the BJP workers', ' BJP Legislature Party', ' the BJP leaders Baldev Bedbak and Anand Barik', '', '', ' BJP Legislature Party high command', ' BJP Legislature Party high command', ' More than 450 workers', ' More than 450 workers', ' KV', '', ' the BJP', ' A party worker', ' Candidates of three major political parties , BJD , Congress and BJP , and some Independent nominees', ' The silk city , particularly the Sub Collector s office area', ' Candidates of three major political parties , BJD , Congress and BJP , and some Independent nominees', '', ' BJP s Bibhuti Jena , Kahnu Charan Pati , Pinki Pradhan', ' Ambedkar Congress Gobinda Sahu', '', ' Senior scribe and social worker Samarjit Mohanty', '', ' rebels', ' parties', ' former contractor Bhagirathi Sethy', '', ' Senior Congress leader Jayadev Jena who is the Congress nominee from Anandpur', ' Senior Congress leader Jayadev Jena who is the Congress nominee from Anandpur', ' Prithviraj Kuanr , the former Congress MLA nominee of Ghasipura', ' Prithviraj Kuanr , the former Congress MLA nominee of Ghasipura', '', ' other BJP aspirants', '', ' BJD leader Patra', ' BJD leader Patra', '', ' other party aspirants like Sidhu Naik', ' Declaration of senior BJP leader Mohan Majhi from Keonjhar Assembly seat', ' who', ' Akhila Naik', ' who', ' The supporters of Akhila Naik , who joined BJP recently aspiring for Keonjhar Assembly ticket', ' The supporters of Akhila Naik , who joined BJP recently aspiring for Keonjhar Assembly ticket', ' Former Congress MLA from Champua Dhanurjay Sidhu', ' Former Congress MLA from Champua Dhanurjay Sidhu', ' BJP MP candidate Ananta Nayak', ' Some leaders', ' Some leaders', '', ' A large number of women and men led by former BJD Khordha town president Jitendra Kumar Pradhan', ' The activists', ' The activists', ' The activists holding placards and shouting slogans against the MLA', ' many dissident candidate from the Khordha BJD', '', ' supporters of the local MLA', '', ' MLA supporters', ' Those', ' the BJP workers', ' BJP Legislature Party', ' the BJP leaders Baldev Bedbak and Anand Barik', '', '', ' BJP Legislature Party high command', ' BJP Legislature Party high command', ' More than 450 workers', ' More than 450 workers', ' KV', '', ' the BJP', ' A party worker', ' Candidates of three major political parties , BJD , Congress and BJP , and some Independent nominees', ' The silk city , particularly the Sub Collector s office area ,', ' Candidates of three major political parties , BJD , Congress and BJP , and some Independent nominees', '', ' BJP s Bibhuti Jena , Kahnu Charan Pati , Pinki Pradhan', ' Ambedkar Congress Gobinda Sahu', '', ' Senior scribe and social worker Samarjit Mohanty', '', ' rebels', ' parties', ' former contractor Bhagirathi Sethy', '', ' Senior Congress leader Jayadev Jena who is the Congress nominee from Anandpur', ' Senior Congress leader Jayadev Jena who is the Congress nominee from Anandpur', ' Prithviraj Kuanr , the former Congress MLA nominee of Ghasipura', ' Prithviraj Kuanr , the former Congress MLA nominee of Ghasipura', '', ' other BJP aspirants', '', ' BJD leader Patra', ' BJD leader Patra', '', ' other party aspirants like Sidhu Naik', ' Declaration of senior BJP leader Mohan Majhi from Keonjhar Assembly seat', ' who', ' Akhila Naik', ' who', ' The supporters of Akhila Naik , who joined BJP recently aspiring for Keonjhar Assembly ticket', ' The supporters of Akhila Naik , who joined BJP recently aspiring for Keonjhar Assembly ticket', ' Former Congress MLA from Champua Dhanurjay Sidhu', ' Former Congress MLA from Champua Dhanurjay Sidhu', ' BJP MP candidate Ananta Nayak', ' Some leaders', ' Some leaders', '', ' A large number of women and men led by former BJD Khordha town president Jitendra Kumar Pradhan', ' The activists', ' The activists', ' The activists holding placards and shouting slogans against the MLA', ' many dissident candidate from the Khordha BJD', '', ' supporters of the local MLA', '', ' MLA supporters', ' Those', ' the BJP workers', ' BJP Legislature Party', ' the BJP leaders Baldev Bedbak and Anand Barik', '', '', ' BJP Legislature Party high command', ' BJP Legislature Party high command', ' More than 450 workers', ' More than 450 workers', ' KV', '', ' the BJP', ' A party worker']\n",
      "['', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', ' The former BJD MLA of Anandpur and former contractor Bhagirathi Sethy who joined BJP', '', '', '', '', '', '', ' But other BJP aspirants are planning to sabotage Prithvi s prospects', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', ' A large number of women and men', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', ' The former BJD MLA of Anandpur and former contractor Bhagirathi Sethy who joined BJP', '', '', '', '', '', '', ' But other BJP aspirants are planning to sabotage Prithvi s prospects', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', ' A large number of women and men', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '']\n",
      "semantic parsing done\n",
      "Candidates of three major political parties, BJD, Congress and BJP, and some Independent nominees filed nominations for different Assembly seats at the Sub Collectors office here on Monday, the last day of filing of nominations for the scheduled first phase polling. The silk city, particularly the Sub Collector s office area, witnessed unprecedented crowd as the candidates came in processions with thousands of supporters to file papers for their respective constituencies. The traffic in the city remained crowded till afternoon. According to information, BJD s Nandini Devi, Surjya Narayan Patro and Bikram Panda filed nominations for Sanakhemundi, Digapahandi and Berhampur, respectively, BJP s Bibhuti Jena, Kahnu Charan Pati, Pinki Pradhan filed papers for Gopalpur, Berhampur and Digapahandi, respectively, Congress Ramesh Jena and Lingaraj Choudhury for Sanakhemundi and Berhampur, respectively and Ambedkar Congress Gobinda Sahu filed papers for Berhampur constituency. Besides, after visiting Lord Jagannath Temple at Khallikote, one Suryamani Baidya filed her nomination in the office of Additional Collector at Chhatrapur for Khallkote constituency. Senior scribe and social worker Samarjit Mohanty reportedly filed papers for the Polasara constituency as a candidate of the All India Forward Bloc.\n",
      "Berhampur - 2.686373126103258\n",
      "papers - 2.3538562909368888\n",
      "office - 2.2520410752259434\n",
      "nominations - 2.1236501384509308\n",
      "Congress - 2.0761004874308138\n",
      "BJP - 1.734775976301211\n",
      "Collector - 1.6546737759941132\n",
      "Jena - 1.5297377648580226\n",
      "BJD - 1.481652455052553\n"
     ]
    }
   ],
   "source": [
    "\n",
    "counter = 1\n",
    "with open('/home/raj/UB_Stuff/CSE_635/Phase5/Semantic_word_labelling/SRL_vs_TLDR.csv','r') as csvinput:\n",
    "    with open('/home/raj/UB_Stuff/CSE_635/Phase5/Semantic_word_labelling/test.csv', 'w') as csvoutput:\n",
    "        writer = csv.writer(csvoutput, lineterminator='\\n')\n",
    "        reader = csv.reader(csvinput)\n",
    "\n",
    "        all = []\n",
    "        row = next(reader)\n",
    "        row.append('Text Rank')\n",
    "        row.append('Semantic Text Rank')\n",
    "        row.append('Semantic Loc')\n",
    "        all.append(row)\n",
    "\n",
    "        for row in reader:\n",
    "            try:\n",
    "                if(counter == 6) :\n",
    "                    break\n",
    "                searchTerm = row[14]\n",
    "                print(searchTerm)\n",
    "                searchTerm = searchTerm.replace('\\n',' ')\n",
    "                #Get data\n",
    "                data = getData2(searchTerm)        \n",
    "                row.append(data['textrank'])\n",
    "                row.append(data['semanticLabeling'])\n",
    "                row.append(data['semanticLoc'])\n",
    "                all.append(row)\n",
    "                counter+=1\n",
    "            except Exception as e:\n",
    "                print(e)\n",
    "                pass\n",
    "        writer.writerows(all)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getData2(text): \n",
    "      \n",
    "    retVal = {}\n",
    "    #Text Rank KeyWords\n",
    "    tr4w = TextRank4Keyword()\n",
    "    text = tr4w.getCoref(text)\n",
    "    tr4w.analyze(text, candidate_pos = ['NOUN', 'PROPN'], window_size=4, lower=False)\n",
    "    keyWords = tr4w.get_keywords(7)\n",
    "    retVal.update({'textrank':keyWords})\n",
    "    #retVal['textrank']\n",
    "    #print('textrank done')\n",
    "    #Split into sentences\n",
    "    sent_text = nltk.sent_tokenize(text)\n",
    "    for sentence in sent_text:\n",
    "        data = {}\n",
    "        data[\"sentence\"] = sentence\n",
    "        inputList.append(data)\n",
    "        \n",
    "    #Semantic Labeling Predector\n",
    "    pred = predictor.predict_batch_json(inputList)\n",
    "    #print('prediction done')\n",
    "    \n",
    "    arg0list = []\n",
    "    arg1list = []\n",
    "    argloclist = []\n",
    "    argtmplist = []\n",
    "    #Iterate over each sentence\n",
    "    for x in range(len(pred)):\n",
    "        #Iterate over each verb in each sentence\n",
    "        verbList = pred[x]['verbs']\n",
    "        for verb in verbList:\n",
    "            tags = verb['tags']\n",
    "            #if ('B-ARG1' in tags or 'I-ARG1'in tags) and ('B-ARG0' in tags or 'I-ARG0' in tags):\n",
    "            if ('B-ARG1' in tags or 'I-ARG1'in tags) and ('B-ARG0' in tags or 'I-ARG0' in tags):\n",
    "                descr = verb['description']\n",
    "                if 'ARG0' in descr:\n",
    "                    s = descr.find('ARG0')+5\n",
    "                    d = descr.find(']')\n",
    "                    arg0list.append(descr[s:d])\n",
    "                if 'ARG1' in descr:\n",
    "                    s = descr.find('ARG1')+5\n",
    "                    d = descr.find(']')\n",
    "                    arg1list.append(descr[s:d]) \n",
    "\n",
    "            if 'B-ARGM-LOC' in tags or 'I-ARGM-LOC' in tags:\n",
    "                descr = verb['description']\n",
    "                if 'ARGM-LOC' in descr:\n",
    "                    s = descr.find('ARGM-LOC')+9\n",
    "                    d = descr.find(']')\n",
    "                    argloclist.append(descr[s:d])\n",
    "\n",
    "\n",
    "    print('semantic parsing done')\n",
    "\n",
    "    arg0 = []\n",
    "    for word in arg0list:\n",
    "        w = \"\"\n",
    "        wordlist = word.split()\n",
    "        for x in wordlist:\n",
    "                w += \" \"+x\n",
    "        #print(word)\n",
    "        arg0.extend(w.split())\n",
    "        #print(arg0)\n",
    "        counts = Counter(arg0)\n",
    "    #print(counts)\n",
    "\n",
    "\n",
    "    arg1 = []\n",
    "    for word in arg1list:\n",
    "        w = \"\"\n",
    "        wordlist = word.split()\n",
    "        for x in wordlist:\n",
    "                w += \" \"+x\n",
    "        #print(word)\n",
    "        arg1.extend(w.split())\n",
    "        #print(arg1)\n",
    "        counts.update(arg1)\n",
    "    print(counts)\n",
    "\n",
    "    argloc = []\n",
    "    for word in argloclist:\n",
    "        word = word.lower()\n",
    "        w = \"\"\n",
    "        wordlist = word.split()\n",
    "        for x in wordlist:\n",
    "            if x not in stop_words:\n",
    "                w += \" \"+x\n",
    "        #print(word)\n",
    "        argloc.extend(w.split())\n",
    "        #print(arg1)\n",
    "        countsLoc = Counter(argloc)\n",
    "    #print(counts)\n",
    "    \n",
    "#     sorted_words = sorted(counts, key: lambda x:-counterlist[x])\n",
    "#     distinct_words_from_list = set(list_to_be_sorted)\n",
    "#     sorted_distinct_list = sorted(distinct_words_from_list, key: lambda x:-counterlist[x])\n",
    "#     sorted_distinct_list = sorted_distinct_list[:10]\n",
    "#     print('Trying to add to counter')\n",
    "#     print(counts)\n",
    "    counterlist = OrderedCounter(counts)\n",
    "    counterlist = counterlist.keys()\n",
    "    counterlist = list(counterlist)\n",
    "    counterlist = counterlist[:7]\n",
    "    retVal.update({'semanticLabeling':counterlist})\n",
    "    \n",
    "    counterlist = OrderedCounter(countsLoc)\n",
    "    counterlist = counterlist.keys()\n",
    "    counterlist = list(counterlist)\n",
    "    counterlist = counterlist[:5]\n",
    "    retVal.update({'semanticLoc':counterlist})\n",
    "#     print('Added successfully to counter')\n",
    "    return retVal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "semantic parsing done\n",
      "Counter({'supporters': 2, 'MLA': 2, 'of': 1, 'the': 1, 'local': 1, 'Those': 1})\n"
     ]
    }
   ],
   "source": [
    "\n",
    "text = r'As the elections nearing, the political scenario in Begunia block of Khordha disdtrict seems to be getting warmer by the day. On Monday, two groups of the BJD organised political gatherings at two different places under the block on the plea of a protest meeting and a Mahashanti Yajna , respectively. This has been a subject of discussion among local people now. While supporters of the local MLA held a protest meeting in the Begunia Mini Stadium in protest against the ransacking of the police station by some rivals of the MLA a week back, former Minister and Rajya Sabha Member Prashant Nanda s son Rushabh along with thousands of his supporters from about 40 gram panchayats attended the Mahashanti Yajna at Brahmeswar Peeth in Lakhanpur village. MLA supporters had come in a rally from Sarua to reach the mini stadium. Among others, BYJD Begunia president Bibhuti Mohanty, BYJD Bolgarh president Rabindra Natha Subuddhi, Bolgarh block president Gopal Behera attended. Those who attended Rushabh s meeting included Prabhat Kumar Maharaj, former Sarpanch Chaitanya Jaysingh, Krushna Chandra Ranabijuli, Hemant Sundray, Surendra Behera, Sangramkeshari Mishra, Dillip Mahabhoi and Jalandhar Mohanty.'\n",
    "res = getData2(text)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'type'>\n"
     ]
    }
   ],
   "source": [
    "print(type(Counter))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}