topic_modelling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Topic Modelling for 400 news articles from India - March to April 2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0,\n",
      "  '0.026*\"cpi\" + 0.023*\"open\" + 0.019*\"office\" + 0.019*\"bjp\" + '\n",
      "  '0.019*\"kamalpur\" + 0.019*\"jitendra\" + 0.015*\"attack\" + 0.015*\"chaudhury\" + '\n",
      "  '0.015*\"clash\" + 0.012*\"administration\"'),\n",
      " (1,\n",
      "  '0.037*\"congress\" + 0.019*\"bhawan\" + 0.019*\"bhubaneswar\" + 0.019*\"odisha\" + '\n",
      "  '0.013*\"ticket\" + 0.013*\"today\" + 0.013*\"capital\" + 0.013*\"mohammad\" + '\n",
      "  '0.013*\"arrest\" + 0.008*\"protest\"'),\n",
      " (2,\n",
      "  '0.020*\"group\" + 0.020*\"house\" + 0.016*\"person\" + 0.016*\"afiz\" + '\n",
      "  '0.016*\"rizul\" + 0.012*\"incident\" + 0.012*\"march\" + 0.012*\"family\" + '\n",
      "  '0.012*\"village\" + 0.012*\"set\"'),\n",
      " (3,\n",
      "  '0.041*\"student\" + 0.030*\"school\" + 0.024*\"staff\" + 0.024*\"protest\" + '\n",
      "  '0.024*\"teaching\" + 0.019*\"gandoh\" + 0.019*\"shortage\" + 0.019*\"education\" + '\n",
      "  '0.013*\"due\" + 0.013*\"demand\"')]\n",
      "\n",
      "Perplexity:  -5.839059734469338\n",
      "\n",
      "Coherence Score:  0.7070388548923625\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "<link rel=\"stylesheet\" type=\"text/css\" href=\"https://cdn.rawgit.com/bmabey/pyLDAvis/files/ldavis.v1.0.0.css\">\n",
       "\n",
       "\n",
       "<div id=\"ldavis_el770221121075616322196539872\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "\n",
       "var ldavis_el770221121075616322196539872_data = {\"mdsDat\": {\"x\": [-0.13826094777669315, -0.0019040518859566383, 0.08858575820604414, 0.05157924145660568], \"y\": [-0.05343389626655058, 0.13018732964207652, -0.08595928818484772, 0.00920585480932177], \"topics\": [1, 2, 3, 4], \"cluster\": [1, 1, 1, 1], \"Freq\": [35.335426330566406, 32.96476364135742, 16.578336715698242, 15.121478080749512]}, \"tinfo\": {\"Category\": [\"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\"], \"Freq\": [4.0, 3.0, 3.0, 2.0, 2.0, 5.0, 3.0, 4.0, 2.0, 2.0, 3.0, 2.0, 2.0, 2.0, 4.0, 4.0, 4.0, 4.0, 1.0, 1.0, 1.0, 3.0, 3.0, 2.0, 1.0, 1.0, 1.0, 1.0, 3.0, 2.0, 5.064811706542969, 4.365364074707031, 3.6675288677215576, 3.6663880348205566, 3.6661055088043213, 2.9702372550964355, 2.9681010246276855, 2.2694294452667236, 2.2676198482513428, 1.5721900463104248, 1.5719215869903564, 1.5718172788619995, 1.5717706680297852, 1.5715830326080322, 1.5714741945266724, 1.5713318586349487, 1.571312665939331, 1.5713038444519043, 1.5712252855300903, 1.570117712020874, 1.56936514377594, 3.669041395187378, 2.2707200050354004, 0.87386155128479, 0.873693585395813, 0.8736653923988342, 0.8736537098884583, 0.8736352920532227, 0.8736161589622498, 0.8735970854759216, 2.2710154056549072, 2.9664971828460693, 1.5717214345932007, 1.5703864097595215, 1.571768045425415, 1.5687103271484375, 1.5723285675048828, 2.2699453830718994, 2.2698676586151123, 1.5739269256591797, 1.5733355283737183, 1.571829915046692, 1.5717699527740479, 1.5714272260665894, 3.5826284885406494, 2.900811195373535, 2.8983609676361084, 2.218484878540039, 2.2178590297698975, 2.2153713703155518, 1.536057710647583, 1.5355829000473022, 1.535573959350586, 1.5353550910949707, 1.5353132486343384, 1.5352602005004883, 1.535394549369812, 1.5351437330245972, 1.5343538522720337, 1.534287691116333, 0.8537455797195435, 0.8537108302116394, 0.8537089824676514, 0.853605329990387, 0.8535996079444885, 0.8536329865455627, 0.8535961508750916, 0.8535935282707214, 0.8535756468772888, 0.8534993529319763, 0.8533843159675598, 0.853531539440155, 0.8535118699073792, 0.853486180305481, 2.9008820056915283, 3.585423231124878, 1.5361928939819336, 2.2179741859436035, 1.5360056161880493, 1.5359032154083252, 2.2217800617218018, 2.2209653854370117, 1.5409502983093262, 1.5369046926498413, 0.8607217669487, 0.8584689497947693, 0.857513427734375, 0.8562625050544739, 3.7708613872528076, 2.7299811840057373, 2.210299253463745, 2.2087042331695557, 1.6904078722000122, 1.6892198324203491, 1.6888940334320068, 1.1702498197555542, 1.169810175895691, 1.1697430610656738, 1.1696876287460327, 2.2092337608337402, 0.6504207253456116, 0.6504136919975281, 0.6503692269325256, 0.6503757834434509, 0.650332510471344, 0.6503024101257324, 0.6502953767776489, 0.6502783894538879, 0.6502588391304016, 0.6502440571784973, 0.6502374410629272, 0.6502270698547363, 0.6501731872558594, 0.6501704454421997, 0.6501390933990479, 0.6501350402832031, 0.6501644253730774, 0.6501296758651733, 1.17135488986969, 1.1740047931671143, 0.6539492011070251, 0.6532271504402161, 0.6519646048545837, 0.6518409252166748, 0.6518089771270752, 0.6517742276191711, 0.6515160799026489, 0.6507630944252014, 0.6506019234657288, 3.107651472091675, 1.616083025932312, 1.6160777807235718, 1.1191939115524292, 1.1185957193374634, 1.1184263229370117, 0.6220089793205261, 0.6219949126243591, 0.6219794750213623, 0.621955931186676, 0.6218544244766235, 0.6218544840812683, 0.6218294501304626, 0.6218284368515015, 0.6218250393867493, 0.6217892169952393, 0.6217780113220215, 0.6217573881149292, 0.6217259168624878, 0.621720552444458, 0.6217184662818909, 0.6217111349105835, 0.621709942817688, 0.6217063665390015, 0.6216843128204346, 0.6216616034507751, 0.6216045022010803, 0.6216000914573669, 0.6215986013412476, 0.6215957999229431, 1.6157370805740356, 1.1182674169540405, 1.1191601753234863, 0.6238483786582947, 0.6230046153068542, 0.6227862238883972, 0.6226629018783569, 0.6225711703300476, 0.6222305297851562, 0.6221396923065186, 0.6220149397850037, 0.6219995021820068, 0.6218799352645874], \"Term\": [\"student\", \"congress\", \"school\", \"staff\", \"teaching\", \"cpi\", \"protest\", \"open\", \"bhawan\", \"bhubaneswar\", \"odisha\", \"gandoh\", \"shortage\", \"education\", \"house\", \"bjp\", \"kamalpur\", \"jitendra\", \"ticket\", \"capital\", \"mohammad\", \"afiz\", \"rizul\", \"arrest\", \"government\", \"doda\", \"department\", \"adequate\", \"due\", \"demand\", \"cpi\", \"open\", \"bjp\", \"kamalpur\", \"jitendra\", \"attack\", \"chaudhury\", \"come\", \"go\", \"tiwn\", \"hooligan\", \"goon\", \"mute\", \"supporter\", \"dhalai\", \"successfully\", \"spectator\", \"remain\", \"lead\", \"platoon\", \"koraput\", \"office\", \"party\", \"manikbhander\", \"massive\", \"hit\", \"center\", \"commission\", \"ahead\", \"however\", \"administration\", \"clash\", \"election\", \"source\", \"hold\", \"area\", \"conduct\", \"march\", \"group\", \"incident\", \"village\", \"injure\", \"reach\", \"today\", \"house\", \"afiz\", \"rizul\", \"family\", \"set\", \"death\", \"jarada\", \"dispute\", \"ablaze\", \"rd\", \"vandalize\", \"stab\", \"ganjam\", \"injury\", \"ali\", \"kamrup\", \"arson\", \"thatched\", \"dig\", \"southern\", \"flee\", \"rival\", \"male\", \"provocation\", \"bomb\", \"provide\", \"identify\", \"maintain\", \"undergo\", \"camp\", \"person\", \"group\", \"accuse\", \"village\", \"mob\", \"yesterday\", \"incident\", \"march\", \"odisha\", \"clash\", \"today\", \"office\", \"arrest\", \"mar\", \"student\", \"school\", \"staff\", \"teaching\", \"gandoh\", \"shortage\", \"education\", \"government\", \"doda\", \"department\", \"adequate\", \"protest\", \"pritam\", \"movement\", \"thapa\", \"class\", \"remote\", \"add\", \"tilogra\", \"assurance\", \"sdm\", \"slogans\", \"study\", \"suffer\", \"pursue\", \"correspondent\", \"high\", \"malikpura\", \"equally\", \"raise\", \"demand\", \"due\", \"area\", \"spot\", \"reach\", \"disperse\", \"take\", \"vehicular\", \"today\", \"sub\", \"mar\", \"congress\", \"bhawan\", \"bhubaneswar\", \"ticket\", \"capital\", \"mohammad\", \"premise\", \"khanm\", \"gate\", \"amir\", \"sajir\", \"allot\", \"ekamra\", \"assembly\", \"file\", \"aspirant\", \"commissionerate\", \"impending\", \"hadaqat\", \"poll\", \"individual\", \"vandalism\", \"rahman\", \"forward\", \"ranjan\", \"vandalise\", \"tafiq\", \"video\", \"khan\", \"damage\", \"odisha\", \"arrest\", \"today\", \"protest\", \"accuse\", \"person\", \"take\", \"incident\", \"report\", \"later\", \"connection\", \"mar\", \"follow\"], \"Total\": [4.0, 3.0, 3.0, 2.0, 2.0, 5.0, 3.0, 4.0, 2.0, 2.0, 3.0, 2.0, 2.0, 2.0, 4.0, 4.0, 4.0, 4.0, 1.0, 1.0, 1.0, 3.0, 3.0, 2.0, 1.0, 1.0, 1.0, 1.0, 3.0, 2.0, 5.499584197998047, 4.799347400665283, 4.099930763244629, 4.099906921386719, 4.09990119934082, 3.4001381397247314, 3.4000892639160156, 2.700260639190674, 2.699069023132324, 2.0004684925079346, 2.0004608631134033, 2.000459909439087, 2.000458240509033, 2.0004518032073975, 2.0004515647888184, 2.0004467964172363, 2.0004465579986572, 2.0004475116729736, 2.0004444122314453, 1.999490737915039, 1.9993470907211304, 4.78364896774292, 3.1989874839782715, 1.3006469011306763, 1.300642728805542, 1.3006420135498047, 1.300641655921936, 1.3006411790847778, 1.3006407022476196, 1.3006396293640137, 3.3837273120880127, 4.765855312347412, 2.4990992546081543, 2.498659610748291, 2.5217602252960205, 2.5203487873077393, 2.683703899383545, 4.750968933105469, 6.1183247566223145, 4.550613880157471, 4.051259994506836, 2.6838042736053467, 3.204760789871216, 4.20282506942749, 4.020312309265137, 3.336129665374756, 3.335583209991455, 2.652509927749634, 2.6525073051452637, 2.6518218517303467, 1.9688758850097656, 1.9684076309204102, 1.9685460329055786, 1.9683128595352173, 1.9683561325073242, 1.9683986902236938, 1.968775749206543, 1.96859872341156, 1.96818208694458, 1.9682159423828125, 1.2848573923110962, 1.2848490476608276, 1.2848548889160156, 1.2848076820373535, 1.2848092317581177, 1.2848635911941528, 1.284813642501831, 1.284820318222046, 1.2848594188690186, 1.2847765684127808, 1.2846343517303467, 1.2848589420318604, 1.2848447561264038, 1.2848100662231445, 4.534860610961914, 6.1183247566223145, 2.4668819904327393, 4.051259994506836, 2.667679786682129, 2.6680617332458496, 4.550613880157471, 4.750968933105469, 3.465036153793335, 4.765855312347412, 4.20282506942749, 4.78364896774292, 2.28240704536438, 2.3048713207244873, 4.249675273895264, 3.2070181369781494, 2.6858646869659424, 2.6857359409332275, 2.1644694805145264, 2.1645991802215576, 2.164640426635742, 1.643316626548767, 1.6433101892471313, 1.6433792114257812, 1.6433519124984741, 3.1848769187927246, 1.1220619678497314, 1.122076153755188, 1.1220325231552124, 1.1220916509628296, 1.1220510005950928, 1.1220479011535645, 1.122046947479248, 1.1220580339431763, 1.1220752000808716, 1.122050404548645, 1.1220444440841675, 1.1220712661743164, 1.122101902961731, 1.122098445892334, 1.122054100036621, 1.12205171585083, 1.1221085786819458, 1.1220636367797852, 2.3268892765045166, 3.0415256023406982, 2.5203487873077393, 3.2044551372528076, 3.204760789871216, 1.8212724924087524, 2.319859266281128, 1.8212565183639526, 4.20282506942749, 1.8058477640151978, 2.3048713207244873, 3.5925917625427246, 2.0967164039611816, 2.096714973449707, 1.5979825258255005, 1.5982023477554321, 1.598257064819336, 1.0993483066558838, 1.0993596315383911, 1.0993646383285522, 1.0993744134902954, 1.0994107723236084, 1.0994125604629517, 1.0994197130203247, 1.0994184017181396, 1.099424958229065, 1.0994374752044678, 1.099441647529602, 1.0994422435760498, 1.0994641780853271, 1.099458932876587, 1.0994588136672974, 1.0994666814804077, 1.0994679927825928, 1.0994651317596436, 1.0994733572006226, 1.0994863510131836, 1.0995020866394043, 1.099502682685852, 1.0995081663131714, 1.099505066871643, 3.465036153793335, 2.28240704536438, 4.20282506942749, 3.1848769187927246, 2.4668819904327393, 4.534860610961914, 2.319859266281128, 4.550613880157471, 2.4830403327941895, 1.6207998991012573, 1.783416509628296, 2.3048713207244873, 1.7987163066864014], \"loglift\": [30.0, 29.0, 28.0, 27.0, 26.0, 25.0, 24.0, 23.0, 22.0, 21.0, 20.0, 19.0, 18.0, 17.0, 16.0, 15.0, 14.0, 13.0, 12.0, 11.0, 10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 0.9578999876976013, 0.9455000162124634, 0.9287999868392944, 0.9284999966621399, 0.9284999966621399, 0.9050999879837036, 0.9043999910354614, 0.8665000200271606, 0.866100013256073, 0.7993999719619751, 0.7991999983787537, 0.7990999817848206, 0.7990999817848206, 0.7990000247955322, 0.7989000082015991, 0.798799991607666, 0.798799991607666, 0.798799991607666, 0.798799991607666, 0.7985000014305115, 0.7980999946594238, 0.7749999761581421, 0.6974999904632568, 0.6425999999046326, 0.6424000263214111, 0.6424000263214111, 0.6424000263214111, 0.642300009727478, 0.642300009727478, 0.642300009727478, 0.6414999961853027, 0.5662000179290771, 0.5764999985694885, 0.5759000182151794, 0.5674999952316284, 0.566100001335144, 0.5055999755859375, 0.30169999599456787, 0.048700001090765, -0.021400000900030136, 0.09449999779462814, 0.505299985408783, 0.3278000056743622, 0.05649999901652336, 0.9944999814033508, 0.9699000120162964, 0.9692000150680542, 0.9309999942779541, 0.9308000206947327, 0.9298999905586243, 0.8615000247955322, 0.8614000082015991, 0.861299991607666, 0.861299991607666, 0.861299991607666, 0.8611999750137329, 0.8611000180244446, 0.8610000014305115, 0.8607000112533569, 0.8607000112533569, 0.7009999752044678, 0.7009000182151794, 0.7009000182151794, 0.7008000016212463, 0.7008000016212463, 0.7008000016212463, 0.7008000016212463, 0.7008000016212463, 0.7008000016212463, 0.7006999850273132, 0.7006999850273132, 0.7006999850273132, 0.7006999850273132, 0.7006999850273132, 0.6629999876022339, 0.5752999782562256, 0.6360999941825867, 0.5073000192642212, 0.557699978351593, 0.5575000047683716, 0.3928000032901764, 0.34929999709129333, 0.2994000017642975, -0.02199999988079071, -0.47600001096725464, -0.6080999970436096, 0.13079999387264252, 0.11949999630451202, 1.6775000095367432, 1.6360000371932983, 1.6022000312805176, 1.6015000343322754, 1.5499000549316406, 1.5491000413894653, 1.5489000082015991, 1.4575999975204468, 1.457200050354004, 1.4571000337600708, 1.4571000337600708, 1.4313000440597534, 1.2517999410629272, 1.2517000436782837, 1.2517000436782837, 1.2517000436782837, 1.2516000270843506, 1.2516000270843506, 1.2516000270843506, 1.2516000270843506, 1.2515000104904175, 1.2515000104904175, 1.2515000104904175, 1.2515000104904175, 1.2513999938964844, 1.2513999938964844, 1.2512999773025513, 1.2512999773025513, 1.2512999773025513, 1.2512999773025513, 1.110700011253357, 0.8450999855995178, 0.4480000138282776, 0.20669999718666077, 0.20469999313354492, 0.769599974155426, 0.5275999903678894, 0.7695000171661377, -0.06710000336170197, 0.7764000296592712, 0.5321999788284302, 1.74399995803833, 1.6287000179290771, 1.6287000179290771, 1.5328999757766724, 1.5321999788284302, 1.532099962234497, 1.3194999694824219, 1.3194999694824219, 1.3194999694824219, 1.3193999528884888, 1.319200038909912, 1.319200038909912, 1.319200038909912, 1.319200038909912, 1.319200038909912, 1.319100022315979, 1.319100022315979, 1.319000005722046, 1.319000005722046, 1.319000005722046, 1.319000005722046, 1.3188999891281128, 1.3188999891281128, 1.3188999891281128, 1.3188999891281128, 1.3188999891281128, 1.3186999559402466, 1.3186999559402466, 1.3186999559402466, 1.3186999559402466, 1.126099944114685, 1.1756000518798828, 0.5659000277519226, 0.2587999999523163, 0.5128999948501587, -0.09629999846220016, 0.5738000273704529, -0.10010000318288803, 0.5051000118255615, 0.9315000176429749, 0.8356999754905701, 0.579200029373169, 0.8270000219345093], \"logprob\": [30.0, 29.0, 28.0, 27.0, 26.0, 25.0, 24.0, 23.0, 22.0, 21.0, 20.0, 19.0, 18.0, 17.0, 16.0, 15.0, 14.0, 13.0, 12.0, 11.0, 10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, -3.6454999446868896, -3.794100046157837, -3.9683001041412354, -3.968600034713745, -3.9686999320983887, -4.179200172424316, -4.179900169372559, -4.448299884796143, -4.449100017547607, -4.815299987792969, -4.815499782562256, -4.8155999183654785, -4.8155999183654785, -4.815700054168701, -4.815800189971924, -4.815899848937988, -4.815899848937988, -4.815899848937988, -4.815999984741211, -4.816699981689453, -4.8171000480651855, -3.967900037765503, -4.447700023651123, -5.402599811553955, -5.4028000831604, -5.402900218963623, -5.402900218963623, -5.402900218963623, -5.402900218963623, -5.4029998779296875, -4.4475998878479, -4.1803998947143555, -4.8155999183654785, -4.816500186920166, -4.8155999183654785, -4.817599773406982, -4.815299987792969, -4.4481000900268555, -4.4481000900268555, -4.814199924468994, -4.814599990844727, -4.8155999183654785, -4.8155999183654785, -4.815800189971924, -3.922300100326538, -4.133399963378906, -4.134200096130371, -4.401500225067139, -4.401800155639648, -4.402900218963623, -4.769100189208984, -4.769499778747559, -4.769499778747559, -4.769599914550781, -4.769599914550781, -4.769700050354004, -4.769599914550781, -4.769700050354004, -4.770299911499023, -4.770299911499023, -5.356500148773193, -5.356500148773193, -5.356500148773193, -5.3566999435424805, -5.3566999435424805, -5.356599807739258, -5.3566999435424805, -5.3566999435424805, -5.3566999435424805, -5.356800079345703, -5.356900215148926, -5.3566999435424805, -5.356800079345703, -5.356800079345703, -4.133399963378906, -3.9214999675750732, -4.769100189208984, -4.401800155639648, -4.769199848175049, -4.769199848175049, -4.400100231170654, -4.400400161743164, -4.765999794006348, -4.768599987030029, -5.348400115966797, -5.35099983215332, -5.352099895477295, -5.353499889373779, -3.1837000846862793, -3.506700038909912, -3.717900037765503, -3.718600034713745, -3.9860999584198, -3.986799955368042, -3.986999988555908, -4.353799819946289, -4.3541998863220215, -4.3541998863220215, -4.354300022125244, -3.718400001525879, -4.941199779510498, -4.941199779510498, -4.941199779510498, -4.941199779510498, -4.941299915313721, -4.941299915313721, -4.941400051116943, -4.941400051116943, -4.941400051116943, -4.941400051116943, -4.941400051116943, -4.941500186920166, -4.941500186920166, -4.941500186920166, -4.9415998458862305, -4.9415998458862305, -4.9415998458862305, -4.9415998458862305, -4.35290002822876, -4.350599765777588, -4.935800075531006, -4.9369001388549805, -4.938799858093262, -4.939000129699707, -4.939000129699707, -4.9390997886657715, -4.939499855041504, -4.9405999183654785, -4.940899848937988, -3.2852001190185547, -3.938999891281128, -3.938999891281128, -4.306399822235107, -4.307000160217285, -4.30709981918335, -4.893799781799316, -4.893899917602539, -4.893899917602539, -4.893899917602539, -4.894100189208984, -4.894100189208984, -4.894100189208984, -4.894100189208984, -4.894100189208984, -4.894199848175049, -4.894199848175049, -4.894199848175049, -4.8942999839782715, -4.8942999839782715, -4.8942999839782715, -4.8942999839782715, -4.8942999839782715, -4.8942999839782715, -4.894400119781494, -4.894400119781494, -4.894499778747559, -4.894499778747559, -4.894499778747559, -4.894499778747559, -3.939300060272217, -4.307300090789795, -4.30649995803833, -4.890900135040283, -4.892199993133545, -4.892600059509277, -4.8927998542785645, -4.892899990081787, -4.893499851226807, -4.893599987030029, -4.893799781799316, -4.893899917602539, -4.894100189208984]}, \"token.table\": {\"Topic\": [2, 2, 4, 3, 3, 1, 2, 2, 1, 2, 4, 4, 1, 3, 2, 4, 2, 4, 4, 3, 1, 4, 4, 1, 2, 2, 4, 1, 1, 1, 2, 3, 1, 1, 4, 1, 2, 4, 2, 4, 3, 1, 4, 2, 2, 3, 3, 1, 2, 1, 3, 2, 3, 1, 3, 3, 4, 1, 4, 3, 2, 4, 2, 1, 4, 4, 3, 2, 4, 1, 1, 3, 1, 2, 4, 3, 1, 1, 3, 1, 2, 1, 2, 4, 1, 2, 4, 4, 1, 2, 2, 2, 1, 1, 2, 4, 4, 1, 3, 4, 1, 2, 2, 3, 1, 2, 3, 4, 1, 2, 1, 1, 2, 4, 3, 1, 2, 4, 1, 2, 1, 1, 4, 1, 2, 4, 1, 4, 4, 3, 3, 4, 2, 2, 3, 4, 3, 4, 2, 1, 2, 3, 1, 3, 1, 2, 4, 2, 2, 4, 3, 3, 2, 3, 3, 1, 4, 2, 1, 1, 2, 3, 2, 3, 3, 3, 2, 3, 1, 3, 1, 4, 1, 3, 4, 3, 3, 2, 4, 3, 1, 1, 2, 3, 4, 2, 4, 4, 2, 1, 3, 4, 1, 2, 1, 2], \"Freq\": [1.015978217124939, 0.810740053653717, 0.4053700268268585, 0.8912275433540344, 0.6085124015808105, 0.5910641551017761, 0.29553207755088806, 0.8992456197738647, 0.7688518166542053, 1.016166090965271, 0.9095766544342041, 0.9096082448959351, 0.7935409545898438, 0.3967704772949219, 0.43813395500183105, 0.43813395500183105, 0.7782964706420898, 0.9095560312271118, 0.9095718264579773, 0.8912194967269897, 0.882317066192627, 0.9538724422454834, 0.9538730978965759, 0.9756262302398682, 0.7782952785491943, 0.7783251404762268, 0.625702977180481, 0.7688512802124023, 0.8823297619819641, 0.6294777989387512, 0.4196518361568451, 0.8911927938461304, 0.7406692504882812, 0.7688515782356262, 0.9095525741577148, 0.745238721370697, 0.3726193606853485, 0.8350517153739929, 0.5607215166091919, 0.5607215166091919, 0.8911874294281006, 0.9091596603393555, 0.9095001220703125, 0.7541984915733337, 0.42975831031799316, 0.42975831031799316, 0.6085022687911987, 0.9997742772102356, 0.778298020362854, 0.5490666627883911, 0.5490666627883911, 1.0160497426986694, 0.6085278391838074, 0.6575647592544556, 0.3287823796272278, 0.9239409565925598, 0.9095707535743713, 0.8002883195877075, 0.40014415979385376, 0.8911793828010559, 0.7540028095245361, 0.9095664024353027, 0.7783256769180298, 0.5559520125389099, 0.5559520125389099, 0.9095331430435181, 0.9240139722824097, 1.0158597230911255, 0.9096162915229797, 0.740996241569519, 0.9997701048851013, 0.6085254549980164, 0.3268868625164032, 0.6537737250328064, 0.9095339775085449, 0.8912225961685181, 0.7688510417938232, 0.7930968403816223, 0.39654842019081116, 0.9997696280479431, 0.9949475526809692, 0.7688524723052979, 0.7784315943717957, 0.9095520973205566, 0.4395011365413666, 0.4395011365413666, 0.2197505682706833, 0.9095383882522583, 0.7452108263969421, 0.37260541319847107, 1.0159510374069214, 1.01580810546875, 0.9756332635879517, 0.9756318926811218, 1.0161486864089966, 0.9094975590705872, 0.909620463848114, 1.0003265142440796, 0.6169793009757996, 0.6169793009757996, 0.9997778534889221, 0.7782955765724182, 0.7783229947090149, 0.8912245035171509, 0.768848180770874, 0.43386369943618774, 0.43386369943618774, 0.43386369943618774, 0.420966774225235, 0.420966774225235, 0.7688506245613098, 0.3748575747013092, 0.7497151494026184, 0.6256815791130066, 0.8912051320075989, 0.9997709393501282, 0.5771945714950562, 0.5771945714950562, 0.8361817598342896, 0.2090454399585724, 0.8334466218948364, 0.6251978278160095, 0.31259891390800476, 0.2205139398574829, 0.6615418195724487, 0.2205139398574829, 1.00025475025177, 0.9095382690429688, 0.9096298217773438, 0.8912163972854614, 0.6279677748680115, 0.31398388743400574, 0.7783454656600952, 0.7783189415931702, 0.8911846280097961, 0.9095308184623718, 0.8912150263786316, 0.9095263481140137, 1.0160986185073853, 0.6240715384483337, 0.31203576922416687, 0.31203576922416687, 0.999776303768158, 0.8912250995635986, 0.40273207426071167, 0.40273207426071167, 0.40273207426071167, 0.778292715549469, 0.8993929624557495, 0.9095781445503235, 0.935448408126831, 0.8912058472633362, 0.7540035843849182, 0.9239585995674133, 0.8912255764007568, 0.8004291653633118, 0.4002145826816559, 0.7783266305923462, 0.9997767806053162, 0.6241310834884644, 0.3120655417442322, 0.3120655417442322, 1.0160542726516724, 0.7446391582489014, 0.9412484169006348, 0.8912302851676941, 0.5537565350532532, 0.5537565350532532, 0.9997766613960266, 0.8912090063095093, 0.999774158000946, 0.9095025658607483, 0.43106064200401306, 0.43106064200401306, 0.43106064200401306, 0.7446748614311218, 0.8912397623062134, 0.7783015370368958, 0.6257890462875366, 0.8912283182144165, 0.9997658133506775, 0.47587040066719055, 0.23793520033359528, 0.23793520033359528, 0.23793520033359528, 0.7783041596412659, 0.9095156192779541, 0.9095318913459778, 1.0160763263702393, 0.5490714907646179, 0.5490714907646179, 0.9095020890235901, 0.49367356300354004, 0.49367356300354004, 0.37480393052101135, 0.7496078610420227], \"Term\": [\"ablaze\", \"accuse\", \"accuse\", \"add\", \"adequate\", \"administration\", \"administration\", \"afiz\", \"ahead\", \"ali\", \"allot\", \"amir\", \"area\", \"area\", \"arrest\", \"arrest\", \"arson\", \"aspirant\", \"assembly\", \"assurance\", \"attack\", \"bhawan\", \"bhubaneswar\", \"bjp\", \"bomb\", \"camp\", \"capital\", \"center\", \"chaudhury\", \"clash\", \"clash\", \"class\", \"come\", \"commission\", \"commissionerate\", \"conduct\", \"conduct\", \"congress\", \"connection\", \"connection\", \"correspondent\", \"cpi\", \"damage\", \"death\", \"demand\", \"demand\", \"department\", \"dhalai\", \"dig\", \"disperse\", \"disperse\", \"dispute\", \"doda\", \"due\", \"due\", \"education\", \"ekamra\", \"election\", \"election\", \"equally\", \"family\", \"file\", \"flee\", \"follow\", \"follow\", \"forward\", \"gandoh\", \"ganjam\", \"gate\", \"go\", \"goon\", \"government\", \"group\", \"group\", \"hadaqat\", \"high\", \"hit\", \"hold\", \"hold\", \"hooligan\", \"house\", \"however\", \"identify\", \"impending\", \"incident\", \"incident\", \"incident\", \"individual\", \"injure\", \"injure\", \"injury\", \"jarada\", \"jitendra\", \"kamalpur\", \"kamrup\", \"khan\", \"khanm\", \"koraput\", \"later\", \"later\", \"lead\", \"maintain\", \"male\", \"malikpura\", \"manikbhander\", \"mar\", \"mar\", \"mar\", \"march\", \"march\", \"massive\", \"mob\", \"mob\", \"mohammad\", \"movement\", \"mute\", \"odisha\", \"odisha\", \"office\", \"office\", \"open\", \"party\", \"party\", \"person\", \"person\", \"person\", \"platoon\", \"poll\", \"premise\", \"pritam\", \"protest\", \"protest\", \"provide\", \"provocation\", \"pursue\", \"rahman\", \"raise\", \"ranjan\", \"rd\", \"reach\", \"reach\", \"reach\", \"remain\", \"remote\", \"report\", \"report\", \"report\", \"rival\", \"rizul\", \"sajir\", \"school\", \"sdm\", \"set\", \"shortage\", \"slogans\", \"source\", \"source\", \"southern\", \"spectator\", \"spot\", \"spot\", \"spot\", \"stab\", \"staff\", \"student\", \"study\", \"sub\", \"sub\", \"successfully\", \"suffer\", \"supporter\", \"tafiq\", \"take\", \"take\", \"take\", \"teaching\", \"thapa\", \"thatched\", \"ticket\", \"tilogra\", \"tiwn\", \"today\", \"today\", \"today\", \"today\", \"undergo\", \"vandalise\", \"vandalism\", \"vandalize\", \"vehicular\", \"vehicular\", \"video\", \"village\", \"village\", \"yesterday\", \"yesterday\"]}, \"R\": 30, \"lambda.step\": 0.01, \"plot.opts\": {\"xlab\": \"PC1\", \"ylab\": \"PC2\"}, \"topic.order\": [1, 3, 4, 2]};\n",
       "\n",
       "function LDAvis_load_lib(url, callback){\n",
       "  var s = document.createElement('script');\n",
       "  s.src = url;\n",
       "  s.async = true;\n",
       "  s.onreadystatechange = s.onload = callback;\n",
       "  s.onerror = function(){console.warn(\"failed to load library \" + url);};\n",
       "  document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "}\n",
       "\n",
       "if(typeof(LDAvis) !== \"undefined\"){\n",
       "   // already loaded: just create the visualization\n",
       "   !function(LDAvis){\n",
       "       new LDAvis(\"#\" + \"ldavis_el770221121075616322196539872\", ldavis_el770221121075616322196539872_data);\n",
       "   }(LDAvis);\n",
       "}else if(typeof define === \"function\" && define.amd){\n",
       "   // require.js is available: use it to load d3/LDAvis\n",
       "   require.config({paths: {d3: \"https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min\"}});\n",
       "   require([\"d3\"], function(d3){\n",
       "      window.d3 = d3;\n",
       "      LDAvis_load_lib(\"https://cdn.rawgit.com/bmabey/pyLDAvis/files/ldavis.v1.0.0.js\", function(){\n",
       "        new LDAvis(\"#\" + \"ldavis_el770221121075616322196539872\", ldavis_el770221121075616322196539872_data);\n",
       "      });\n",
       "    });\n",
       "}else{\n",
       "    // require.js not available: dynamically load d3 & LDAvis\n",
       "    LDAvis_load_lib(\"https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js\", function(){\n",
       "         LDAvis_load_lib(\"https://cdn.rawgit.com/bmabey/pyLDAvis/files/ldavis.v1.0.0.js\", function(){\n",
       "                 new LDAvis(\"#\" + \"ldavis_el770221121075616322196539872\", ldavis_el770221121075616322196539872_data);\n",
       "            })\n",
       "         });\n",
       "}\n",
       "</script>"
      ],
      "text/plain": [
       "PreparedData(topic_coordinates=              x         y  topics  cluster       Freq\n",
       "topic                                                \n",
       "0     -0.138261 -0.053434       1        1  35.335426\n",
       "2     -0.001904  0.130187       2        1  32.964764\n",
       "3      0.088586 -0.085959       3        1  16.578337\n",
       "1      0.051579  0.009206       4        1  15.121478, topic_info=     Category      Freq             Term     Total  loglift  logprob\n",
       "term                                                                \n",
       "230   Default  4.000000          student  4.000000  30.0000  30.0000\n",
       "294   Default  3.000000         congress  3.000000  29.0000  29.0000\n",
       "223   Default  3.000000           school  3.000000  28.0000  28.0000\n",
       "228   Default  2.000000            staff  2.000000  27.0000  27.0000\n",
       "235   Default  2.000000         teaching  2.000000  26.0000  26.0000\n",
       "148   Default  5.000000              cpi  5.000000  25.0000  25.0000\n",
       "216   Default  3.000000          protest  3.000000  24.0000  24.0000\n",
       "177   Default  4.000000             open  4.000000  23.0000  23.0000\n",
       "288   Default  2.000000           bhawan  2.000000  22.0000  22.0000\n",
       "289   Default  2.000000      bhubaneswar  2.000000  21.0000  21.0000\n",
       "47    Default  3.000000           odisha  3.000000  20.0000  20.0000\n",
       "207   Default  2.000000           gandoh  2.000000  19.0000  19.0000\n",
       "225   Default  2.000000         shortage  2.000000  18.0000  18.0000\n",
       "204   Default  2.000000        education  2.000000  17.0000  17.0000\n",
       "31    Default  4.000000            house  4.000000  16.0000  16.0000\n",
       "141   Default  4.000000              bjp  4.000000  15.0000  15.0000\n",
       "165   Default  4.000000         kamalpur  4.000000  14.0000  14.0000\n",
       "164   Default  4.000000         jitendra  4.000000  13.0000  13.0000\n",
       "327   Default  1.000000           ticket  1.000000  12.0000  12.0000\n",
       "291   Default  1.000000          capital  1.000000  11.0000  11.0000\n",
       "313   Default  1.000000         mohammad  1.000000  10.0000  10.0000\n",
       "87    Default  3.000000             afiz  3.000000   9.0000   9.0000\n",
       "124   Default  3.000000            rizul  3.000000   8.0000   8.0000\n",
       "5     Default  2.000000           arrest  2.000000   7.0000   7.0000\n",
       "208   Default  1.000000       government  1.000000   6.0000   6.0000\n",
       "202   Default  1.000000             doda  1.000000   5.0000   5.0000\n",
       "199   Default  1.000000       department  1.000000   4.0000   4.0000\n",
       "191   Default  1.000000         adequate  1.000000   3.0000   3.0000\n",
       "203   Default  3.000000              due  3.000000   2.0000   2.0000\n",
       "98    Default  2.000000           demand  2.000000   1.0000   1.0000\n",
       "...       ...       ...              ...       ...      ...      ...\n",
       "287    Topic4  0.621828         assembly  1.099418   1.3192  -4.8941\n",
       "300    Topic4  0.621825             file  1.099425   1.3192  -4.8941\n",
       "286    Topic4  0.621789         aspirant  1.099437   1.3191  -4.8942\n",
       "292    Topic4  0.621778  commissionerate  1.099442   1.3191  -4.8942\n",
       "305    Topic4  0.621757        impending  1.099442   1.3190  -4.8942\n",
       "303    Topic4  0.621726          hadaqat  1.099464   1.3190  -4.8943\n",
       "315    Topic4  0.621721             poll  1.099459   1.3190  -4.8943\n",
       "306    Topic4  0.621718       individual  1.099459   1.3190  -4.8943\n",
       "330    Topic4  0.621711        vandalism  1.099467   1.3189  -4.8943\n",
       "318    Topic4  0.621710           rahman  1.099468   1.3189  -4.8943\n",
       "301    Topic4  0.621706          forward  1.099465   1.3189  -4.8943\n",
       "319    Topic4  0.621684           ranjan  1.099473   1.3189  -4.8944\n",
       "329    Topic4  0.621662        vandalise  1.099486   1.3189  -4.8944\n",
       "326    Topic4  0.621605            tafiq  1.099502   1.3187  -4.8945\n",
       "331    Topic4  0.621600            video  1.099503   1.3187  -4.8945\n",
       "307    Topic4  0.621599             khan  1.099508   1.3187  -4.8945\n",
       "297    Topic4  0.621596           damage  1.099505   1.3187  -4.8945\n",
       "47     Topic4  1.615737           odisha  3.465036   1.1261  -3.9393\n",
       "5      Topic4  1.118267           arrest  2.282407   1.1756  -4.3073\n",
       "77     Topic4  1.119160            today  4.202825   0.5659  -4.3065\n",
       "216    Topic4  0.623848          protest  3.184877   0.2588  -4.8909\n",
       "85     Topic4  0.623005           accuse  2.466882   0.5129  -4.8922\n",
       "50     Topic4  0.622786           person  4.534861  -0.0963  -4.8926\n",
       "233    Topic4  0.622663             take  2.319859   0.5738  -4.8928\n",
       "34     Topic4  0.622571         incident  4.550614  -0.1001  -4.8929\n",
       "123    Topic4  0.622231           report  2.483040   0.5051  -4.8935\n",
       "211    Topic4  0.622140            later  1.620800   0.9315  -4.8936\n",
       "18     Topic4  0.622015       connection  1.783417   0.8357  -4.8938\n",
       "44     Topic4  0.622000              mar  2.304871   0.5792  -4.8939\n",
       "257    Topic4  0.621880           follow  1.798716   0.8270  -4.8941\n",
       "\n",
       "[202 rows x 6 columns], token_table=      Topic      Freq            Term\n",
       "term                                 \n",
       "0         2  1.015978          ablaze\n",
       "85        2  0.810740          accuse\n",
       "85        4  0.405370          accuse\n",
       "190       3  0.891228             add\n",
       "191       3  0.608512        adequate\n",
       "1         1  0.591064  administration\n",
       "1         2  0.295532  administration\n",
       "87        2  0.899246            afiz\n",
       "139       1  0.768852           ahead\n",
       "88        2  1.016166             ali\n",
       "284       4  0.909577           allot\n",
       "285       4  0.909608            amir\n",
       "192       1  0.793541            area\n",
       "192       3  0.396770            area\n",
       "5         2  0.438134          arrest\n",
       "5         4  0.438134          arrest\n",
       "6         2  0.778296           arson\n",
       "286       4  0.909556        aspirant\n",
       "287       4  0.909572        assembly\n",
       "193       3  0.891219       assurance\n",
       "140       1  0.882317          attack\n",
       "288       4  0.953872          bhawan\n",
       "289       4  0.953873     bhubaneswar\n",
       "141       1  0.975626             bjp\n",
       "8         2  0.778295            bomb\n",
       "11        2  0.778325            camp\n",
       "291       4  0.625703         capital\n",
       "142       1  0.768851          center\n",
       "143       1  0.882330       chaudhury\n",
       "15        1  0.629478           clash\n",
       "...     ...       ...             ...\n",
       "130       2  0.553757             sub\n",
       "130       3  0.553757             sub\n",
       "185       1  0.999777    successfully\n",
       "232       3  0.891209          suffer\n",
       "186       1  0.999774       supporter\n",
       "326       4  0.909503           tafiq\n",
       "233       1  0.431061            take\n",
       "233       3  0.431061            take\n",
       "233       4  0.431061            take\n",
       "235       3  0.744675        teaching\n",
       "236       3  0.891240           thapa\n",
       "75        2  0.778302        thatched\n",
       "327       4  0.625789          ticket\n",
       "238       3  0.891228         tilogra\n",
       "188       1  0.999766            tiwn\n",
       "77        1  0.475870           today\n",
       "77        2  0.237935           today\n",
       "77        3  0.237935           today\n",
       "77        4  0.237935           today\n",
       "79        2  0.778304         undergo\n",
       "329       4  0.909516       vandalise\n",
       "330       4  0.909532       vandalism\n",
       "136       2  1.016076       vandalize\n",
       "239       1  0.549071       vehicular\n",
       "239       3  0.549071       vehicular\n",
       "331       4  0.909502           video\n",
       "80        1  0.493674         village\n",
       "80        2  0.493674         village\n",
       "83        1  0.374804       yesterday\n",
       "83        2  0.749608       yesterday\n",
       "\n",
       "[196 rows x 3 columns], R=30, lambda_step=0.01, plot_opts={'xlab': 'PC1', 'ylab': 'PC2'}, topic_order=[1, 3, 4, 2])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pprint import pprint\n",
    "\n",
    "# Gensim\n",
    "import gensim\n",
    "import gensim.corpora as corpora\n",
    "from gensim.utils import simple_preprocess\n",
    "from gensim.models import CoherenceModel\n",
    "\n",
    "# spacy for lemmatization\n",
    "import spacy\n",
    "\n",
    "import ast\n",
    "\n",
    "# Plotting tools\n",
    "import pyLDAvis\n",
    "import pyLDAvis.gensim  # don't skip this\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "# NLTK Stop words\n",
    "from nltk.corpus import stopwords\n",
    "stop_words = stopwords.words('english')\n",
    "stop_words.extend(['say', 'says', 'from', 'subject', 're', 'edu', 'use', 'also', 'would', 'district', 'hospital', 'police', 'arrest', 'station', 'indonesia', 'indonesian'])\n",
    "# stop_words.extend(['Papua', 'papua', 'Jokowi', 'Prabowo', 'Jakarta', 'Widodo', 'jakarta', 'jokowi', 'prabowo', 'widodo', 'subianto', 'year'])\n",
    "# stop_words.extend(['election', 'people', 'presidential', 'papuan', 'year', 'april', 'festival', 'president'])\n",
    "# stop_words.extend(['country', 'state', 'antara', 'photo'])\n",
    "#stop_words.extend(['thai', 'thailand', 'breaking', 'news', 'national', 'bangkok', 'yesterday', 'year'])\n",
    "csvfile = open('sample_input.csv', 'r')\n",
    "df1 = pd.read_csv(csvfile)\n",
    "\n",
    "articles_all = []\n",
    "for index, row in df1.iterrows():\n",
    "    text = row['text']\n",
    "    articles_all.append(text)\n",
    "\n",
    "#TODO: Remove any URLS?\n",
    "data = articles_all\n",
    "\n",
    "def sent_to_words(sentences):\n",
    "    for sentence in sentences:\n",
    "        yield(gensim.utils.simple_preprocess(str(sentence), deacc=True))  # deacc=True removes punctuations\n",
    "\n",
    "data_words = list(sent_to_words(data))\n",
    "\n",
    "# print(data_words[:1])\n",
    "\n",
    "# Build the bigram and trigram models\n",
    "bigram = gensim.models.Phrases(data_words, min_count=5, threshold=100) # higher threshold fewer phrases.\n",
    "trigram = gensim.models.Phrases(bigram[data_words], threshold=100)\n",
    "\n",
    "# Faster way to get a sentence clubbed as a trigram/bigram\n",
    "bigram_mod = gensim.models.phrases.Phraser(bigram)\n",
    "trigram_mod = gensim.models.phrases.Phraser(trigram)\n",
    "\n",
    "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
    "def remove_stopwords(texts):\n",
    "    return [[word for word in simple_preprocess(str(doc)) if word not in stop_words] for doc in texts]\n",
    "\n",
    "def make_bigrams(texts):\n",
    "    return [bigram_mod[doc] for doc in texts]\n",
    "\n",
    "def make_trigrams(texts):\n",
    "    return [trigram_mod[bigram_mod[doc]] for doc in texts]\n",
    "\n",
    "def lemmatization(texts, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV']):\n",
    "    \"\"\"https://spacy.io/api/annotation\"\"\"\n",
    "    texts_out = []\n",
    "    for sent in texts:\n",
    "        doc = nlp(\" \".join(sent))\n",
    "        texts_out.append([token.lemma_ for token in doc if token.pos_ in allowed_postags])\n",
    "    return texts_out\n",
    "\n",
    "\n",
    "# Remove Stop Words\n",
    "data_words_nostops = remove_stopwords(data_words)\n",
    "\n",
    "# Form Bigrams\n",
    "data_words_bigrams = make_bigrams(data_words_nostops)\n",
    "\n",
    "# Initialize spacy 'en' model, keeping only tagger component (for efficiency)\n",
    "# python3 -m spacy download en\n",
    "nlp = spacy.load('en', disable=['parser', 'ner'])\n",
    "\n",
    "# Do lemmatization keeping only noun, adj, vb, adv\n",
    "data_lemmatized = lemmatization(data_words_bigrams, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV'])\n",
    "\n",
    "# print(data_lemmatized[:1])\n",
    "\n",
    "# Create Dictionary\n",
    "id2word = corpora.Dictionary(data_lemmatized)\n",
    "\n",
    "# Create Corpus\n",
    "texts = data_lemmatized\n",
    "\n",
    "for i in texts:\n",
    "    for j in i:\n",
    "        if j == 'say':\n",
    "            i.remove(j)\n",
    "\n",
    "# Term Document Frequency\n",
    "corpus = [id2word.doc2bow(text) for text in texts]\n",
    "\n",
    "# View\n",
    "# print(corpus[:1])\n",
    "\n",
    "# Human readable format of corpus (term-frequency)\n",
    "l = [[(id2word[id], freq) for id, freq in cp] for cp in corpus[:1]]\n",
    "#print(l)\n",
    "\n",
    "# Build LDA model\n",
    "lda_model = gensim.models.ldamodel.LdaModel(corpus=corpus,\n",
    "                                           id2word=id2word,\n",
    "                                           num_topics=4,\n",
    "                                           random_state=100,\n",
    "                                           update_every=1,\n",
    "                                           chunksize=100,\n",
    "                                           passes=10,\n",
    "                                           alpha='auto',\n",
    "                                           per_word_topics=True)\n",
    "\n",
    "# Print the Keyword in the 10 topics\n",
    "pprint(lda_model.print_topics())\n",
    "# doc_lda = lda_model[corpus]\n",
    "\n",
    "# Compute Perplexity\n",
    "print('\\nPerplexity: ', lda_model.log_perplexity(corpus))  # a measure of how good the model is. lower the better.\n",
    "\n",
    "# Compute Coherence Score\n",
    "coherence_model_lda = CoherenceModel(model=lda_model, texts=data_lemmatized, dictionary=id2word, coherence='c_v')\n",
    "coherence_lda = coherence_model_lda.get_coherence()\n",
    "print('\\nCoherence Score: ', coherence_lda)\n",
    "pyLDAvis.enable_notebook()\n",
    "vis = pyLDAvis.gensim.prepare(lda_model, corpus, id2word)\n",
    "vis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using a mallet model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/akshayakp/anaconda3/lib/python3.7/site-packages/smart_open/smart_open_lib.py:160: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() or inspect.getfullargspec()\n",
      "  args, varargs, keywords, defaults = inspect.getargspec(kallable)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0,\n",
      "  [('congress', 0.05172413793103448),\n",
      "   ('house', 0.04310344827586207),\n",
      "   ('rizul', 0.034482758620689655),\n",
      "   ('afiz', 0.034482758620689655),\n",
      "   ('mob', 0.02586206896551724),\n",
      "   ('report', 0.02586206896551724),\n",
      "   ('death', 0.02586206896551724),\n",
      "   ('bhawan', 0.02586206896551724),\n",
      "   ('family', 0.02586206896551724),\n",
      "   ('accuse', 0.02586206896551724)]),\n",
      " (1,\n",
      "  [('cpi', 0.046052631578947366),\n",
      "   ('march', 0.039473684210526314),\n",
      "   ('office', 0.039473684210526314),\n",
      "   ('open', 0.039473684210526314),\n",
      "   ('bjp', 0.03289473684210526),\n",
      "   ('jitendra', 0.03289473684210526),\n",
      "   ('kamalpur', 0.03289473684210526),\n",
      "   ('attack', 0.02631578947368421),\n",
      "   ('chaudhury', 0.02631578947368421),\n",
      "   ('spot', 0.02631578947368421)]),\n",
      " (2,\n",
      "  [('student', 0.058333333333333334),\n",
      "   ('school', 0.041666666666666664),\n",
      "   ('protest', 0.041666666666666664),\n",
      "   ('due', 0.03333333333333333),\n",
      "   ('teaching', 0.03333333333333333),\n",
      "   ('reach', 0.03333333333333333),\n",
      "   ('staff', 0.03333333333333333),\n",
      "   ('shortage', 0.025),\n",
      "   ('demand', 0.025),\n",
      "   ('gandoh', 0.025)]),\n",
      " (3,\n",
      "  [('group', 0.058394160583941604),\n",
      "   ('clash', 0.043795620437956206),\n",
      "   ('incident', 0.043795620437956206),\n",
      "   ('person', 0.043795620437956206),\n",
      "   ('today', 0.043795620437956206),\n",
      "   ('odisha', 0.0364963503649635),\n",
      "   ('village', 0.0364963503649635),\n",
      "   ('set', 0.021897810218978103),\n",
      "   ('injure', 0.021897810218978103),\n",
      "   ('conduct', 0.021897810218978103)])]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/akshayakp/anaconda3/lib/python3.7/site-packages/smart_open/smart_open_lib.py:160: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() or inspect.getfullargspec()\n",
      "  args, varargs, keywords, defaults = inspect.getargspec(kallable)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Coherence Score:  0.6526016721316937\n"
     ]
    }
   ],
   "source": [
    "# Download File: http://mallet.cs.umass.edu/dist/mallet-2.0.8.zip\n",
    "mallet_path = './mallet-2.0.8/bin/mallet' # update this path\n",
    "ldamallet = gensim.models.wrappers.LdaMallet(mallet_path, corpus=corpus, num_topics=4, id2word=id2word)\n",
    "\n",
    "# Show Topics\n",
    "pprint(ldamallet.show_topics(formatted=False))\n",
    "\n",
    "# Compute Coherence Score\n",
    "coherence_model_ldamallet = CoherenceModel(model=ldamallet, texts=data_lemmatized, dictionary=id2word, coherence='c_v')\n",
    "coherence_ldamallet = coherence_model_ldamallet.get_coherence()\n",
    "\n",
    "print('\\nCoherence Score: ', coherence_ldamallet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_coherence_values(dictionary, corpus, texts, limit, start=4, step=1):\n",
    "    \"\"\"\n",
    "    Compute c_v coherence for various number of topics\n",
    "\n",
    "    Parameters:\n",
    "    ----------\n",
    "    dictionary : Gensim dictionary\n",
    "    corpus : Gensim corpus\n",
    "    texts : List of input texts\n",
    "    limit : Max num of topics\n",
    "\n",
    "    Returns:\n",
    "    -------\n",
    "    model_list : List of LDA topic models\n",
    "    coherence_values : Coherence values corresponding to the LDA model with respective number of topics\n",
    "    \"\"\"\n",
    "    coherence_values = []\n",
    "    model_list = []\n",
    "    for num_topics in range(start, limit, step):\n",
    "        model = gensim.models.wrappers.LdaMallet(mallet_path, corpus=corpus, num_topics=num_topics, id2word=id2word)\n",
    "        model_list.append(model)\n",
    "        coherencemodel = CoherenceModel(model=model, texts=texts, dictionary=dictionary, coherence='c_v')\n",
    "        coherence_values.append(coherencemodel.get_coherence())\n",
    "\n",
    "    return model_list, coherence_values\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/akshayakp/anaconda3/lib/python3.7/site-packages/smart_open/smart_open_lib.py:160: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() or inspect.getfullargspec()\n",
      "  args, varargs, keywords, defaults = inspect.getargspec(kallable)\n",
      "/Users/akshayakp/anaconda3/lib/python3.7/site-packages/smart_open/smart_open_lib.py:160: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() or inspect.getfullargspec()\n",
      "  args, varargs, keywords, defaults = inspect.getargspec(kallable)\n",
      "/Users/akshayakp/anaconda3/lib/python3.7/site-packages/smart_open/smart_open_lib.py:160: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() or inspect.getfullargspec()\n",
      "  args, varargs, keywords, defaults = inspect.getargspec(kallable)\n",
      "/Users/akshayakp/anaconda3/lib/python3.7/site-packages/smart_open/smart_open_lib.py:160: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() or inspect.getfullargspec()\n",
      "  args, varargs, keywords, defaults = inspect.getargspec(kallable)\n",
      "/Users/akshayakp/anaconda3/lib/python3.7/site-packages/smart_open/smart_open_lib.py:160: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() or inspect.getfullargspec()\n",
      "  args, varargs, keywords, defaults = inspect.getargspec(kallable)\n",
      "/Users/akshayakp/anaconda3/lib/python3.7/site-packages/smart_open/smart_open_lib.py:160: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() or inspect.getfullargspec()\n",
      "  args, varargs, keywords, defaults = inspect.getargspec(kallable)\n"
     ]
    }
   ],
   "source": [
    "# Can take a long time to run.\n",
    "model_list, coherence_values = compute_coherence_values(dictionary=id2word, corpus=corpus, texts=texts, start=2, limit=5, step=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xd8FVXawPHfkxAIvYWaQkLvUi6gqNiRtYAiu2JH15fVV3RdXxGwiw3LrrqurgtWXBVZQA0ooqy9QlBMSGghtNBJ6CGQ8rx/zESv15QLyc2kPN/PJx/unDln7pPhJk/mzJxzRFUxxhhjjleY1wEYY4yp3iyRGGOMKRdLJMYYY8rFEokxxphysURijDGmXCyRGGOMKRdLJMYYY8rFEokxxphysURijDGmXOp4HUBliIqK0vj4eK/DMMaYamXZsmW7VbVVWfVqRSKJj48nKSnJ6zCMMaZaEZGNwdSzri1jjDHlYonEGGNMuYQ0kYjICBFZLSLpIjK5mP3jRGSXiCx3v653y8/wK1suIrkicpG771URWe+3r18ovwdjjDGlC9k9EhEJB54DzgEygaUikqiqaQFV31bVCf4Fqvop0M89TgsgHfjIr8pEVZ1Tnvjy8vLIzMwkNze3PIcJmcjISGJiYoiIiPA6FGOMKVUob7YPBtJVNQNARGYBo4DARFKWMcBCVc2pyOAyMzNp3Lgx8fHxiEhFHrrcVJWsrCwyMzNJSEjwOhxjjClVKLu2ooHNftuZblmgS0QkWUTmiEhsMfvHAm8FlD3stnlKROoV9+YiMl5EkkQkadeuXb/Zn5ubS8uWLatcEgEQEVq2bFllr5aMMcZfKBNJcb+hA5djnA/Eq2pfYDHw2q8OINIO6AMs8iueAnQHBgEtgEnFvbmqTldVn6r6WrUq/jHoqphEilTl2Iwxxl8oE0km4H+FEQNs9a+gqlmqesTdnAEMDDjGH4B3VDXPr802dRwBXsHpQjPGGONny97DPDA/lbyCwpC/VygTyVKgi4gkiEhdnC6qRP8K7hVHkZHAyoBjXEZAt1ZRG3H+ZL8IWFHBcRtjTLVVWKj8+7uNDP/b57y9dDOrth0I+XuG7Ga7quaLyAScbqlw4GVVTRWRqUCSqiYCt4jISCAfyAbGFbUXkXicK5rPAw79hoi0wuk6Ww7cEKrvwRhjqpONWYeYPDeFbzOyOKVzFI+O7kNsiwYhf9+QTpGiqh8AHwSU3ev3egrOPY/i2m6gmJvzqnpmxUbpnZkzZ/Lkk08iIvTt25fXX3/d65CMMdVQQaHy2jcbeGLRauqECdNG9+HSQbGVdq+1Vsy1VZYH5qeStnV/hR6zZ/sm3HdhrxL3p6am8vDDD/P1118TFRVFdnZ2hb6/MaZ2SN95kElzk1m2cQ9ndGvFI6P70K5p/UqNwRKJRz755BPGjBlDVFQUAC1atPA4ImNMdZJfUMiML9fz1OI11I8I529/OIGL+0d78sSnJRIo9cohVFTVHvE1xhyXVdv3c8ecZJIz93FurzY8eFFvWjeO9Cwem7TRI2eddRazZ88mKysLwLq2jDFlyiso5JnFa7nw2a/Ysucw/7i8Py9cOdDTJAJ2ReKZXr16cdddd3HaaacRHh5O//79efXVV70OyxhTRa3Yso+Jc5JZuW0/I09oz30X9qRlo2In9qh0lkg8dM0113DNNdd4HYYxpgo7kl/A3/+7lhc+z6Blw7pMv2ogw3u19TqsX7FEYowxVdSPm/YwcU4y6TsPMmZgDPec35OmDarejOCWSIwxpoo5fLSAv328mpe+Wk/bJpG8eu0gTu/W2uuwSlSrE0lVfnJKNXB+S2NMbbBkfTZ3zPmJDVk5XD4kjim/607jyKp3FeKv1iaSyMhIsrKyquRU8kXrkURGevskhjGm8hw6ks/jH67itW83EtuiPm9eP4ShnaO8DisotTaRxMTEkJmZSXFrlVQFRSskGmNqvq/W7mbyvGS27D3MuKHx3DGiGw3qVp9fz9Un0goWERFhqw8aYzy1PzePRz9YyVtLNtMxqiGz/3QSg+Kr3ywXtTaRGGOMlz5dtZMp81LYeSCXP53Wkb+c3ZXIiHCvwzoulkiMMaYS7c05ytQFacz7YQtd2zTihatOpl9sM6/DKhdLJMYYU0k+XLGdu99dwd6co9xyZmduOrMz9epUz6sQfyGda0tERojIahFJF5HJxewfJyK7RGS5+3W9374Cv/JEv/IEEfleRNaKyNvu6ovGGFNl7T54hJve/IEb/r2M1o3r8d6Ek7lteLcakUQghFckIhIOPAecg7N++1IRSVTVtICqb6vqhGIOcVhV+xVT/hjwlKrOEpEXgD8C/6zI2I0xpiKoKvOTt3F/YioHc/O5fXhX/nRaJyLCa9Z8uaHs2hoMpKtqBoCIzAJGAYGJJGjuOu1nApe7Ra8B92OJxBhTxezcn8td767g47QdnBDbjCfG9KVrm8ZehxUSoUwk0cBmv+1MYEgx9S4RkWHAGuAvqlrUJlJEknDWc5+mqu8CLYG9qprvd8zfLMdrjDFeUVXmLMvkwQVpHMkv5M7zunPdyQnUqWFXIf5CmUiKGy4eOO/HfOAtVT0iIjfgXGEUrckep6pbRaQj8ImIpADFrYdb7FwiIjIeGA8QFxd3PPEbY8wx2br3MFPmpfD5ml0Mim/OY5f0pWOrRl6HFXKhTCSZQKzfdgyw1b+Cqmb5bc7Auf9RtG+r+2+GiHwG9AfmAs1EpI57VfKbY/q1nw5MB/D5fDZxlTEmZFSVt5Zs5pEPVlKoygMje3HViR0IC6ta0y+FSigTyVKgi4gkAFuAsfxybwMAEWmnqtvczZHASre8OZDjXqlEAScDj6uqisinwBhgFnAN8F4IvwdjjCnVpqwcJs9L5pt1WQzt1JLHLulLbIsGXodVqUKWSFQ1X0QmAIuAcOBlVU0VkalAkqomAreIyEic+yDZwDi3eQ/gXyJSiPOI8jS/p70mAbNE5CHgR+ClUH0PxhhTksJCZea3G3jsw9WEhwmPju7D2EGxVW4S2MogtWG6cp/Pp0lJSV6HYYypITJ2HeSOOckkbdzD6d1a8cjFfWjfrL7XYVU4EVmmqr6y6tnIdmOMCVJ+QSEvfbWev328hnp1wvjr709g9IDoWnkV4s8SiTHGBGHNjgNM/M9P/JS5j+E92/DQRb1p3cTWDAJLJMYYU6q8gkJe+Gwdf/9kLY0jI3j2sv5c0Lddrb8K8WeJxBhjSrBiyz7umJNM2rb9XNC3HQ+M7EXLRvW8DqvKsURijDEBjuQX8I9P0vnnZ+to1qAuL1w5kBG923odVpVlicQYY/ws37yXif/5ibU7D3LJgBjuuaAHzRrYJOOlsURijDFAbl4BT328hhlfZtCmSSSvjBvEGd1bex1WtWCJxBhT6y3dkM0dc5JZv/sQlw2OY8p53WkSGeF1WNWGJRJjTK2VczSfxz9czWvfbiC6WX3euH4IJ3eO8jqsascSiTGmVvomfTeT5iWzOfsw44bGM/HcbjSsZ78Sj4edNWNMrXIgN49HPljFW0s2kRDVkNl/OonBCS28Dqtas0RijKk1Pl29kzvnpbBjfy7jh3XkL2d3pX7dmrFuupcskRhjarx9OXlMXZDG3B8y6dK6Ec/fOJT+cc29DqvGsERijKnRPkrdzl3vriD70FFuPrMzE87sTL06dhVSkSyRGGNqpKyDR7h/fhrzf9pKj3ZNeGXcIHpHN/U6rBrJEokxpkZRVRYkb+O+xFQO5Obxf+d05YbTOxERHuZ1aDVWSM+siIwQkdUiki4ik4vZP05EdonIcvfrere8n4h8KyKpIpIsIpf6tXlVRNb7tekXyu/BGFN97DyQyw3/XsbNb/1IbPP6LLj5VG4+q4slkRAL2RWJiIQDzwHnAJnAUhFJ9Fsyt8jbqjohoCwHuFpV14pIe2CZiCxS1b3u/omqOidUsRtjqhdVZd4PW5i6II3DeQVM/l13rj8lgTqWQCpFKLu2BgPpqpoBICKzgFFAYCL5DVVd4/d6q4jsBFoBe0tuZYypjbbtO8yd81L4dPUuBnZozuNj+tKpVSOvw6pVQpmuo4HNftuZblmgS9zuqzkiEhu4U0QGA3WBdX7FD7ttnhIRWxzAmFpIVXlrySaG/+0LvsvI5r4LezL7TydZEvFAKBNJccuHacD2fCBeVfsCi4HXfnUAkXbA68C1qlroFk8BugODgBbApGLfXGS8iCSJSNKuXbuO/7swxlQ5m7NzuPKl75kyL4Xe0U1ZdOswrj05gfAwW7XQC6Hs2soE/K8wYoCt/hVUNctvcwbwWNGGiDQB3gfuVtXv/Npsc18eEZFXgNuLe3NVnQ5MB/D5fIEJzBhTDRUWKq9/t5HHPlxFmAgPX9ybywbFEWYJxFOhTCRLgS4ikgBsAcYCl/tXEJF2folhJLDSLa8LvAPMVNX/FNdGnAWTLwJWhPB7MMZUEet3H2LSnGSWbMhmWNdWPDq6D9HN6nsdliGEiURV80VkArAICAdeVtVUEZkKJKlqInCLiIwE8oFsYJzb/A/AMKCliBSVjVPV5cAbItIKp+tsOXBDqL4HY4z3CgqVl79az5MfraZenTCeGNOXMQNjcP6WNFWBqNb8Xh+fz6dJSUleh2GMOUZrdxxg4pxklm/ey9k92vDwxb1p0yTS67BqDRFZpqq+surZyHZjTJWTV1DI9C8yeGbxWhrWC+eZsf0YeUJ7uwqpoiyRGGOqlNSt+7hjTjKpW/dzft92PDCyF1GN7Cn/qswSiTGmSjiaX8g/PlnL85+to1mDurxw5QBG9G7ndVgmCJZIjDGe+2nzXu6Yk8zqHQcY3T+aey7oSfOGdb0OywTJEokxxjO5eQU8tXgNM77IoHXjSF4e5+PM7m28DsscI0skxhhPJG3I5o65yWTsOsRlg2OZcl4PmkRGeB2WOQ5BJRIRqQ/EqerqEMdjjKnhco7m88Si1bz6zQbaN63Pv/84hFO6RHkdlimHMhOJiFwIPIkzcWKCu/7HVFUdGergjDE1yzfrdjN5bgqbsnO4+qQOTBrRnYb1rGOkugvmf/B+nCnhPwNQ1eUiEh+yiIwxNc6B3DymLVzFG99vIr5lA94efyJDOrb0OixTQYJJJPmqus8GAhljjsfna3YxZW4y2/bncv0pCfzf8G7UrxvudVimAgWTSFaIyOVAuIh0AW4BvgltWMaY6m5fTh4PvZ/Gf5Zl0rl1I+beOJQBcc29DsuEQDCJ5GbgLuAI8CbOJIwPhTIoY0z19nHaDu56J4WsQ0e56YxO3HxmFyIj7Cqkpio1kbjrrj+gqhNxkokxxpQo+9BRHpifynvLt9K9bWNeHjeI3tFNvQ7LhFipiURVC0RkYGUFY4ypvt5P3sa9761gf24efzm7Kzee3om6dUK5CKupKoLp2vpRRBKB/wCHigpVdV7IojLGVBs7D+Ry33upLFyxnb4xTXljzBC6t23idVimEgWTSFoAWcCZfmUKWCIxphZTVd5dvoUH5qeRc7SASSO68z+nJlAn3K5CapsyE4mqXnu8BxeREcAzOCskvqiq0wL2jwOewFmKF+Afqvqiu+8a4G63/CFVfc0tHwi8CtQHPgD+rLVhdS5jqpBt+w5z1zsr+GTVTgbENePxMSfQuXUjr8MyHglmZHsM8CxwMs6VyFc4v7wzy2gXDjwHnANkAktFJFFV0wKqvq2qEwLatgDuA3zuey5z2+4B/gmMB77DSSQjgIVlfR/GmPJTVWYnbeahBSvJKyzkngt6Mm5oPOFhNs6sNgvmGvQVIBFoD0QD892ysgwG0lU1Q1WPArOAUUHGdS7wsapmu8njY2CEiLQDmqjqt+5VyEzgoiCPaYwph83ZOVz10hImzU2hV3QTFt06jD+ekmBJxAR1j6SVqvonjldF5NYg2kUDm/22M4EhxdS7RESGAWuAv6jq5hLaRrtfmcWU/4aIjMe5ciEuLi6IcI0xxSksVP79/UamLVyFAA9d1JvLB8cRZgnEuIK5ItktIleKSLj7dSXOzfeyFPcpC7yXMR+IV9W+wGLgtTLaBnNMp1B1uqr6VNXXqlWrIMI1xgTasPsQY2d8x73vpTKwQ3MW/WUYV57YwZKI+ZVgrkiuA/4BPIXzS/sbt6wsmUCs33YMsNW/gqr6J6QZwGN+bU8PaPuZWx5T2jGNMeVXUKi88vV6nvxoNRHhYTw+pi+/HxiDzblnihPMU1ubgOOZMn4p0EVEEnCeyhoLXO5fQUTaqeo2d3MksNJ9vQh4RESKJuYZDkxR1WwROSAiJwLfA1fjPAhgjKkg6TsPMHFOMj9u2stZ3Vvz8MV9aNs00uuwTBUWzFNbr+E8pbXX3W4O/FVVS70qUdV8EZmAkxTCgZdVNVVEpgJJqpoI3CIiI4F8IBsY57bNFpEHcZIROOufZLuvb+SXx38XYk9sGVMh8gsK+dcXGTyzeC0N6oXzzNh+jDyhvV2FmDJJWUMwRORHVe1fVllV5vP5NCkpyeswjKmyVm7bz8Q5P7Fiy37O69OWB0b2plXjel6HZTwmIstU1VdWvWDukYSJSHP3MdyiMR62pJkxNcDR/EKe+zSd5z5Np1mDCJ6/YgDn9WnndVimmgkmIfwV+EZE5rjbvwceDl1IxpjKkJK5j4lzfmLV9gNc1K89913Yi+YN63odlqmGgrnZPlNEknDm2hJgdDGj040x1URuXgHP/Hct07/IIKpRXV682sfZPdt4HZapxoK52d4JWKeqaSJyOnC2iGwtuvlujKk+lm3MZuKcZDJ2HeJSXyx3nt+DpvUjvA7LVHPBdG3NBXwi0hl4EWcQ4ZvAeaEMzBhTcQ4fLeDJj1bz8tfrad+0PjOvG8ywrjZQ11SMYBJJofso72jgGVV9VkR+DHVgxpiK8e26LCbPS2ZjVg5XndiBSb/rTqN69ryMqTjBfJryROQynMF/F7pldi1sTBV38Eg+0xau5N/fbaJDywbMGn8iJ3Zs6XVYpgYKJpFcC9wAPKyq692R6v8ObVjGmPL4Ys0upsxLYeu+w/zxlARuH96N+nXDvQ7L1FDBPLWVBtzit70emFZyC2OMV/YdzuPh99OYnZRJp1YNmXPDUAZ2aF52Q2PKwTpKjakh/rtyB3e+k8Lug0e58fRO/PmsLkRG2FWICT1LJMZUc3sOHeWB+am8u3wr3ds2ZsbVPvrGNPM6LFOLBJ1IRKShqh4KZTDGmGOzMGUb97y3gr05efz5rC7cdEZn6tYJZpkhYypOMAMSh+KMH2kExInICcCfVPV/Qx2cMaZ4uw4c4b7EFXyQsp3e0U14/Y9D6NGuiddhmVoqmCuSp3DWUE8EUNWf3KVxjTGVTFVJ/Gkr9yemcuhIAXeM6Mb4UztSJ9yuQox3guraUtXNAWsSFIQmHGNMSbbvy+Xud1NYvHIn/eOa8cSYvnRu3djrsIwJas32zW73lopIXRG5nV9WMiyViIwQkdUiki4ik0upN0ZEVER87vYVIrLc76tQRPq5+z5zj1m0r3UwsRhTXakqs5du5pynPuer9N3cfX4P5tww1JKIqTKCuSK5AXgGiMZZM/0j4KayGolIOPAccI7bbqmIJAbOHCwijXHGqXxfVKaqbwBvuPv7AO+p6nK/Zleoqq1UZWq8zD05TJmXwpdrdzM4oQWPX9KX+KiGXodlzK8EMyBxN3DFcRx7MJCuqhkAIjILGAUETkH/IPA4cHsJx7kMeOs43t+YaquwUHljySamfbASBR4c1YsrhnQgLMyWvTVVT5ldWyLymog089tuLiIvB3HsaGCz33amW+Z/7P5ArKouKOU4l/LbRPKK2611j9iC0qaG2Zh1iMtf/I573l3BgA7NWXTrMK46Kd6SiKmyguna6uu/9oiq7nETQFmK+9T/vEC8iIThPBE2rsQDiAwBclR1hV/xFaq6xe0SmwtcBcwspu14YDxAXFxcEOEa462CQuXVbzbwxKJVRISF8dglffiDLxb7W8lUdaFcsz0TiPXbjgG2+m03BnoDn7k/KG2BRBEZ6Xf/YywBVyOqusX994CIvInThfabRKKq04HpAD6fTwP3G1OVpO88yB1zfuKHTXs5s3trHrm4D22bRnodljFBCeWa7UuBLu5swVtwksLlRTtVdR8QVbQtIp8BtxclEfeK5ffAML86dYBmqrpbRCKAC4DFQcRiTJWUX1DIjC/X89TiNTSoG87Tl/ZjVL/2dhViqpVg12xfBpzBMazZ7i6GNQFYBIQDL6tqqohMBZJUNbGMQwwDMotu1rvqAYvcJBKOk0RmlBWLMVXRqu37uWNOMsmZ+xjRqy1TL+pF68Z2FWKqH1Etu9fHfZS3DX6JR1U3hTCuCuXz+TQpyZ4WNlVDbl4B//gknRc+X0fT+hFMHdWb8/u28zosY35DRJapqq+sesHMtXUzcB+wA2dEu+DcNO9b3iCNqW2WrM9m8rxkMnYdYvSAaO45vyfNG9b1OixjyiWYeyR/BrqpalaogzGmptqfm8djC1fxxvebiGlen5nXDWZY11Zeh2VMhQgmkWwG9oU6EGNqqo/TdnDPuyvYeSCX609J4LbhXWlQ15YCMjVHMJ/mDJxHdN8HjhQVqurfQhaVMTXAzgO5PJCYxvsp2+jetjEvXDWQfrG24JSpeYJJJJvcr7rulzGmFKrKf5Iyeej9NHLzC5l4bjfGD+tIhE31bmqoYB7/fQBshURjgrEx6xBT5qXwzbosBse34NFL+tCpVSOvwzImpIJ5ausk4CVshURjSpRfUMhLXzkDCyPCwnj44t5cNijO5scytUIwXVtPYyskGlOiFVv2MXleMiu27Oecnm14cFRvm97E1Cq2QqIxxyk3r4CnF69lxpcZNG9Ql+evGMDvere16U1MrRPU47/+KyTiLEIV1AqJxtRU36zbzZ3zUtiQlcOlvljuPK8HTRtEeB2WMZ4I2QqJxtRE+3LyeHThSmYt3UyHlg148/ohDO0cVXZDY2qwUhOJO8fWVap6PCskGlOjLEzZxr2JqWQfOsqfTuvIrWd1pX7dcK/DMsZzpSYSVS0QkVE4C1AZUyvt2J/Lve+tYFHqDnq1b8Ir4wbRO7qp12EZU2UE07X1tYj8A3gb+Hkciar+ELKojKkCCguVWUs38+gHKzlaUMjk33Xn+lMSqGMDC435lWASyVD336l+ZQqcWfHhGFM1ZOw6yJR5KXy/PpuTOrbk0dF9iI9q6HVYxlRJwYxsP6MyAjGmKsgrKGT6Fxk889+1RNYJ4/FL+vJ7X4w90mtMKcq8RheRNiLykogsdLd7isgfgzm4iIwQkdUiki4ik0upN0ZEVER87na8iBwWkeXu1wt+dQeKSIp7zL+L/YSbCpKcuZcLn/2KJxat5uwerVl822n8YVCsJRFjyhBM19arwCvAXe72Gpz7JS+V1sh94us54Bycx4aXikhi4DK9ItIYZ2zK9wGHWKeq/Yo59D+B8cB3wAfACGBhEN+HMcXKOZrPUx+v4aWv1tOqcT3+ddVAzu3V1uuwjKk2grlrGKWqs4FCcNZiJ7iR7YOBdFXNUNWjwCxgVDH1HgQeB3LLOqCItAOaqOq36qwRPBO4KIhYjCnWl2t3ce7TXzDjy/WMHRzHx7edZknEmGMUTCI5JCItcW6wIyInEtxCV9E4i2IVyXTLfiYi/YFYVV1QTPsEEflRRD4XkVP9jplZ2jH9jj1eRJJEJGnXrl1BhGtqkz2HjvJ/s3/iqpeWEBEWxtvjT+SRi/vQJNJGpxtzrILp2roNZ8LGTiLyNdAKGBNEu+I6lvXnnSJhOONTxhVTbxsQp6pZIjIQeFdEepV1zF8Vqk4HpgP4fL5i65jaR1VZkLyNB+ansjcnjwlndGbCmZ2JjLCBhcYcr2Ce2vpBRE4DuuH8Il+tqnlBHDsTiPXbjgG2+m03BnrjrL4I0BZIFJGRqpqEuxqjqi4TkXVAV/eYMaUc05gSbd17mHveXcF/V+2kb0xTZl43hJ7tm3gdljHVXrALRw8G4t36A0QEVZ1ZRpulQBcRSQC2AGOBy4t2quo+4OdJikTkM+B2VU0SkVZAtjuyviPQBchQ1WwROeB2r30PXA08G+T3YGqpwkLlje838tiHqykoVO4+vwfXnpxAuK0VYkyFCGZhq9eBTsByfrnJXnSju0Sqmi8iE4BFQDjwsqqmishUIElVE0tpPgyYKiJFN/ZvUNVsd9+NOE+S1cd5Wsue2DIlSt95gElzU1i2cQ+ndonikYv7ENuigddhGVOjiPPwUykVRFYCPbWsilWYz+fTpKQkr8MwlehofiH//Gwdz32aToN64dxzfk9GD4i2MSHGHAMRWaaqvrLqBdO1tQLn/sW2ckdlTCX4YdMeJs9NZs2Og4w8oT33XtiTqEb1vA7LmBqrxEQiIvNxurAaA2kisgT3BjiAqo4MfXjGBO/QkXyeWLSa177dQNsmkbx0jY+zerTxOixjarzSrkierLQojCmnT1fv5O53VrB132GuPrEDE0d0p1G9YJ8lMcaUR4k/aar6edFrEWkDDHI3l6jqzlAHZkwwsg4e4cEFaby7fCudWzdizg0nMbBDC6/DMqZWCeaprT8ATwCf4YwjeVZEJqrqnBDHZkyJVJV3l29h6vw0Dh7J589ndeF/z+hEvTo2sNCYyhbMtf9dwKCiqxB3jMdiwBKJ8UTmnhzuemcFn6/ZRf+4Zjx2SV+6tmnsdVjG1FrBJJKwgK6sLIKbo8uYClVQqLz2zQae/Gg1APdf2JOrToq3gYXGeCyYRPKhiCwC3nK3L8UGAZpKtnr7ASbNTWb55r2c3q0VD1/ch+hm9b0OyxhDcHNtTRSR0cApOPdIpqvqOyGPzBjgSH4Bz32SzvOfraNJ/QieGduPkSe0t4GFxlQhpY0j6Qy0UdWvVXUeMM8tHyYinVR1XWUFaWqnpRuymTw3mXW7DjG6fzR3X9CTFg3reh2WMSZAaVckTwN3FlOe4+67MCQRmVrvQG4ej3+4mte/20h0s/q8dt1gTuvayuuwjDElKC2RxKtqcmChOztvfMgiMrXa4rQd3P3uCnYcyOW6kxP4v+FdaWgDC42p0kr7CY0sZZ/d5TQVateBI9w/P5X3k7fRrU1j/nnlAPrHNfc6LGOl9iAgAAAUqElEQVRMEEpLJEtF5H9UdYZ/oYj8EVgW2rBMbaGqzFmWyUPvr+Tw0QJuH96V8cM6UbeOPWFuTHVRWiK5FXhHRK7gl8ThA+oCF4c6MFPzbcrKYco7yXydnsWg+OY8OrovnVs38josY8wxKm2urR3AUBE5A2dJXID3VfWTYA8uIiOAZ3AWtnpRVaeVUG8M8B+cEfRJInIOMA0naR0FJha9r7uSYjvgsNt8uM39Vb3kFxTyytcb+OvHq6kTFsZDF/Xm8sFxhNnAQmOqpWDGkXwKfHqsBxaRcOA54ByctdaXikiiqqYF1GsM3IKzdG6R3cCFqrpVRHrjrLIY7bf/Cnddd1PNpG7dx+S5KaRs2cfZPdrw4EW9aNfUbrkZU52F8nGYwUC6qmYAiMgsYBSQFlDvQeBx4PaiAlX90W9/KhApIvVU9QimWsrNK+CZ/65l+hcZNG8QwXOXD+C8Pm1tYKExNUAoE0k0sNlvOxMY4l9BRPoDsaq6QERup3iXAD8GJJFXRKQAmAs8VJ2XAa4NvsvIYsq8FNbvPsQffDHceV4PmjWwgYXG1BShTCTF/an58y98EQkDngLGlXgAkV7AY8Bwv+IrVHWL2yU2F7gKmFlM2/HAeIC4uLjjCN+U177DeUxbuJK3lmwmrkUD3rh+CCd3jvI6LGNMBQtlIskEYv22Y4CtftuNcW7if+Z2b7QFEkVkpHvDPQZ4B7jafzoWVd3i/ntARN7E6UL7TSJR1enAdACfz2dXLJXswxXbufe9Few+eIQ/DevIrWd3pX5dWyvEmJoolIlkKdBFRBKALcBY4PKinaq6D/j5z1P3aazb3STSDHgfmKKqX/vVqQM0U9XdIhIBXICzNoqpInbuz+Xe91L5MHU7Pds14aVrBtEnpqnXYRljQihkiURV80VkAs4TV+HAy6qaKiJTgSRVTSyl+QSgM3CPiNzjlg0HDgGL3CQSjpNEZhR/CFOZVJW3l27m4Q9WcjS/kEkjunP9qQlEhNvAQmNqOqkN96l9Pp8mJdnTwqGyfvchpsxL5ruMbE7s2IJHR/clIaqh12EZY8pJRJapqq+sejYbnjlueQWFvPjlep5evIa6dcKYNroPlw6KtUd6jallLJGY45KSuY9Jc5NJ27af3/VuywMje9G6SWnzfBpjaipLJOaYHD5awFOL1/DilxlENarHC1cOZETvtl6HZYzxkCUSE7Sv03czZV4Km7JzuGxwHJN/152m9SO8DssY4zFLJKZMe3OO8vD7K/nPskwSohoya/yJnNixpddhGWOqCEskpkSqygcp27kvMZU9OUf539M7cctZXYiMsIGFxphfWCIxxdq27zD3vJvK4pU76BPdlJnXDaZn+yZeh2WMqYIskZhfKSxU3liyiccWriK/sJC7z+/BuKHx1LGBhcaYElgiMT9L33mQKfOSWbphD6d0juKRi/sQ17KB12EZY6o4SySGo/mF/OvzdTz7STr164bz5O9P4JIB0Taw0BgTFEsktdzyzXuZPDeZVdsPcEHfdtx3YS9aNa7ndVjGmGrEEkktdehIPn/9aA2vfLOetk0iefFqH2f3bON1WMaYasgSSS30+Zpd3DkvhS17D3P1SR2YeG43GkfawEJjzPGxRFKLZB86ykML0pj34xY6tWrInBtOwhffwuuwjDHVnCWSWkBVSfxpKw/MT+NAbh63nNWFm87oRL06NrDQGFN+lkhquC17D3P3Oyl8unoX/WKb8dglfenWtrHXYRljapCQjjITkREislpE0kVkcin1xoiIiojPr2yK2261iJx7rMes7QoKlVe/Xs85f/uc79dnc9+FPZl741BLIsaYCheyKxIRCQeeA84BMoGlIpKoqmkB9RoDtwDf+5X1xFnjvRfQHlgsIl3d3WUes7Zbs+MAk+Ym8+OmvZzWtRUPX9ybmOY2sNAYExqh7NoaDKSragaAiMwCRgGBv/QfBB4HbvcrGwXMUtUjwHoRSXePR5DHrJWO5Bfw/KfreP6zdBrVq8PTl/ZjVL/2NrDQGBNSoUwk0cBmv+1MYIh/BRHpD8Sq6gIRuT2g7XcBbaPd16Ues7ZatjGbSXNTSN95kIv7R3P3+T1o2cgGFhpjQi+UiaS4P4P1550iYcBTwLhjaFvcPR0tpgwRGQ+MB4iLiysj1Orr4JF8Hv9wFa9/t5H2Tevz6rWDOL1ba6/DMsbUIqFMJJlArN92DLDVb7sx0Bv4zO16aQskisjIMtqWdsyfqep0YDqAz+crNtlUd5+s2sFd76xg+/5cxg2N5/bh3WhYzx7EM8ZUrlD+1lkKdBGRBGALzs3zy4t2quo+IKpoW0Q+A25X1SQROQy8KSJ/w7nZ3gVYgnOlUuIxa4vdB4/wwPw05v+0lW5tGvP8FQPoH9fc67CMMbVUyBKJquaLyARgERAOvKyqqSIyFUhS1cRS2qaKyGycm+j5wE2qWgBQ3DFD9T1UNarKvB+28OD7aeQcKeC2c7pyw2mdqFvH1goxxnhHVGtkr8+v+Hw+TUpK8jqMctmcncOd76Tw5drd+Do0Z9olfejc2saEGGNCR0SWqaqvrHrWoV7FFRQqr3y9nr9+tIYwgQdH9eKKIR0IC7NHeo0xVYMlkips5bb9TJ6bzE+Z+zire2sevKg37ZvV9zosY4z5FUskVVBuXgHPfrKWf32eQdP6ETx7WX8u6NvOBhYaY6okSyRVzPcZWUyZl0LG7kOMGRjDXef1oHnDul6HZYwxJbJEUkXsz81j2sJVvPn9JmJb1Of1Pw7m1C6tvA7LGGPKZImkCliUup1731vBrgNH+J9TE/jLOV1pUNf+a4wx1YP9tvLQzgO53J+Yygcp2+netjEzrvbRN6aZ12EZY8wxsUTiAVVldtJmHn5/Jbn5hUw8txvjh3UkItwGFhpjqh9LJJVsw+5DTJmXwrcZWQxOaMG00X3o2KqR12EZY8xxs0RSSfILCnnxq/U89fEa6oaH8cjFfRg7KNYGFhpjqj1LJJVgxZZ9TJqbTOrW/Qzv2YYHL+pNmyaRXodljDEVwhJJCB0+WsDT/13Di1+up0XDurxw5QBG9G7ndVjGGFOhLJGEyDfpu5nyTgobs3IYOyiWKb/rQdMGEV6HZYwxFc4SSQXbl5PHwx+kMTspk/iWDXjzf4YwtFNU2Q2NMaaaskRSQVSVhSu2c+97qezJOcoNp3Xi1rO7EBkR7nVoxhgTUpZIKsD2fbnc894KPk7bQe/oJrx67SB6Rzf1OixjjKkUIU0kIjICeAZnNcMXVXVawP4bgJuAAuAgMF5V00TkCmCiX9W+wABVXe4uydsOOOzuG66qO0P5fZSksFB5a+kmpn2wirzCQu48rzvXnZxAHRtYaIypRUKWSEQkHHgOOAfIBJaKSKKqpvlVe1NVX3DrjwT+BoxQ1TeAN9zyPsB7qrrcr90Vqurpkofrdh1kytwUlmzIZminljw6ug8dWjb0MiRjjPFEKK9IBgPpqpoBICKzgFE467ADoKr7/eo3BIpb9/cy4K0QxnlM8goK+dfn6/j7J+lE1gnj8TF9+f3AGFsrxBhTa4UykUQDm/22M4EhgZVE5CbgNqAucGYxx7kUJwH5e0VECoC5wENazMLzIjIeGA8QFxd3PPH/xk+b9zJpbjKrth/g/D7tuG9kT1o3toGFxpjaLZSd+cX9if6bX/iq+pyqdgImAXf/6gAiQ4AcVV3hV3yFqvYBTnW/riruzVV1uqr6VNXXqlX51vXIOZrPgwvSuPj5r9mTc5TpVw3kuSsGWBIxxhhCe0WSCcT6bccAW0upPwv4Z0DZWAK6tVR1i/vvARF5E6cLbWa5oy3BF2t2cec7KWTuOcyVJ8Zxx4juNIm0gYXGGFMklIlkKdBFRBKALThJ4XL/CiLSRVXXupvnA2v99oUBvweG+ZXVAZqp6m4RiQAuABaH6huYMi+Ft5ZsomOrhsz+00kMTmgRqrcyxphqK2SJRFXzRWQCsAjn8d+XVTVVRKYCSaqaCEwQkbOBPGAPcI3fIYYBmUU36131gEVuEgnHSSIzQvU9xLdswM1nduamMzrbwEJjjCmBFHOfusbx+XyalOTp08LGGFPtiMgyVfWVVc9GzhljjCkXSyTGGGPKxRKJMcaYcrFEYowxplwskRhjjCkXSyTGGGPKxRKJMcaYcrFEYowxplxqxYBEEdkFbDzO5lHA7goMp6JYXMfG4jo2FtexqalxdVDVMme9rRWJpDxEJCmYkZ2VzeI6NhbXsbG4jk1tj8u6towxxpSLJRJjjDHlYomkbNO9DqAEFtexsbiOjcV1bGp1XHaPxBhjTLnYFYkxxphyqbWJRERiReRTEVkpIqki8udi6oiI/F1E0kUkWUQG+O27RkTWul/XBLYNcVxXuPEki8g3InKC374NIpIiIstFpMIWYQkyrtNFZJ/73stF5F6/fSNEZLV7LidXclwT/WJaISIFItLC3Req8xUpIktE5Cc3rgeKqVNPRN52z8n3IhLvt2+KW75aRM6t5LhuE5E09/P1XxHp4LevwO9cJlZyXONEZJff+1/vty9UP4/BxPWUX0xrRGSv376QnC+/44eLyI8isqCYfZX3+VLVWvkFtAMGuK8bA2uAngF1zgMWAgKcCHzvlrcAMtx/m7uvm1diXEOL3g/4XVFc7vYGIMqj83U6sKCYtuHAOqAjUBf4KbBtKOMKqH8h8EklnC8BGrmvI4DvgRMD6vwv8IL7eizwtvu6p3uO6gEJ7rkLr8S4zgAauK9vLIrL3T5Y0efqGOIaB/yjmLah/HksM66A+jfjrAYb0vPld/zbgDdL+LmrtM9Xrb0iUdVtqvqD+/oAsBKIDqg2Cpipju+AZiLSDjgX+FhVs1V1D/AxMKKy4lLVb9z3BfgOiKmI9y5vXKUYDKSraoaqHgVm4ZxbL+K6DHirIt67jLhUVQ+6mxHuV+ANyVHAa+7rOcBZIiJu+SxVPaKq64F0nHNYKXGp6qeqmuNuVtbnK5jzVZJQ/jwea1yV8vkCEJEY4HzgxRKqVNrnq9YmEn/uJV9/nL82/EUDm/22M92yksorKy5/f8S5aiqiwEciskxExld0TEHEdZLbDbBQRHq5ZVXifIlIA5xfMHP9ikN2vtxuh+XATpxfdCV+vlQ1H9gHtCTE5yuIuPwFfr4iRSRJRL4TkYsqKqZjiOsSt8ttjojEumVV4ny5XYAJwCd+xSE7X8DTwB1AYQn7K+3zVesTiYg0wvnFcquq7g/cXUwTLaW8suIqqnMGzg/6JL/ik1V1AE6X100iMqwS4/oBZ0qFE4BngXeLmhVzqEo/XzjdWl+rarZfWcjOl6oWqGo/nL/oB4tI78Cwi2tWSnllxeUEJ3Il4AOe8CuOU2ek9OXA0yLSqRLjmg/Eq2pfYDG//LVdJc4XTvfRHFUt8CsLyfkSkQuAnaq6rLRqxZSF5PNVqxOJiETg/PJ5Q1XnFVMlE4j1244BtpZSXllxISJ9cS5pR6lqVlG5qm51/90JvEMFdYkEE5eq7i/qBlDVD4AIEYmiCpwv11gCuh1Ceb783mMv8Bm/7W75+byISB2gKZBNiM9XEHEhImcDdwEjVfWIX5ui85Xhtu1fWXGpapZfLDOAge5rz8+Xq7TPV0Wfr5OBkSKyAaer+EwR+XdAncr7fJXnBkt1/sLJyjOBp0upcz6/vtm+xC1vAazHubHX3H3dohLjisPp1xwaUN4QaOz3+htgRCXG1ZZfxiYNBja57erg3ABN4Jeb7b0qKy63XtEPUcNKOl+tgGbu6/rAl8AFAXVu4tc3Q2e7r3vx65uhGVTczfZg4uqPcwO2S0B5c6Ce+zoKWEvFPTQRTFzt/F5fDHznvg7lz2OZcbn7uuE8uCGVcb4C3vt0ir/ZXmmfrzrUXicDVwEpbv8nwJ04v6RR1ReAD3Ce3EoHcoBr3X3ZIvIgsNRtN1V/3V0S6rjuxenrfN65d0a+OpfPbYB33LI6wJuq+mElxjUGuFFE8oHDwFh1Prn5IjIBWITzBNfLqppaiXGB84vnI1U95Nc2lOerHfCaiITjXPnPVtUFIjIVSFLVROAl4HURScdJcmPdmFNFZDaQBuQDN+mvu0tCHdcTQCPgP+652aSqI4EewL9EpNBtO01V0yoxrltEZCTOOcnGeYor1D+PwcQFzk32We7nvUgoz1exvPp82ch2Y4wx5VKr75EYY4wpP0skxhhjysUSiTHGmHKxRGKMMaZcLJEYY4wpF0skxvgRERWRv/pt3y4i91fwe1zrNyPsUfll9uFpx3GsWBF5uyLjM+ZY2eO/xvgRkVxgGzBIVXeLyO04s7/eH6L32wD4VHV3KI5vTGWwKxJjfi0fZ3nSvwTuEJFXRWSM3/ZB99/TReRzEZktznoU08RZM2aJe7UR9PxKIhIlIonyy1ozvd3yh0TkNXHWXlkrIte55Z2LBmKKSB1x1sZY4bb/X7f8CfllfZHHynNyjClObR7ZbkxJngOSReTxY2hzAs5I5mycKSdeVNXB4iy0dTNwa5DHeRBnfZmRIjIceBVn4kSAPjhr0TQBfhCR9wPa3gi0B05Q1QIRaSEibXBmZ+ilqioizY7hezImKHZFYkwAdWYPngnccgzNlqqzNsoRnHmqPnLLU4D4YzjOKcDrbhwfAe1FpKG7711VzVVngskvgEEBbc/GmVupwG2fjZPYCoEZInIxcAhjKpglEmOK9zTOFP0N/crycX9mxJmEqq7fviN+rwv9tgs5tiv/wCm+/bcDb2gGbktgmarm4VzRvAtcAgRexRhTbpZIjCmG+9f8bJxkUmQDv0xdPgpntbyK9gVwBfw8lXum30STF4mzDncUcCoQuMb8RziTZoa77VuISGOgiaouwLnvU+HTvhtj90iMKdlfgQl+2zOA90RkCfBfQtNNdC/wiogkAwdxZ5x2LcVZ1iAWuE9Vd7iJosi/gC4493fygX8CC4B5IlIP5w/H20IQs6nl7PFfY6oBEXkI2K2qT3sdizGBrGvLGGNMudgViTHGmHKxKxJjjDHlYonEGGNMuVgiMcYYUy6WSIwxxpSLJRJjjDHlYonEGGNMufw/+4HKhw/ohz0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Show graph\n",
    "limit=5; start=2; step=1;\n",
    "x = range(start, limit, step)\n",
    "plt.plot(x, coherence_values)\n",
    "plt.xlabel(\"Num Topics\")\n",
    "plt.ylabel(\"Coherence score\")\n",
    "plt.legend((\"coherence_values\"), loc='best')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num Topics = 2  has Coherence Value of 0.3714\n",
      "Num Topics = 3  has Coherence Value of 0.4705\n",
      "Num Topics = 4  has Coherence Value of 0.5721\n"
     ]
    }
   ],
   "source": [
    "# Print the coherence scores\n",
    "for m, cv in zip(x, coherence_values):\n",
    "    print(\"Num Topics =\", m, \" has Coherence Value of\", round(cv, 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0,\n",
      "  [('house', 0.043859649122807015),\n",
      "   ('afiz', 0.03508771929824561),\n",
      "   ('family', 0.02631578947368421),\n",
      "   ('accuse', 0.02631578947368421),\n",
      "   ('set', 0.02631578947368421),\n",
      "   ('mob', 0.02631578947368421),\n",
      "   ('death', 0.02631578947368421),\n",
      "   ('area', 0.02631578947368421),\n",
      "   ('ali', 0.017543859649122806),\n",
      "   ('dispute', 0.017543859649122806)]),\n",
      " (1,\n",
      "  [('group', 0.061068702290076333),\n",
      "   ('congress', 0.04580152671755725),\n",
      "   ('incident', 0.04580152671755725),\n",
      "   ('clash', 0.04580152671755725),\n",
      "   ('person', 0.04580152671755725),\n",
      "   ('odisha', 0.03816793893129771),\n",
      "   ('village', 0.03816793893129771),\n",
      "   ('bhubaneswar', 0.022900763358778626),\n",
      "   ('report', 0.022900763358778626),\n",
      "   ('bhawan', 0.022900763358778626)]),\n",
      " (2,\n",
      "  [('cpi', 0.046357615894039736),\n",
      "   ('open', 0.039735099337748346),\n",
      "   ('office', 0.039735099337748346),\n",
      "   ('jitendra', 0.033112582781456956),\n",
      "   ('kamalpur', 0.033112582781456956),\n",
      "   ('march', 0.033112582781456956),\n",
      "   ('bjp', 0.033112582781456956),\n",
      "   ('chaudhury', 0.026490066225165563),\n",
      "   ('party', 0.026490066225165563),\n",
      "   ('attack', 0.026490066225165563)]),\n",
      " (3,\n",
      "  [('student', 0.05426356589147287),\n",
      "   ('today', 0.046511627906976744),\n",
      "   ('protest', 0.03875968992248062),\n",
      "   ('school', 0.03875968992248062),\n",
      "   ('due', 0.031007751937984496),\n",
      "   ('teaching', 0.031007751937984496),\n",
      "   ('staff', 0.031007751937984496),\n",
      "   ('demand', 0.023255813953488372),\n",
      "   ('shortage', 0.023255813953488372),\n",
      "   ('gandoh', 0.023255813953488372)])]\n"
     ]
    }
   ],
   "source": [
    "# Select the model and print the topics\n",
    "optimal_model = model_list[2]\n",
    "model_topics = optimal_model.show_topics(formatted=False)\n",
    "pprint(model_topics)\n",
    "# pprint(optimal_model.print_topics(num_words=10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/akshayakp/anaconda3/lib/python3.7/site-packages/smart_open/smart_open_lib.py:160: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() or inspect.getfullargspec()\n",
      "  args, varargs, keywords, defaults = inspect.getargspec(kallable)\n",
      "/Users/akshayakp/anaconda3/lib/python3.7/site-packages/smart_open/smart_open_lib.py:160: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() or inspect.getfullargspec()\n",
      "  args, varargs, keywords, defaults = inspect.getargspec(kallable)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Document_No</th>\n",
       "      <th>Dominant_Topic</th>\n",
       "      <th>Topic_Perc_Contrib</th>\n",
       "      <th>Keywords</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.3957</td>\n",
       "      <td>group, congress, incident, clash, person, odis...</td>\n",
       "      <td>Odisha Sun Times Bureau Brahmapur, Mar 23 Poli...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.4158</td>\n",
       "      <td>house, afiz, family, accuse, set, mob, death, ...</td>\n",
       "      <td>ABHIJIT TALUKDAR Tamulpur, March 23, 2019 Tens...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.6553</td>\n",
       "      <td>cpi, open, office, jitendra, kamalpur, march, ...</td>\n",
       "      <td>Tripura News 10 CPI M supporters injured after...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.6197</td>\n",
       "      <td>student, today, protest, school, due, teaching...</td>\n",
       "      <td>Excelsior Correspondent DODA, Mar 23 The stude...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.3338</td>\n",
       "      <td>group, congress, incident, clash, person, odis...</td>\n",
       "      <td>Jeypore The Koraput district administration to...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.3906</td>\n",
       "      <td>group, congress, incident, clash, person, odis...</td>\n",
       "      <td>Bhubaneswar The Commissionerate Police arreste...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Document_No  Dominant_Topic  Topic_Perc_Contrib  \\\n",
       "0            0             1.0              0.3957   \n",
       "1            1             0.0              0.4158   \n",
       "2            2             2.0              0.6553   \n",
       "3            3             3.0              0.6197   \n",
       "4            4             1.0              0.3338   \n",
       "5            5             1.0              0.3906   \n",
       "\n",
       "                                            Keywords  \\\n",
       "0  group, congress, incident, clash, person, odis...   \n",
       "1  house, afiz, family, accuse, set, mob, death, ...   \n",
       "2  cpi, open, office, jitendra, kamalpur, march, ...   \n",
       "3  student, today, protest, school, due, teaching...   \n",
       "4  group, congress, incident, clash, person, odis...   \n",
       "5  group, congress, incident, clash, person, odis...   \n",
       "\n",
       "                                                Text  \n",
       "0  Odisha Sun Times Bureau Brahmapur, Mar 23 Poli...  \n",
       "1  ABHIJIT TALUKDAR Tamulpur, March 23, 2019 Tens...  \n",
       "2  Tripura News 10 CPI M supporters injured after...  \n",
       "3  Excelsior Correspondent DODA, Mar 23 The stude...  \n",
       "4  Jeypore The Koraput district administration to...  \n",
       "5  Bhubaneswar The Commissionerate Police arreste...  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def format_topics_sentences(ldamodel=lda_model, corpus=corpus, texts=data):\n",
    "    # Init output\n",
    "    sent_topics_df = pd.DataFrame()\n",
    "\n",
    "    # Get main topic in each document\n",
    "    for i, row in enumerate(ldamodel[corpus]):\n",
    "        row = sorted(row, key=lambda x: (x[1]), reverse=True)\n",
    "        # Get the Dominant topic, Perc Contribution and Keywords for each document\n",
    "        for j, (topic_num, prop_topic) in enumerate(row):\n",
    "            if j == 0:  # => dominant topic\n",
    "                wp = ldamodel.show_topic(topic_num)\n",
    "                topic_keywords = \", \".join([word for word, prop in wp])\n",
    "                sent_topics_df = sent_topics_df.append(pd.Series([int(topic_num), round(prop_topic,4), topic_keywords]), ignore_index=True)\n",
    "            else:\n",
    "                break\n",
    "    sent_topics_df.columns = ['Dominant_Topic', 'Perc_Contribution', 'Topic_Keywords']\n",
    "\n",
    "    # Add original text to the end of the output\n",
    "    contents = pd.Series(texts)\n",
    "    sent_topics_df = pd.concat([sent_topics_df, contents], axis=1)\n",
    "    return(sent_topics_df)\n",
    "\n",
    "\n",
    "df_topic_sents_keywords = format_topics_sentences(ldamodel=optimal_model, corpus=corpus, texts=data)\n",
    "\n",
    "# Format\n",
    "df_dominant_topic = df_topic_sents_keywords.reset_index()\n",
    "df_dominant_topic.columns = ['Document_No', 'Dominant_Topic', 'Topic_Perc_Contrib', 'Keywords', 'Text']\n",
    "\n",
    "# Show\n",
    "df_dominant_topic.head(40)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Topic_Num</th>\n",
       "      <th>Topic_Perc_Contrib</th>\n",
       "      <th>Keywords</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.4158</td>\n",
       "      <td>house, afiz, family, accuse, set, mob, death, ...</td>\n",
       "      <td>ABHIJIT TALUKDAR Tamulpur, March 23, 2019 Tens...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.3957</td>\n",
       "      <td>group, congress, incident, clash, person, odis...</td>\n",
       "      <td>Odisha Sun Times Bureau Brahmapur, Mar 23 Poli...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.6553</td>\n",
       "      <td>cpi, open, office, jitendra, kamalpur, march, ...</td>\n",
       "      <td>Tripura News 10 CPI M supporters injured after...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3.0</td>\n",
       "      <td>0.6197</td>\n",
       "      <td>student, today, protest, school, due, teaching...</td>\n",
       "      <td>Excelsior Correspondent DODA, Mar 23 The stude...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Topic_Num  Topic_Perc_Contrib  \\\n",
       "0        0.0              0.4158   \n",
       "1        1.0              0.3957   \n",
       "2        2.0              0.6553   \n",
       "3        3.0              0.6197   \n",
       "\n",
       "                                            Keywords  \\\n",
       "0  house, afiz, family, accuse, set, mob, death, ...   \n",
       "1  group, congress, incident, clash, person, odis...   \n",
       "2  cpi, open, office, jitendra, kamalpur, march, ...   \n",
       "3  student, today, protest, school, due, teaching...   \n",
       "\n",
       "                                                Text  \n",
       "0  ABHIJIT TALUKDAR Tamulpur, March 23, 2019 Tens...  \n",
       "1  Odisha Sun Times Bureau Brahmapur, Mar 23 Poli...  \n",
       "2  Tripura News 10 CPI M supporters injured after...  \n",
       "3  Excelsior Correspondent DODA, Mar 23 The stude...  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Group top 5 sentences under each topic\n",
    "sent_topics_sorteddf_mallet = pd.DataFrame()\n",
    "\n",
    "sent_topics_outdf_grpd = df_topic_sents_keywords.groupby('Dominant_Topic')\n",
    "\n",
    "# Number of Documents for Each Topic\n",
    "topic_counts = df_topic_sents_keywords['Dominant_Topic'].value_counts()\n",
    "\n",
    "# Percentage of Documents for Each Topic\n",
    "topic_contribution = round(topic_counts/topic_counts.sum(), 4)\n",
    "\n",
    "for i, grp in sent_topics_outdf_grpd:\n",
    "    sent_topics_sorteddf_mallet = pd.concat([sent_topics_sorteddf_mallet, \n",
    "                                             grp.sort_values(['Perc_Contribution'], ascending=[0]).head(1)], \n",
    "                                            axis=0)\n",
    "\n",
    "# Reset Index    \n",
    "sent_topics_sorteddf_mallet.reset_index(drop=True, inplace=True)\n",
    "\n",
    "# Format\n",
    "sent_topics_sorteddf_mallet.columns = ['Topic_Num', \"Topic_Perc_Contrib\", \"Keywords\", \"Text\"]\n",
    "\n",
    "# Show\n",
    "sent_topics_sorteddf_mallet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Dominant_Topic</th>\n",
       "      <th>Topic_Keywords</th>\n",
       "      <th>Num_Documents</th>\n",
       "      <th>Perc_Documents</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>group, congress, incident, clash, person, odis...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.1667</td>\n",
       "      <td>[odisha, sun, time, bureau, brahmapur, mar, to...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>house, afiz, family, accuse, set, mob, death, ...</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.5000</td>\n",
       "      <td>[abhijit, talukdar, march, tension, grip, kamr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.0</td>\n",
       "      <td>cpi, open, office, jitendra, kamalpur, march, ...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.1667</td>\n",
       "      <td>[tripura, news, cpi, supporter, injure, jiten,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3.0</td>\n",
       "      <td>student, today, protest, school, due, teaching...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.1667</td>\n",
       "      <td>[excelsior, correspondent, doda, mar, student,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>group, congress, incident, clash, person, odis...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[jeypore, koraput, administration, today, proh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1.0</td>\n",
       "      <td>group, congress, incident, clash, person, odis...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[bhubaneswar, commissionerate, arrest, person,...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Dominant_Topic                                     Topic_Keywords  \\\n",
       "0             1.0  group, congress, incident, clash, person, odis...   \n",
       "1             0.0  house, afiz, family, accuse, set, mob, death, ...   \n",
       "2             2.0  cpi, open, office, jitendra, kamalpur, march, ...   \n",
       "3             3.0  student, today, protest, school, due, teaching...   \n",
       "4             1.0  group, congress, incident, clash, person, odis...   \n",
       "5             1.0  group, congress, incident, clash, person, odis...   \n",
       "\n",
       "   Num_Documents  Perc_Documents  \\\n",
       "0            1.0          0.1667   \n",
       "1            3.0          0.5000   \n",
       "2            1.0          0.1667   \n",
       "3            1.0          0.1667   \n",
       "4            NaN             NaN   \n",
       "5            NaN             NaN   \n",
       "\n",
       "                                                Text  \n",
       "0  [odisha, sun, time, bureau, brahmapur, mar, to...  \n",
       "1  [abhijit, talukdar, march, tension, grip, kamr...  \n",
       "2  [tripura, news, cpi, supporter, injure, jiten,...  \n",
       "3  [excelsior, correspondent, doda, mar, student,...  \n",
       "4  [jeypore, koraput, administration, today, proh...  \n",
       "5  [bhubaneswar, commissionerate, arrest, person,...  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Number of Documents for Each Topic\n",
    "topic_counts = df_topic_sents_keywords['Dominant_Topic'].value_counts()\n",
    "\n",
    "# Percentage of Documents for Each Topic\n",
    "topic_contribution = round(topic_counts/topic_counts.sum(), 4)\n",
    "\n",
    "# Topic Number and Keywords\n",
    "topic_num_keywords = df_topic_sents_keywords[['Dominant_Topic', 'Topic_Keywords']]\n",
    "\n",
    "# Concatenate Column wise\n",
    "df_dominant_topics = pd.concat([topic_num_keywords, topic_counts, topic_contribution], axis=1)\n",
    "\n",
    "contents = pd.Series(texts)\n",
    "df_dominant_topics = pd.concat([df_dominant_topics, contents], axis=1)\n",
    "\n",
    "# Change Column names\n",
    "df_dominant_topics.columns = ['Dominant_Topic', 'Topic_Keywords', 'Num_Documents', 'Perc_Documents', 'Text']\n",
    "\n",
    "\n",
    "# Show\n",
    "df_dominant_topics.to_csv('output/topic_modelling.csv')\n",
    "\n",
    "pd.set_option('display.max_rows', 500)\n",
    "df_dominant_topics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "grouped = df_dominant_topics.groupby('Dominant_Topic')\n",
    "df0 = grouped.get_group(2.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Dominant_Topic</th>\n",
       "      <th>Topic_Keywords</th>\n",
       "      <th>Num_Documents</th>\n",
       "      <th>Perc_Documents</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.0</td>\n",
       "      <td>cpi, open, office, jitendra, kamalpur, march, ...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.1667</td>\n",
       "      <td>[tripura, news, cpi, supporter, injure, jiten,...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Dominant_Topic                                     Topic_Keywords  \\\n",
       "2             2.0  cpi, open, office, jitendra, kamalpur, march, ...   \n",
       "\n",
       "   Num_Documents  Perc_Documents  \\\n",
       "2            1.0          0.1667   \n",
       "\n",
       "                                                Text  \n",
       "2  [tripura, news, cpi, supporter, injure, jiten,...  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}