Lyrics Analysis II.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "import pickle\n",
    "from wordcloud import WordCloud, STOPWORDS \n",
    "import matplotlib.pyplot as plt \n",
    "import pandas as pd \n",
    "import re\n",
    "from nltk.corpus import stopwords\n",
    "from textblob import TextBlob\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "from nltk.corpus import cmudict\n",
    "d = cmudict.dict()\n",
    "from textstat.textstat import textstatistics, easy_word_set, legacy_round \n",
    "\n",
    "from nltk.corpus import wordnet as wn\n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "from nltk import word_tokenize, pos_tag\n",
    "from collections import defaultdict\n",
    "\n",
    "tag_map = defaultdict(lambda : wn.NOUN)\n",
    "tag_map['J'] = wn.ADJ\n",
    "tag_map['V'] = wn.VERB\n",
    "tag_map['R'] = wn.ADV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "data=pickle.load(open('MSD.pickle','rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def text_processing(raw):\n",
    "    clean=re.sub('[\\n]',' ',raw.lower())\n",
    "    clean=re.sub('[^a-zA-Z]',' ',raw.lower())\n",
    "    return clean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tag_lemma(text):\n",
    "    tokens = word_tokenize(text.lower())\n",
    "    lmtzr = WordNetLemmatizer()\n",
    "    p=[]\n",
    "    for token, tag in pos_tag(tokens):\n",
    "        lemma = str(lmtzr.lemmatize(token, tag_map[tag[0]]))\n",
    "        p.append(lemma)        \n",
    "    return ' '.join(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_lines(lines):\n",
    "    c=0\n",
    "    for i in lines:\n",
    "        if i!='':\n",
    "            c=c+1\n",
    "    return c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def syllables_count(word): \n",
    "    return textstatistics().syllable_count(word) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "def unique_lines_more_than_once(lines):\n",
    "    l=[]\n",
    "    p=0\n",
    "    for j in range(len(lines)):\n",
    "            if lines[j]!='':\n",
    "                  l.append(lines[j])\n",
    "    from itertools import groupby\n",
    "    c=([len(list(group)) for key, group in groupby(l)])\n",
    "    #count=0\n",
    "    #for i in range(len(c)):\n",
    "        #if c[i]>1:\n",
    "            #count=count+1\n",
    "    #return count\n",
    "    return len(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1979\n"
     ]
    }
   ],
   "source": [
    "with open('msd_read.csv', 'a',errors='replace') as the_file:\n",
    "    the_file.write('Year,FK Score,SMOG Grade\\n')\n",
    "    for year in data:\n",
    "        lines=0\n",
    "        unique=0\n",
    "        line=0\n",
    "        for i in range(len(data[year])):\n",
    "            #total no of unique lines that appear more than once/ total no of lines \n",
    "            #unique=unique_lines_more_than_once(data[year][i].splitlines())+unique\n",
    "            lines=count_lines(data[year][i].splitlines())+lines\n",
    "            #total no of unique lines that appear more than once/ total no of unique lines \n",
    "            #l=unique_lines_more_than_once(data[year][i].splitlines())\n",
    "            #unique=u+unique\n",
    "            #line=l+line\n",
    "        # 1- total number of unique lines/ total number of lines\n",
    "        #repeat=1-(line/lines)\n",
    "        w=tag_lemma(text_processing(''.join(data[year]))).split()\n",
    "        words=len(w)\n",
    "        syll=0\n",
    "        poll=0\n",
    "        for i in range(len(w)):\n",
    "            syll=syllables_count(w[i])+syll\n",
    "            if syllables_count(w[i])>2:\n",
    "                poll=poll+1\n",
    "        fk=(0.39*(words/lines))+(11.8*(syll/words))-15.59\n",
    "        smog=1.0430*np.sqrt(poll*30/lines)+3.1291\n",
    "        #print(str(year)+','+str(fk)+'\\n')\n",
    "        the_file.write(str(year)+','+str(fk)+','+str(smog)+'\\n')\n",
    "    print(year)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.read_csv('msd_read.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x190643cf60>]"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xm0HGWd//H3hyTABYIBEgQCISgYQFACl0VhBkQQ0CPggqKOoqK4MKO45GfQ0XEZRxSVRZwDGRaBQVZDQBQRhIDAACYQCASDIIiEKEESEiCEJHx/fzzVnc7ldnf1vV3dfft+Xuf0SXetT1Xq1reetRQRmJmZAazT7gSYmVnncFAwM7MyBwUzMytzUDAzszIHBTMzK3NQMDOzMgcFswZJ+pCk39aY/0+S5rcyTWbN4qDQpSTNlLRY0nrtTkszZMfzoqTnJD0tabqkLVuw34mSQtLI0rSIuCgi3laxTEjavmL+7yNiUtFpK1q7znkjsjR+ot3p6CYOCl1I0kTgn4AADi9oHyPrL9V0/xoRGwHbAxsBP2xDGoab0jl/HTAGOKXRDbTpWrEBclDoTh8B7gB+BhxTmihpH0l/kzSiYtq7JN2XfV9H0lRJj0j6h6TLJG2azSs9MR8r6XHgxmz65dk2n5V0i6TXV2x7M0m/lLRU0h8k/aekWyvm7yjpeknPSJov6X15Di4ilgAzgN0qtpUn7cdJelLSQklfyrMucEv275LsiflNkj5aOg5Jpfn3ZvPfL+kASU9k86dKuqIy/ZJOk3R69v1Vks7J0rQgO0cjsnkflXSrpB9mub5HJR1WsZ1a624v6ebs/+VpSZdm0yXpFElPZfPuk7RLjnP+DPALYJdsO+tl6Xpc0t8lnSmpJ5t3gKQnJH1F0t+A87LpR0iak10Pj0g6dDDnQNJ3SQ8/Z2Tn/ox6x2E5RIQ/XfYBHgY+C+wBrAReXTHvEeDgit+XA1Oz7yeQgsnWwHrAWcDF2byJpJzHBcCGQE82/ePA6Gz5U4E5Fdu+JPtsAOwM/BW4NZu3Yfb7Y8BIYHfgaeD1VY5pJvCJ7PtmwA3AVRXz86T94my/uwKLgIMaWHdkxb4+WjqO7HcA21f8PgB4Ivu+LfACsHH2ewSwENgn+z0j29+GwObAXcCnKvazEvhktt5ngCcB5Vj3YuBrpAe/9YH9sumHALNJT/0CdgK2zHHOx5IeBC7Mfp8KXA1smv3//xL4XsXxrwK+n53PHmAv4Fng4CxN44Edm3AOymn0p0n3j3YnwJ8m/4fCftkf0djs9x+BL1TM/0/g3Oz7aOB5YNvs94PAWyuW3TLb1siKm+Nraux7TLbMq7I/4JXApD77LgWF9wO/77P+WcB/VNn2TNLN9dlsH3OACRXz86R9x4r5PwDOaWDdAQWF7PetwEey7wcDj2TfXw2sIAuw2bQPADdV7OfhinkbZPvaIse6FwDTgK37nMcDgYeAfYB16lxLpXO+BFgAXASMIwWT54HXViz7JuDRiuN/CVi/z//tKf3sY8DnoCKNDgpN/Lisr/scA/w2Ip7Ofv88m3ZKxe/bJX0GeDdwd0T8JZu3LXClpJcrtrea9Idb8tfSlyyL/13gKNLNorTeWNLT4cjK5ft83xbYW9KSimkjgQtrHNvnIuJsSbsC15Ce7B8fSNqBv5ByDHnXHYyfk250FwAfzH6X9jsKWCiptOw6fdL5t9KXiHghW24j0hN6rXX/H/Ad4C5Ji4EfRcS5EXFjVszyU2CCpCuBL0fE0ipp/1xEnF05QdLmpJvz7Ip9i/QgULIoIl6s+L0N8Ot+tj+Yc2AFcFDoIlmZ7vuAEVlZLqTs+xhJb4yIeyNinqS/AIex9g0K0h/ixyPitn62PTH7Wjms7geBI4CDgMdIOYTFpBvEIlIRwtakJ1NIN4bKfd0cEQc3epwRMVfSfwI/lbR7pEfGPGnfhpRzAphAKoYopaXauts2mr5+XA78SNLWwLtIT9Wl/a4g5epWNbjNmutGxN9IRS5I2g+4QdItEfFwRJwOnJ7d3C8DpgBfb2DfTwPLSUV9C6os03f45b8Cr230OHLwMM9N5orm7nIk6Ql3Z1Il7G6kMuPfkyqfS34OfA74Z9INq+RM4LulG6GkcZKOqLG/0aQ/6H+Qnhz/qzQjIlYD04FvStpA0o590nAN8DpJH5Y0KvvsKWmnnMd6Pqn8udS6Kk/av56l5fWkuoxLc6y7iJQDek2NtPy91vyIWEQq5jiPVMTyYDZ9IfBbUsDYWKnC+7WS9q938PXWlXRUFoQgBeoAVmfneG9Jo0hFQC+SrpncIuJl4H+AU7LAgqTxkg6psdo5wMckvTVL63hJOw7mHGRqnntrnINCdzkGOC8iHo+Iv5U+wBnAh7SmaeDFpHLfGyuKmQBOI1Ue/lbSMlLl69419ncBqRhmATAvW77Sv5JyD38jFQtdTAoiRMQy4G3A0aQn9r+xpmKyroh4CTidNU+4edJ+M6kS/nfADyPit/XWjYgXSEVkt0laImmffpLzTeD8bH61FlQ/J+Woft5n+keAdUnnbzFwBalOI49a6+4J3CnpuezYPh8RjwIbk27oi0n/d/9gYE17v0I6l3dIWkqq+K/aNyMi7iIF4lNI9UI3k4qO6h1HPacB781aJp0+gOOwPko1+GaFk/R9UgXhMXUXbu5+JwKPAqMGWERhNmw4p2CFUeqH8IbUNF57AccCV7Y7XWZWnSuarUijSUVGWwFPAT8CrmprisysJhcfmZlZmYuPzMysrLDiI0nrk8aNWS/bzxUR8R/9LPc+UuuNAO6NiA/W2u7YsWNj4sSJTU+vmVk3mz179tMRMa7eckXWKawADoyI57I20bdKujYiys0WJe0AnAjsGxGLS22ea5k4cSKzZs0qLtVmZl0o67RaV2FBIetl+lz2c1T26VuB8UngpxGxOFvnqaLSY2Zm9RVapyBphKQ5pJYn10fEnX0WeR2pV+ttku4oDaXbz3aOkzRL0qxFixYVmWQzs2Gt0KAQEasjYjfS+Dd76ZXjto8EdiD1rv0AcLakMf1sZ1pE9EZE77hxdYvEzMxsgFrS+ijSS1FmAn1zAk+QxsRfmXXBn08KEmZm1gaFBYVsULEx2fce0rgvf+yz2AzgLdkyY0nFSX8uKk1mZlZbka2PtiQNEjaCFHwui4hrJH0bmBURVwPXAW+TNI80UuOUiPhHgWkyM7MahlyP5t7e3nCTVDOzxkiaHRG99Zbz2Ec2aDPuWcDJ183nySXL2WpMD1MOmcSRk8e3O1lmNgAOCjYoM+5ZwInT57J8ZXpPy4Ilyzlx+lwABwazIchjH9mgnHzd/HJAKFm+cjUnXze/TSkys8FwULBBeXLJ8oamm1lnc1CwQdlqTE9D082sszko2KBMOWQSPaNGrDWtZ9QIphxS9XW9ZtbBXNFsg1KqTHbrI7Pu4KBgg3bk5PEOAmZdwsVHZmZW5qBgZmZlDgpmZlbmoGBmZmUOCmZmVuagYGZmZQ4KZmZW5qBgZmZlDgpmZlbmoGBmZmUOCmZmVlZYUJC0vqS7JN0r6QFJ36qx7HslhaS67w81M7PiFDkg3grgwIh4TtIo4FZJ10bEHZULSRoNfA64s8C0mJlZDoXlFCJ5Lvs5KvtEP4t+B/gB8GJRaTEzs3wKrVOQNELSHOAp4PqIuLPP/MnANhFxTZ3tHCdplqRZixYtKjDFZmbDW6FBISJWR8RuwNbAXpJ2Kc2TtA5wCvClHNuZFhG9EdE7bty44hJsZjbMtaT1UUQsAWYCh1ZMHg3sAsyU9BiwD3C1K5vNzNqnyNZH4ySNyb73AAcBfyzNj4hnI2JsREyMiInAHcDhETGrqDSZmVltReYUtgRuknQf8AdSncI1kr4t6fAC92tmZgNUWJPUiLgPmNzP9G9UWf6AotJiZmb5uEezmZmVOSiYmVmZg4KZmZU5KJiZWZmDgpmZlTkomJlZmYOCmZmVOSiYmVmZg4KZmZU5KJiZWZmDgpmZlRX5Ok4zM2uCGfcs4OTr5vPkkuVsNaaHKYdM4sjJ4wvZl4OCmVkHm3HPAk6cPpflK1cDsGDJck6cPhegkMDg4iMzsw528nXzywGhZPnK1Zx83fxC9uegYGbWwZ5csryh6YPloGBm1sG2GtPT0PTBqhsUJP0uzzQzM2u+KYdMomfUiLWm9YwawZRDJhWyv6oVzZLWBzYAxkraBFA2a2Ngq0JSY2ZmaylVJndC66NPASeQAsBs1gSFpcBPC0mNmZm9wpGTxxcWBPqqGhQi4jTgNEn/FhE/aXTDWU7jFmC9bD9XRMR/9Fnmi8AngFXAIuDjEfGXRvdlZmbNUbefQkT8RNKbgYmVy0fEBXVWXQEcGBHPSRoF3Crp2oi4o2KZe4DeiHhB0meAHwDvb/QgzMysOeoGBUkXAq8F5gClxrIB1AwKERHAc9nPUdkn+ixzU8XPO4B/yZVqMzMrRJ4ezb3AztlNviGSRpDqI7YHfhoRd9ZY/Fjg2irbOQ44DmDChAmNJsPMzHLK00/hfmCLgWw8IlZHxG7A1sBeknbpbzlJ/0IKPidX2c60iOiNiN5x48YNJClmZpZDnpzCWGCepLtI9QQARMTheXcSEUskzQQOJQWZMkkHAV8D9o+IFf2sbmZmLZInKHxzIBuWNA5YmQWEHuAg4Pt9lpkMnAUcGhFPDWQ/ZmbWPHlaH90saVtgh4i4QdIGwIh66wFbAudn9QrrAJdFxDWSvg3MioirScVFGwGXSwJ4vJEciJmZNVee1kefJFXybkpqhTQeOBN4a631IuI+YHI/079R8f2gBtNrZmYFylPRfDywL6knMxHxJ2DzIhNlZmbtkScorIiIl0o/JI2kT38DMzPrDnmCws2Svgr0SDoYuBz4ZbHJMjOzdsgTFKaSxiWaSxok79fAvxeZKDMza488TVJ7gHMj4n+g3Eu5B3ihyIRZ52nly8PNrD3y5BR+RwoCJT3ADcUkxzpV6eXhC5YsJ1jz8vAZ9yxod9LMrInyBIX1I6I0sB3Z9w2KS5J1ola/PNzM2iNPUHhe0u6lH5L2AIp5Y7R1rFa/PNzM2iNPncLnST2On8x+b4nfeTDsbDWmhwX9BICiXh5uZu1RM6cgaR1gXWBH4DPAZ4GdImJ2C9JmHaTVLw83s/aomVOIiJcl/Sgi3kSf0U1teGn1y8PNrD3yFB/9VtJ7gOkDedGOdY9WvjzczNojT1D4IrAhsFrSckCkt21uXGjKzMys5fIMnT26FQmx4ced4cw6T90mqUr+RdLXs9/bSNqr+KRZN3NnOLPOlKefwn8DbwI+mP1+DvhpYSmyhsy4ZwH7nnQj2039FfuedOOQuam6M5xZZ8pTp7B3ROwu6R6AiFgsad2C02U5lJ62SzfX0tM20PHFMO4MZ9aZ8uQUVmaD4AWU3738cqGpslyG8tN2tU5v7gxn1l55gsLpwJXA5pK+C9wK/Fe9lSStL+kuSfdKekDSt/pZZj1Jl0p6WNKdkiY2mP5hoVoR0VB+2nZnOLPOlKf10UWSZpPeySzgyIh4MMe2VwAHRsRzkkYBt0q6NiLuqFjmWGBxRGwv6Wjg+3gIjbXUKiIaykNPuDOcWWeqGhQkrQ98Gtie9IKdsyJiVd4NZx3dSqOrjso+fTu/HQF8M/t+BXCGJLmT3Bq1ioimHDJprYABQ+tp253hzDpPreKj84FeUkA4DPhhoxuXNELSHOAp4PqIuLPPIuOBvwJkAedZYLNG99PNahURHTl5PN97966MH9ODgPFjevjeu3f1jdbMBqxW8dHOEbErgKRzgLsa3XhErAZ2kzQGuFLSLhFROYaS+lut7wRJxwHHAUyYMKHRZAxp9YqI/LRtZs1UK6ewsvSlkWKj/kTEEmAmcGifWU8A2wBIGgm8Cnimn/WnRURvRPSOGzduMEkZclwha2atVCsovFHS0uyzDHhD6bukpfU2LGlclkNAUg9wEPDHPotdDRyTfX8vcKPrE9bmIiIza6WqxUcRMaLavJy2BM7P+jisA1wWEddI+jYwKyKuBs4BLpT0MCmHcPQg99mVXESUj8dSMhu8PD2aByQi7gMm9zP9GxXfXwSOKioNNnwM5d7dZp0kT+c1s443lHt3m3USBwXrCkO5d7dZJ8kzdPaG2buakfQ6SYdnPZTNOobHUjJrjjw5hVuA9SWNB34HfAz4WZGJMmuUm+6aNUeeimZFxAuSjgV+EhE/KA2jbdYp6o2l5JZJZvnkCgqS3gR8iDSAXd71zFqqWtNdt0wyyy9P8dEJwInAlRHxgKTXADcVmyyz5nHLJLP88gydfTNwM0BW4fx0RHyu6ISZNYtbJpnll6f10c8lbSxpQ2AeMF/SlOKTZt2gE94h7ZZJZvnlKT7aOSKWAkcCvwYmAB8uNFXWFUpl+QuWLCdYU5bf6sDglklm+eUJCqOyfglHAldFxEr6Gd7arK9OKcv3oIJm+eVpRXQW8BhwL3CLpG2BuqOkmnVSWb4HFTTLJ09F8+nA6RWT/iLpLcUlybrFUH6HtFmrdUpfmjwVza+WdI6ka7PfO7PmHQhmVbks3yyfTql/g3x1Cj8DrgO2yn4/ROq7YFaTy/LN8umU+jfIV6cwNiIuk3QipFdzSlpdbyUzcFm+WR6dVP+WJyg8L2kzshZHkvYBni00VU3WKWV1Zmb96aT6tzzFR18kvUv5tZJuAy4A/q3QVDVRJ5XVmZn1p5Pq3/K0Prpb0v7AJEDA/KyvwpBQq6zOuQUz6wT1Rvltpbyjne4FTMyW310SEXFBrRUkbUPKVWwBvAxMi4jT+izzKuB/Sb2kRwI/jIjzGjqCOtpRVufiKjNrVKfUv9UNCpIuBF4LzAFKj9xBuuHXsgr4UpbTGA3MlnR9RMyrWOZ4YF5EvFPSONK4ShdFxEsNH0kVrS6r8zDNZjaU5ckp9JLGP2poaIuIWAgszL4vk/QgMJ40qF55MWC0JAEbAc+QgknTTDlk0lo3aSi2rM7FVWY2lOUJCveTioAWDnQnkiYCk4E7+8w6g1SJ/SQwGnh/RLzcz/rHAccBTJgwoaF9t7qsrpOalpmZNSpXPwVgnqS7gBWliRFxeJ4dSNoI+AVwQjbaaqVDSMVSB5KKqK6X9Pu+y0XENGAaQG9vb8OD8bWyrK6TmpaZmTUqT1D45kA3no2u+gvgooiY3s8iHwNOyoqmHpb0KLAjcNdA99lurS6uMjNrplxvXstGRt0hIm6QtAEwot56WT3BOcCDEfHjKos9DrwV+L2kV5Oavf45d+o7UCc1LTMza1Se1kefJJXnb0oq4hkPnEm6mdeyL+llPHMlzcmmfZXU/JSIOBP4DvAzSXNJfSC+EhFPD+A4OkqnNC0zM2tUnuKj40n9FO4EiIg/Sdq83koRcSvpRl9rmSeBt+VIgw0z7uth1h55gsKKiHgplQaBpJH4zWtWIPf1MGufPEHhZklfBXokHQx8Fvhlscmy4azeMMLOQZgVJ8+AeFOBRcBc4FPAr4F/LzJRNrxV69NRyjF4cEOz4uQJCkcAF0TEURHx3oj4n0Z7N5s1olqfjhFSx7yIxKxb5QkKhwMPSbpQ0juyOgWzwlQbRnh1lWeRwfYWn3HPAvY96Ua2m/or9j3pRuc8bFirGxQi4mPA9sDlwAeBRySdXXTCbPiq9hrP8VVyEIPpLe73bZitLddTf0SslHQtqdVRD6lI6RNFJsyGt2p9PZrdW9wDGJqtLU/ntUOBo4G3ADOBs4H3FZsss1cqore4BzC0ZhvqfWzy5BQ+ClwCfCoiVtRZ1qxQze4t7gEMrZm6oY9NnjqFoyNihgOCdaNOejeuDX31+tgMBVVzCpJujYj9JC1j7R7MAiIiNi48dWYF8wCG1kzdUBxZNShExH7Zv6Nblxyz1vMAhtYs3VAcWbf4KHtHc91pZmbDXTcUR+apaH595Y+s89oexSTHzGzo6obiyFp1CieS3n/QI6n0ekwBL5G9GnMoOvZYOPfcga27004gwbx5sOmmsO22sHIlHHAAfPrTsPPOaf5QNtSb05m121AvjlS9YYwkfS8iTmxReurq7e2NWbNmDXj9Cy+Ej3ykiQkq0A47wFFHwTvfCXvvXXzA6ducDlLW93vv3nVIX+RmBpJmR0Rv3eWqBQVJO0bEHyXt3t/8iLh7kGkckMEGhUasXg1//StccglEwGabwY03wqWXtmT3hVp33ZS7+cxnYMcd07R9T7qx30qy8WN6uG3qgS1OoZk1UzOCwrSIOE7STf3Mjohoy12ilUGhSM8/DxdfDKedBvffn6a9+tXw97+3N13NsssucM45sNde7U6JmUETgkLFhtaPiBfrTWuVbgkKRVqyBC67DM4/H26/vd2pKcZdd8Gee7Y7FTacDbX6t2YGhbsjYvd60/pZbxvgAmAL4GVgWkSc1s9yBwCnAqOApyNi/1rbdVAoTp46hccfhylTUtDpRqNHw8yZsHvNq9uGi2o3/qFY/5Y3KFTtpyBpC0l7kFofTZa0e/Y5ANggRxpWAV+KiJ2AfYDjJe3cZx9jgP8GDo+I1wNH5diuFaTakNWVF/mECalOJaL5n/nz4Q1vaN/xAyxbBnvskSr1m/155zvhscfae3yWX61h1bthOItqatUpHEMaDK8XqHw0XwqcHxHTG9qRdBVwRkRcXzHts8BWEZH79Z7OKdhA3H8/TJ0Kv/pVu1NSjG9/Gz73OXjVq9qdku5Rq+HFk1mg6EvAoye9o/C0DcSgcwoRcX5EvAX4aES8peJzxAACwkRgMnBnn1mvAzaRNFPSbEn9NhaVdJykWZJmLVq0qJFdmwGp4vuaa4rJ4UyfDtts097j+8Y3YMyYYnI4p54KL7alBrG9ao1jVG3YiqE0nEU1eV7HeZukc7KX7CBpZ0nH5t2BpI2AXwAnRMTSPrNLvaPfARwCfF3S6/puIyKmRURvRPSOGzcu767NWuJd70p1LYMNLlfevYA3f+9GJn7lV7z5ezcyffYCzjoLNtqovcf3hS9AT08xAeeUU2DVqvYeXzW1bvzdMJxFNXmCwnnAdcBW2e+HgBPybFzSKFJAuKhK7uIJ4DcR8XxEPA3cArwxz7bNukl/5ddfvXIum++5gGXLmp+7Wb0azjyz3UcNX/wijBrV3ECzxRapKG3mTHj55YGnrdaNP0/921CVp/XRHyJiT0n3RMTkbNqciNitznoCzgeeiYh+g4iknYAzSLmEdYG7gKMj4v5q23WdgrVKK5scdlPHwdWrYcaMNHLACy+0OzWDN2r0ClYuW48R663igENX8LXPb8iuu6bOrI2OMtDOZqx56xTyDIj3vKTNyN6pIGkf4Nkc6+0LfBiYK2lONu2rwASAiDgzIh6U9BvgPlKz1bNrBQSzVmn1G7S6YRz+khEj4D3vSZ9me+kluOqqNMrA3Xen1ly77QZz5tRddcBWLlsPgNUrRvK7q0byu6sGs7XxwHjWWf8llh04j+PvWsR1O23Adq/ahA03hEmTYJ992ltkmCensDvwE2AX4H5gHPDeiLiv+OS9knMK1gqtfnLvppzCULV8eSpymjsXHngArrsOVqxInUFbbeRIuO22VMR33nlrpt9778CbbTctpxARd0vaH5hEanE1PyJWDixZ1ipDrbdlp2n1k/uUQyb12xmqqIpLXx+v1NMDhx2WPs324osw8WN3sGLBJqx+YV1GbPAS6261mNVLe1j+yOa88NCWay0/ZQqMH59GXt5kE1i8OE2v8wzfFHWDgqT1gc8C+5GKkH4v6cx2DXNh9XXDy8PbrdVv0GrlOPy+Plpv/fXhtW98gQXb/uMV8yb98z+4beqW/awFX/5y+rRSntZHF5BetPMTUqXwzoDfvNbBurm3Zau0o8nhkZPHc9vUA3n0pHdw29QDC7tB+/poj6HSjDVPRfOkiKhsJnqTpHuLSpANXjdVWrZLN7xBqxpfH+0xVK6pPEHhHkn7RMQdAJL2Bm4rNlk2GN3w8vBO0Clv0Gp2+b+vj/bplGuqljzFR3sDt0t6TNJjwP8B+0uaK6ktLZCstqGSTbX6ag3KNlC+PqyWPDmFQwtPhTXVUMmmWn21yv8H+v85HK8Pt7bKL0+T1L9I2g/YISLOkzQWGB0RjxafPBuooZBNtfqKKv8fTtdHvdZWDhhry9Mk9T9Iw2dPIo2DtC7wv6Qey2ZWoG4u/2/Vzbheays3z11bnjqFdwGHA88DRMSTwOgiE2VmSbeW/xdRV1JNrdyWm+e+Up6g8FKksTBKYx9tWGySzKykW0fjbOXNuNYQ2G6e+0p5Kpovk3QWMEbSJ4GPA2cXmywzK+nG8v9W3oxrDSFy8nXzu7Z4bqDyVDT/UNLBpNdwTgK+UflKTTOzRrWyrqRea6tWjjk1FOTJKZAFgesBJI2Q9KGIuKjQlFlh3NrC2q3VAwBWy20Nx+a59VQNCpI2Bo4nDQB+NSkoHA9MAeYADgpDkAdDK56Dbn2ddDPuxuK5waj6PgVJVwGLST2Y3wpsQmqO+vmIKPCVFrX5fQqD43H7i9U36EJ6Am5H5bCDk1VqxvsUXhMRu2YbOxt4GpgQEcualEZrA7e2KFYRPZAHotU5Qgeg7lGrSWr5RToRsRp41AFh6KvVPM8Gr1OCbiubfLayz4EVr1ZQeKOkpdlnGfCG0ndJS1uVQGuubu0M1Sk6JegWEZxm3LOAfU+6ke2m/op9T7qxfNN3B7DuUrX4KCJGVJuXh6RtSC/o2QJ4GZgWEadVWXZP4A7g/RFxxWD2a7V1UgVfN2p1q5pqBtrks1oxUK3iqE7JHVlz5GqSOkCrgC9l73geDcyWdH1EzKtcSNII4PvAdQWmxSq4tUVxOiXoDiQ41brx18oNtHp8JtdfFKuwoBARC4GF2fdlkh4kNW+d12fRfwN+AexZVFqK4AvTqumEoDuQ4FTrxl8rN3DK+3drWe7ITaqLV2ROoUzSRGAycGef6eNJA+4dSI2gIOk44DiACRMmFJXM3Hxh2lDQaHCqdeOvlRtoZe6oU1p3dbPCg4KkjUg5gRMiom8F9anAVyJitaSq24iIacA0SP0UikprX9VyA74wrRvVuvHXK45qVe7I9RfFKzQoSBpFCggXRcT0fhbpBS7JAsJY4O2SVkXEjCLTlYcr1my4qXWKxPgnAAAJo0lEQVTjH0xuoJlFrd38folOUVhQULrTnwM8GBE/7m+ZiNiuYvmfAdd0QkCAzqpYM2uFejf+geQGml3U2imtu7pZkTmFfYEPA3MllYbF+CowASAizixw34PWKRVrZq3U7GKgZhe1dkrrrm5WZOujW4HqFQWvXP6jRaVlIDqlYs1sKCuiqLVW4HKrwMFrSeujoahTKtbMhrJWFrW6VWBz5Hkd57DUra9BtOaoNuTDcE1HNa0cVsXDbTSHcwo1ODdg/emUJ9JOSUctrSxqdavA5nBQMGtQp/RT6ZR01DPQVkuNBhK3CmwOFx+ZNahTnkg7JR3NNtChuD0CcHM4KJg1qFOGx+6UdDTbQOsGXA/YHC4+MmtQp3Sg6pR0NNtgckCuBxw8BwWzBnVKP5VOSUezuW6gvRTRsvHlmqK3tzdmzZrV7mSYWUH6tqqClANyUdDgSJodEb31lnNOwcw6SrfmgIYKBwUz6ziuG2gfB4UW8rgsZtbpHBRaZCj0PjUzc1BokaHS+9SK5dyidToHhRbp1t6nlp9zizYUDPseza0aZbJbe59afh7F04aCYR0UBjrGykB4XBZzbtGGgmEdFFr55OZxWcy5RRsKhnWdQquf3Nz2enjr1rGKrLsUllOQtI2kmyQ9KOkBSZ/vZ5kPSbov+9wu6Y1Fpac/fnKzVnJu0YaCInMKq4AvRcTdkkYDsyVdHxHzKpZ5FNg/IhZLOgyYBuxdYJrW4ic3azXnFq3TFRYUImIhsDD7vkzSg8B4YF7FMrdXrHIHsHVR6emPx1gxM1tbS+oUJE0EJgN31ljsWODaKusfBxwHMGHChKamzU9uZmZrFN76SNJGwC+AEyJiaZVl3kIKCl/pb35ETIuI3ojoHTduXHGJNTMb5grNKUgaRQoIF0XE9CrLvAE4GzgsIv5RZHrMzKy2IlsfCTgHeDAiflxlmQnAdODDEfFQUWkxM7N8iswp7At8GJgraU427avABICIOBP4BrAZ8N8phrAqz5uBzMysGEW2ProVUJ1lPgF8oqg0DCUePdPMOsGw7tHcKTx6ppl1imE99lGn8OiZZtYpHBQ6gEfPNLNO4aDQATwGk5l1CgeFDuB3LZhZp3BFcwfwGExm1ikcFDqEx2Ays07g4iMzMytzUDAzszIHBTMzK3NQMDOzMgcFMzMrc1AwM7MyBwUzMytTRLQ7DQ2RtAj4S7vTMUhjgafbnYgO4vOxNp+PNXwu1jaY87FtRNR9n/GQCwrdQNIsv0xoDZ+Ptfl8rOFzsbZWnA8XH5mZWZmDgpmZlTkotMe0diegw/h8rM3nYw2fi7UVfj5cp2BmZmXOKZiZWZmDgpmZlTkoNImkcyU9Jen+imlvlPR/kuZK+qWkjbPpH5I0p+LzsqTdsnl7ZMs/LOl0SWrXMQ1Ug+dilKTzs+kPSjqxYp1DJc3PzsXUdhxLMzR4PtaVdF42/V5JB1Ss0w3XxjaSbsr+rx+Q9Pls+qaSrpf0p+zfTbLpyo71YUn3Sdq9YlvHZMv/SdIx7TqmwRjA+dgxu25WSPpyn2015+8lIvxpwgf4Z2B34P6KaX8A9s++fxz4Tj/r7Qr8ueL3XcCbAAHXAoe1+9iKPBfAB4FLsu8bAI8BE4ERwCPAa4B1gXuBndt9bC04H8cD52XfNwdmA+t00bWxJbB79n008BCwM/ADYGo2fSrw/ez727NjFbAPcGc2fVPgz9m/m2TfN2n38bXgfGwO7Al8F/hyxXaa9vfinEKTRMQtwDN9Jk8Cbsm+Xw+8p59VPwBcDCBpS2DjiPi/SP/TFwBHFpPi4jR4LgLYUNJIoAd4CVgK7AU8HBF/joiXgEuAI4pOexEaPB87A7/L1nsKWAL0dtG1sTAi7s6+LwMeBMaT/m/PzxY7nzXHdgRwQSR3AGOyc3EIcH1EPBMRi0nn8NAWHkpTNHo+IuKpiPgDsLLPppr29+KgUKz7gcOz70cB2/SzzPvJggLpYniiYt4T2bRuUO1cXAE8DywEHgd+GBHPkI77rxXrd9O5gOrn417gCEkjJW0H7JHN67prQ9JEYDJwJ/DqiFgI6UZJeiKG6tdB110fOc9HNU07Hw4Kxfo4cLyk2aSs4UuVMyXtDbwQEaWy5v7KiLulzXC1c7EXsBrYCtgO+JKk19Dd5wKqn49zSX/Qs4BTgduBVXTZ+ZC0EfAL4ISIWFpr0X6mRY3pQ1ID56PqJvqZNqDzMXIgK1k+EfFH4G0Akl4HvKPPIkezJpcA6WawdcXvrYEni0xjq9Q4Fx8EfhMRK4GnJN0G9JKeeipzVl1zLqD6+YiIVcAXSstJuh34E7CYLrk2JI0i3QAviojp2eS/S9oyIhZmxUNPZdOfoP/r4AnggD7TZxaZ7qI0eD6qqXaeGuacQoEkbZ79uw7w78CZFfPWIRUbXFKalmUTl0naJ2tZ8hHgqpYmuiA1zsXjwIFZK5MNSZWJfyRVxO4gaTtJ65IC6NWtT3kxqp0PSRtk5wFJBwOrImJet1wbWdrPAR6MiB9XzLoaKLUgOoY1x3Y18JHs+tgHeDY7F9cBb5O0SdYy523ZtCFlAOejmub9vbS79r1bPqQn/oWkCqAngGOBz5NaEzwEnETWgzxb/gDgjn6200sqb34EOKNynaHyaeRcABsBlwMPAPOAKRXbeXu2/CPA19p9XC06HxOB+aQKxxtIwx1307WxH6lY4z5gTvZ5O7AZqYL9T9m/m2bLC/hpdsxzgd6KbX0ceDj7fKzdx9ai87FFdg0tJTVCeILUAKFpfy8e5sLMzMpcfGRmZmUOCmZmVuagYGZmZQ4KZmZW5qBgZmZlDgpmNWTt42+VdFjFtPdJ+k0702VWFDdJNatD0i6kvhSTSaNRzgEOjYhHBrHNkZF6L5t1FAcFsxwk/YA0cN+GwLKI+E42hv/xpKGKbwf+NSJeljSNNFR2D3BpRHw728YTwFmk0TxPjYjL23AoZjV57COzfL4F3E0auK43yz28C3hzRKzKAsHRwM9J4+A/kw0HfpOkKyJiXrad5yNi33YcgFkeDgpmOUTE85IuBZ6LiBWSDiK97GRW9gK0HtYMXfwBSceS/r62Ir0joRQULm1tys0a46Bglt/L2QfSmDznRsTXKxeQtANpXKO9ImKJpP8F1q9Y5PmWpNRsgNz6yGxgbgDeJ2ksgKTNJE0ANgaWAUsr3hBmNmQ4p2A2ABExV9K3gBuy4a9XAp8mvRxnHmk00z8Dt7UvlWaNc+sjMzMrc/GRmZmVOSiYmVmZg4KZmZU5KJiZWZmDgpmZlTkomJlZmYOCmZmV/X+36TaFY2tUqwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Average Repetitiveness Score\n",
    "plt.xlabel('Year')\n",
    "plt.ylabel('Repetitiveness Percent')\n",
    "plt.title('Average Repetitiveness Percent')\n",
    "x=df['Year']\n",
    "y=df['Repetitiveness']\n",
    "bb=plt.scatter(x,y,label='Repetitiveness') \n",
    "z = np.polyfit(x, y, 1)\n",
    "p = np.poly1d(z)\n",
    "plt.plot(x,p(x),\"b--\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1991\n",
      "1984\n",
      "2005\n",
      "1965\n",
      "1986\n",
      "2007\n",
      "1985\n",
      "2000\n",
      "1969\n",
      "1973\n",
      "1994\n",
      "1996\n",
      "1974\n",
      "1967\n",
      "1990\n",
      "1995\n",
      "1999\n",
      "1970\n",
      "1975\n",
      "2004\n",
      "2002\n",
      "2003\n",
      "1981\n",
      "1993\n",
      "1983\n",
      "1982\n",
      "1966\n",
      "1971\n",
      "1968\n",
      "2001\n",
      "1972\n",
      "1989\n",
      "1976\n",
      "1987\n",
      "1988\n",
      "1998\n",
      "2008\n",
      "1992\n",
      "1980\n",
      "1997\n",
      "1977\n",
      "2009\n",
      "2006\n",
      "2010\n",
      "1978\n",
      "1979\n"
     ]
    }
   ],
   "source": [
    "with open('msd_repeat.csv', 'a',errors='replace') as the_file:\n",
    "    the_file.write('Year,Repetitiveness\\n')\n",
    "    for year in data:\n",
    "        lines=0\n",
    "        unique=0\n",
    "        line=0\n",
    "        for i in range(len(data[year])):\n",
    "            #total no of unique lines that appear more than once/ total no of lines \n",
    "            #unique=unique_lines_more_than_once(data[year][i].splitlines())+unique\n",
    "            lines=count_lines(data[year][i].splitlines())+lines\n",
    "            #total no of unique lines that appear more than once/ total no of unique lines \n",
    "            l=unique_lines_more_than_once(data[year][i].splitlines())\n",
    "            #unique=u+unique\n",
    "            line=l+line\n",
    "        # 1- total number of unique lines/ total number of lines\n",
    "        print(year)\n",
    "        repeat=1-(line/lines)\n",
    "        the_file.write(str(year)+','+str(repeat*100)+'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}