Evaluation notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### for emotion detection\n",
    "https://github.com/monologg/GoEmotions-pytorch\n",
    "### For toxicity detection\n",
    "perspective api\n",
    "### For politeness detection\n",
    "https://github.com/AlafateABULIMITI/politeness-detection\n",
    "### For language quality\n",
    "https://github.com/WanzhengZhu/GPS/blob/master/language_quality.py\n",
    "### Other metrics are aleady added add two metric from the GPS model.\n",
    "### Implement GPS model as a baseline.\n",
    "save_path='../HULK_new/Counterspeech/Saved_Models'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from googleapiclient import discovery"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import time\n",
    "API_KEY = 'AIzaSyAUYdcELBp_FIPyiCQ6UfDEDIfDJDiShLE'\n",
    "\n",
    "client = discovery.build(\n",
    "      \"commentanalyzer\",\n",
    "      \"v1alpha1\",\n",
    "      developerKey=API_KEY,\n",
    "      discoveryServiceUrl=\"https://commentanalyzer.googleapis.com/$discovery/rest?version=v1alpha1\",\n",
    "      static_discovery=False,\n",
    "    )\n",
    "\n",
    "def get_non_toxicity_score(sentence):\n",
    "    analyze_request = {\n",
    "      'comment': { 'text': sentence },\n",
    "      'requestedAttributes': {'TOXICITY': {}}\n",
    "    }\n",
    "    response = client.comments().analyze(body=analyze_request).execute()\n",
    "    tox=response['attributeScores']['TOXICITY']['summaryScore']['value']\n",
    "    time.sleep(1.0)\n",
    "    return 1-tox"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.04108893999999996"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_non_toxicity_score(\"I hate niggers\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#BLEU, METEOR, Diversity, Novelty\n",
    "import glob\n",
    "import json\n",
    "from loguru import logger\n",
    "import pandas as pd \n",
    "from Generation.eval import *\n",
    "path_datasets = '/home/adarsh-binny' + '/HULK_new/Counterspeech/Datasets'\n",
    "path_result   = '/home/adarsh-binny' + '/HULK_new/Counterspeech/Results'\n",
    "save_path     = '/home/adarsh-binny' + '/HULK_new/Counterspeech/metrics_results/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def metrics(trained_on,tested_on):\n",
    "    train_path = glob.glob(path_datasets+'/*'+trained_on+'*/*rain*')[0]\n",
    "    gen        = glob.glob(path_result+'/*'+trained_on+'*'+tested_on+'*1628*')\n",
    "    ref        = glob.glob(path_result+'/*'+tested_on+'*'+'references'+'*')[0]\n",
    "    \n",
    "#     print(gen,ref)\n",
    "    with open(ref, 'r') as file:\n",
    "        ref_dict   = json.loads(file.read())\n",
    "        \n",
    "    scores = {}\n",
    "    for files in gen:\n",
    "        with open(files, 'r') as file:\n",
    "            gen_dict  = json.loads(file.read())\n",
    "        emotion   = gen_dict['params']['task_name'][0][1]\n",
    "        gpu_id    = gen_dict['params']['gpu_id']\n",
    "        hypo = []\n",
    "        refs = []\n",
    "        \n",
    "        for key in gen_dict['samples']:\n",
    "            for sentences in gen_dict['samples'][key]['counterspeech_model']:\n",
    "                hypo.append(sentences)\n",
    "                refs.append(ref_dict['samples'][key]['counterspeech_model'])\n",
    "        \n",
    "        train = pd.read_csv(train_path)\n",
    "        train_set = list(zip(train['initiator_message'].tolist(),train['reply_message'].tolist()))\n",
    "        params = [hypo,refs]\n",
    "        bleu, bleu_4, meteor_ = nltk_metrics(params)\n",
    "        train_corpus = training_corpus(train_set)\n",
    "        diversity, novelty = diversity_and_novelty(train_corpus,hypo)\n",
    "        data_dict = {\n",
    "                     'bleu':bleu,\n",
    "                     'bleu_4':bleu_4,\n",
    "                     'diversity':diversity,\n",
    "                     'novelty':novelty, \n",
    "                     'meteor':meteor_\n",
    "                   }\n",
    "        key = trained_on+'_'+tested_on+'_'+emotion+'_'+str(gpu_id)\n",
    "        scores[key] = data_dict\n",
    "        logger.info(f'Key:{key}--bleu :{bleu}--bleu_4:{bleu_4}--diversity:{diversity}--novelty:{novelty}--meteor:{meteor_}')\n",
    "    \n",
    "    json.dump(scores, open(save_path +tested_on+'.json','w'),indent = 4)\n",
    "                    \n",
    "    \n",
    "        \n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "****Reddit****"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-11 14:15:19.397 | INFO     | __main__:metrics:39 - Key:Reddit_Reddit_joy_1--bleu :0.19159508198821035--bleu_4:0.19159508198821035--diversity:0.7714839932389237--novelty:0.8665133391825054--meteor:0.15676525866753221\n",
      "/home/adarsh-binny/.conda/envs/test_env/lib/python3.7/site-packages/nltk/translate/bleu_score.py:516: UserWarning: \n",
      "The hypothesis contains 0 counts of 4-gram overlaps.\n",
      "Therefore the BLEU score evaluates to 0, independently of\n",
      "how many N-gram overlaps of lower order it contains.\n",
      "Consider using lower n-gram order or use SmoothingFunction()\n",
      "  warnings.warn(_msg)\n",
      "2021-08-11 14:59:40.330 | INFO     | __main__:metrics:39 - Key:Reddit_Reddit_toxic_1--bleu :0.2524448470938123--bleu_4:0.2524601042958655--diversity:0.7596694295375596--novelty:0.817504188579462--meteor:0.15125937613078166\n",
      "2021-08-11 15:53:26.998 | INFO     | __main__:metrics:39 - Key:Reddit_Reddit_joy_0--bleu :0.23484611013826465--bleu_4:0.23485025476819776--diversity:0.7425532033954653--novelty:0.8235129741410291--meteor:0.1708030453337513\n",
      "2021-08-11 16:59:00.615 | INFO     | __main__:metrics:39 - Key:Reddit_Reddit_anger_1--bleu :0.18716731595873118--bleu_4:0.18717024084885148--diversity:0.7952776168215064--novelty:0.8772981963923815--meteor:0.13964258453556538\n",
      "2021-08-11 18:21:32.207 | INFO     | __main__:metrics:39 - Key:Reddit_Reddit_sadness_0--bleu :0.16858535825164075--bleu_4:0.16858535825164075--diversity:0.7809108277084241--novelty:0.8817657691109974--meteor:0.16170848827884668\n",
      "2021-08-11 19:48:14.561 | INFO     | __main__:metrics:39 - Key:Reddit_Reddit_fear_0--bleu :0.16529066626954123--bleu_4:0.16529066626954123--diversity:0.7567048311240059--novelty:0.8806601379136259--meteor:0.1673527289617203\n"
     ]
    },
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: '/home/adarsh-binny/HULK_new/Counterspeech/metrics_results/Reddit.json'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-15-562649cb15f8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmetrics\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Reddit'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'Reddit'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-14-b84581d54223>\u001b[0m in \u001b[0;36mmetrics\u001b[0;34m(trained_on, tested_on)\u001b[0m\n\u001b[1;32m     39\u001b[0m         \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'Key:{key}--bleu :{bleu}--bleu_4:{bleu_4}--diversity:{diversity}--novelty:{novelty}--meteor:{meteor_}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 41\u001b[0;31m     \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscores\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msave_path\u001b[0m \u001b[0;34m+\u001b[0m\u001b[0mtested_on\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'.json'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'w'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mindent\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     42\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     43\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/home/adarsh-binny/HULK_new/Counterspeech/metrics_results/Reddit.json'"
     ]
    }
   ],
   "source": [
    "metrics('Reddit','Reddit')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "****CONAN****"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-11 20:05:30.253 | INFO     | __main__:metrics:39 - Key:CONAN_CONAN_fear_0--bleu :0.3628025900977262--bleu_4:0.3628025900977262--diversity:0.7905926672229553--novelty:0.8762991525843769--meteor:0.18265533594843028\n",
      "2021-08-11 20:09:17.151 | INFO     | __main__:metrics:39 - Key:CONAN_CONAN_joy_1--bleu :0.38790688956093866--bleu_4:0.38790688956093866--diversity:0.8103381803436333--novelty:0.877103428085856--meteor:0.17037657143026955\n",
      "2021-08-11 20:13:38.460 | INFO     | __main__:metrics:39 - Key:CONAN_CONAN_sadness_0--bleu :0.36060291577376724--bleu_4:0.36060291577376724--diversity:0.817835742407302--novelty:0.8833927203844146--meteor:0.17052843278504146\n",
      "2021-08-11 20:15:52.122 | INFO     | __main__:metrics:39 - Key:CONAN_CONAN_toxic_1--bleu :0.5009974364496044--bleu_4:0.5009974364496044--diversity:0.8087049867720634--novelty:0.84259901060552--meteor:0.1670292599359696\n",
      "2021-08-11 20:18:12.730 | INFO     | __main__:metrics:39 - Key:CONAN_CONAN_joy_0--bleu :0.5024940427181879--bleu_4:0.5025128542329175--diversity:0.7966439992002184--novelty:0.8368270669465472--meteor:0.17637176053283168\n"
     ]
    },
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: '/home/adarsh-binny/HULK_new/Counterspeech/metrics_results/CONAN.json'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-16-8d687e8c3365>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmetrics\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'CONAN'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'CONAN'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-14-b84581d54223>\u001b[0m in \u001b[0;36mmetrics\u001b[0;34m(trained_on, tested_on)\u001b[0m\n\u001b[1;32m     39\u001b[0m         \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'Key:{key}--bleu :{bleu}--bleu_4:{bleu_4}--diversity:{diversity}--novelty:{novelty}--meteor:{meteor_}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 41\u001b[0;31m     \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscores\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msave_path\u001b[0m \u001b[0;34m+\u001b[0m\u001b[0mtested_on\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'.json'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'w'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mindent\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     42\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     43\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/home/adarsh-binny/HULK_new/Counterspeech/metrics_results/CONAN.json'"
     ]
    }
   ],
   "source": [
    "metrics('CONAN','CONAN')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "****Gab****"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/adarsh-binny/.conda/envs/test_env/lib/python3.7/site-packages/nltk/translate/bleu_score.py:516: UserWarning: \n",
      "The hypothesis contains 0 counts of 3-gram overlaps.\n",
      "Therefore the BLEU score evaluates to 0, independently of\n",
      "how many N-gram overlaps of lower order it contains.\n",
      "Consider using lower n-gram order or use SmoothingFunction()\n",
      "  warnings.warn(_msg)\n",
      "2021-08-12 04:21:23.149 | INFO     | __main__:metrics:39 - Key:Gab_Gab_joy_0--bleu :0.2608544714994889--bleu_4:0.2608689462987191--diversity:0.716992532148161--novelty:0.7950649138065341--meteor:0.1670955105141134\n"
     ]
    }
   ],
   "source": [
    "metrics('Gab','Gab')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/adarsh-binny/.conda/envs/test_env/lib/python3.7/site-packages/nltk/translate/bleu_score.py:516: UserWarning: \n",
      "The hypothesis contains 0 counts of 4-gram overlaps.\n",
      "Therefore the BLEU score evaluates to 0, independently of\n",
      "how many N-gram overlaps of lower order it contains.\n",
      "Consider using lower n-gram order or use SmoothingFunction()\n",
      "  warnings.warn(_msg)\n",
      "/home/adarsh-binny/.conda/envs/test_env/lib/python3.7/site-packages/nltk/translate/bleu_score.py:516: UserWarning: \n",
      "The hypothesis contains 0 counts of 4-gram overlaps.\n",
      "Therefore the BLEU score evaluates to 0, independently of\n",
      "how many N-gram overlaps of lower order it contains.\n",
      "Consider using lower n-gram order or use SmoothingFunction()\n",
      "  warnings.warn(_msg)\n",
      "/home/adarsh-binny/.conda/envs/test_env/lib/python3.7/site-packages/nltk/translate/bleu_score.py:516: UserWarning: \n",
      "The hypothesis contains 0 counts of 4-gram overlaps.\n",
      "Therefore the BLEU score evaluates to 0, independently of\n",
      "how many N-gram overlaps of lower order it contains.\n",
      "Consider using lower n-gram order or use SmoothingFunction()\n",
      "  warnings.warn(_msg)\n",
      "/home/adarsh-binny/.conda/envs/test_env/lib/python3.7/site-packages/nltk/translate/bleu_score.py:516: UserWarning: \n",
      "The hypothesis contains 0 counts of 4-gram overlaps.\n",
      "Therefore the BLEU score evaluates to 0, independently of\n",
      "how many N-gram overlaps of lower order it contains.\n",
      "Consider using lower n-gram order or use SmoothingFunction()\n",
      "  warnings.warn(_msg)\n",
      "/home/adarsh-binny/.conda/envs/test_env/lib/python3.7/site-packages/nltk/translate/bleu_score.py:516: UserWarning: \n",
      "The hypothesis contains 0 counts of 4-gram overlaps.\n",
      "Therefore the BLEU score evaluates to 0, independently of\n",
      "how many N-gram overlaps of lower order it contains.\n",
      "Consider using lower n-gram order or use SmoothingFunction()\n",
      "  warnings.warn(_msg)\n",
      "/home/adarsh-binny/.conda/envs/test_env/lib/python3.7/site-packages/nltk/translate/bleu_score.py:516: UserWarning: \n",
      "The hypothesis contains 0 counts of 4-gram overlaps.\n",
      "Therefore the BLEU score evaluates to 0, independently of\n",
      "how many N-gram overlaps of lower order it contains.\n",
      "Consider using lower n-gram order or use SmoothingFunction()\n",
      "  warnings.warn(_msg)\n",
      "2021-08-18 17:21:22.747 | INFO     | __main__:f:42 - Key:Reddit_Reddit_toxic_1--bleu :0.2524448470938123--bleu_4:0.2524601042958655--diversity:0.7596694295375596--novelty:0.817504188579462--meteor:0.15125937613078166\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-18 17:32:57.604 | INFO     | __main__:f:42 - Key:Reddit_Reddit_joy_0--bleu :0.23484611013826465--bleu_4:0.23485025476819776--diversity:0.7425532033954653--novelty:0.8235129741410291--meteor:0.1708030453337513\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-18 17:33:35.760 | INFO     | __main__:f:42 - Key:Reddit_Reddit_polite_0--bleu :0.22794142358663474--bleu_4:0.22794399253536715--diversity:0.7549264084475641--novelty:0.8401164150391763--meteor:0.15446076367512562\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-18 17:45:03.121 | INFO     | __main__:f:42 - Key:Reddit_Reddit_anger_1--bleu :0.18716731595873118--bleu_4:0.18717024084885148--diversity:0.7952776168215064--novelty:0.8772981963923815--meteor:0.13964258453556538\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-18 17:49:38.253 | INFO     | __main__:f:42 - Key:Reddit_Reddit_joy_1--bleu :0.19159508198821035--bleu_4:0.19159508198821035--diversity:0.7714839932389237--novelty:0.8665133391825054--meteor:0.15676525866753221\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-18 17:50:50.815 | INFO     | __main__:f:42 - Key:Reddit_Reddit_love_1--bleu :0.07697119894057808--bleu_4:0.07703693717970177--diversity:0.71348081008246--novelty:0.8641648999032181--meteor:0.07999716764632224\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "49\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-18 17:54:51.009 | INFO     | __main__:f:42 - Key:Reddit_Reddit_surprise_1--bleu :0.13611898938762165--bleu_4:0.13611998925849308--diversity:0.7937278998813059--novelty:0.8899470546827594--meteor:0.11957825549265254\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-18 18:01:56.468 | INFO     | __main__:f:42 - Key:Reddit_Reddit_sadness_0--bleu :0.16858535825164075--bleu_4:0.16858535825164075--diversity:0.7809108277084241--novelty:0.8817657691109974--meteor:0.16170848827884668\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-18 18:06:13.119 | INFO     | __main__:f:42 - Key:Reddit_Reddit_fear_0--bleu :0.16529066626954123--bleu_4:0.16529066626954123--diversity:0.7567048311240059--novelty:0.8806601379136259--meteor:0.1673527289617203\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "36\n"
     ]
    }
   ],
   "source": [
    "trained_on = 'Reddit'\n",
    "tested_on  = 'Reddit'\n",
    "gen        = glob.glob(path_result+'/*'+trained_on+'*'+tested_on+'*1628*')\n",
    "ref        = glob.glob(path_result+'/*'+tested_on+'*'+'references'+'*')[0]\n",
    "train_path = glob.glob(path_datasets+'/*'+trained_on+'*/*rain*')[0]    \n",
    "\n",
    "#     print(gen,ref)\n",
    "with open(ref, 'r') as file:\n",
    "    ref_dict   = json.loads(file.read())\n",
    "    \n",
    "import time\n",
    "scores = {}\n",
    "from multiprocessing import Pool\n",
    "def f(x):\n",
    "    files = gen[x]\n",
    "    with open(files, 'r') as file:\n",
    "        gen_dict  = json.loads(file.read())\n",
    "    emotion   = gen_dict['params']['task_name'][0][1]\n",
    "    gpu_id    = gen_dict['params']['gpu_id']\n",
    "    hypo = []\n",
    "    refs = []\n",
    "    for key in gen_dict['samples']:\n",
    "        for sentences in gen_dict['samples'][key]['counterspeech_model']:\n",
    "            hypo.append(sentences)\n",
    "            refs.append(ref_dict['samples'][key]['counterspeech_model'])\n",
    "\n",
    "    train = pd.read_csv(train_path)\n",
    "    train_set = list(zip(train['initiator_message'].tolist(),train['reply_message'].tolist()))\n",
    "    params = [hypo,refs]\n",
    "    bleu, bleu_4, meteor_ = nltk_metrics(params)\n",
    "    train_corpus = training_corpus(train_set)\n",
    "    diversity, novelty = diversity_and_novelty(train_corpus,hypo)\n",
    "    data_dict = {\n",
    "                 'bleu':bleu,\n",
    "                 'bleu_4':bleu_4,\n",
    "                 'diversity':diversity,\n",
    "                 'novelty':novelty, \n",
    "                 'meteor':meteor_\n",
    "               }\n",
    "    key = trained_on+'_'+tested_on+'_'+emotion+'_'+str(gpu_id)\n",
    "    scores[key] = data_dict\n",
    "    logger.info(f'Key:{key}--bleu :{bleu}--bleu_4:{bleu_4}--diversity:{diversity}--novelty:{novelty}--meteor:{meteor_}')\n",
    "    print(x*x)\n",
    "p = Pool(len(gen))\n",
    "p.map(f, [x for x in range(len(gen))])\n",
    "p.close()\n",
    "p.join()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "****Toxicity_Score****"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def toxicity(trained_on,tested_on):\n",
    "    train_path = glob.glob(path_datasets+'/*'+trained_on+'*/*rain*')[0]\n",
    "    gen        = glob.glob(path_result+'/*'+trained_on+'*'+tested_on+'*1628*')\n",
    "    ref        = glob.glob(path_result+'/*'+tested_on+'*'+'references'+'*')[0]\n",
    "    \n",
    "#     print(gen,ref)\n",
    "    with open(ref, 'r') as file:\n",
    "        ref_dict   = json.loads(file.read())\n",
    "        \n",
    "    scores = {}\n",
    "    for files in gen:\n",
    "        with open(files, 'r') as file:\n",
    "            gen_dict  = json.loads(file.read())\n",
    "        emotion   = gen_dict['params']['task_name'][0][1]\n",
    "        gpu_id    = gen_dict['params']['gpu_id']\n",
    "        \n",
    "        detox = 0.0\n",
    "        tot   = 0.0\n",
    "        \n",
    "        for key in gen_dict['samples']:\n",
    "            for sentence in gen_dict['samples'][key]['counterspeech_model']:\n",
    "                detox += get_non_toxicity_score(sentence)\n",
    "                tot   += 1\n",
    "        \n",
    "        detox /= tot\n",
    "        print(f'emotion = {emotion}, detox = {detox}')\n",
    "        key = trained_on+'_'+tested_on+'_'+emotion+'_'+str(gpu_id)\n",
    "        logger.info(f'Key -> {key}-- Detox -> {detox}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "ename": "HttpError",
     "evalue": "<HttpError 429 when requesting https://commentanalyzer.googleapis.com/v1alpha1/comments:analyze?key=AIzaSyAUYdcELBp_FIPyiCQ6UfDEDIfDJDiShLE&alt=json returned \"Quota exceeded for quota metric 'Analysis requests (AnalyzeComment)' and limit 'Analysis requests (AnalyzeComment) per minute' of service 'commentanalyzer.googleapis.com' for consumer 'project_number:182130340501'.\". Details: \"[{'@type': 'type.googleapis.com/google.rpc.ErrorInfo', 'reason': 'RATE_LIMIT_EXCEEDED', 'domain': 'googleapis.com', 'metadata': {'quota_limit': 'AnalyzeRequestsPerMinutePerProject', 'consumer': 'projects/182130340501', 'quota_metric': 'CommentAnalyzerService/analyze_requests', 'service': 'commentanalyzer.googleapis.com'}}]\">",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mHttpError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-20-e3351016eb9c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtoxicity\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Reddit'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'Reddit'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-19-d45f53c3e26c>\u001b[0m in \u001b[0;36mtoxicity\u001b[0;34m(trained_on, tested_on)\u001b[0m\n\u001b[1;32m     20\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mkey\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mgen_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'samples'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0msentence\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mgen_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'samples'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'counterspeech_model'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 22\u001b[0;31m                 \u001b[0mdetox\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mget_non_toxicity_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentence\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     23\u001b[0m                 \u001b[0mtot\u001b[0m   \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-14-a60026ea1ee7>\u001b[0m in \u001b[0;36mget_non_toxicity_score\u001b[0;34m(sentence)\u001b[0m\n\u001b[1;32m     16\u001b[0m       \u001b[0;34m'requestedAttributes'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m'TOXICITY'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m     }\n\u001b[0;32m---> 18\u001b[0;31m     \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclient\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcomments\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0manalyze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbody\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0manalyze_request\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m     \u001b[0mtox\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mresponse\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'attributeScores'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'TOXICITY'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'summaryScore'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'value'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m     \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1.0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/test_env/lib/python3.7/site-packages/googleapiclient/_helpers.py\u001b[0m in \u001b[0;36mpositional_wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    129\u001b[0m                 \u001b[0;32melif\u001b[0m \u001b[0mpositional_parameters_enforcement\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mPOSITIONAL_WARNING\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    130\u001b[0m                     \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwarning\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 131\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mwrapped\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    132\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    133\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mpositional_wrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/test_env/lib/python3.7/site-packages/googleapiclient/http.py\u001b[0m in \u001b[0;36mexecute\u001b[0;34m(self, http, num_retries)\u001b[0m\n\u001b[1;32m    935\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    936\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mresp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0;36m300\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 937\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mHttpError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muri\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muri\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    938\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpostproc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontent\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    939\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mHttpError\u001b[0m: <HttpError 429 when requesting https://commentanalyzer.googleapis.com/v1alpha1/comments:analyze?key=AIzaSyAUYdcELBp_FIPyiCQ6UfDEDIfDJDiShLE&alt=json returned \"Quota exceeded for quota metric 'Analysis requests (AnalyzeComment)' and limit 'Analysis requests (AnalyzeComment) per minute' of service 'commentanalyzer.googleapis.com' for consumer 'project_number:182130340501'.\". Details: \"[{'@type': 'type.googleapis.com/google.rpc.ErrorInfo', 'reason': 'RATE_LIMIT_EXCEEDED', 'domain': 'googleapis.com', 'metadata': {'quota_limit': 'AnalyzeRequestsPerMinutePerProject', 'consumer': 'projects/182130340501', 'quota_metric': 'CommentAnalyzerService/analyze_requests', 'service': 'commentanalyzer.googleapis.com'}}]\">"
     ]
    }
   ],
   "source": [
    "toxicity('Reddit','Reddit')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "toxicity('Gab','Gab')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "toxicity('CONAN','CONAN')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:.conda-test_env]",
   "language": "python",
   "name": "conda-env-.conda-test_env-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}