Evaluation notebook-Politeness.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-1-92ae64a6dc1e>, line 2)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-1-92ae64a6dc1e>\"\u001b[0;36m, line \u001b[0;32m2\u001b[0m\n\u001b[0;31m    https://github.com/monologg/GoEmotions-pytorch\u001b[0m\n\u001b[0m           ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "### for emotion detection\n",
    "https://github.com/monologg/GoEmotions-pytorch\n",
    "### For toxicity detection\n",
    "perspective api\n",
    "### For politeness detection\n",
    "https://github.com/AlafateABULIMITI/politeness-detection\n",
    "### For language quality\n",
    "https://github.com/WanzhengZhu/GPS/blob/master/language_quality.py\n",
    "### Other metrics are aleady added add two metric from the GPS model.\n",
    "### Implement GPS model as a baseline.\n",
    "save_path='../HULK_new/Counterspeech/Saved_Models'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import time\n",
    "from tqdm.notebook import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#BLEU, METEOR, Diversity, Novelty\n",
    "import glob\n",
    "import json\n",
    "from loguru import logger\n",
    "import pandas as pd \n",
    "from Generation.eval import *\n",
    "path_datasets = '/home/adarsh-binny' + '/HULK_new/Counterspeech/Datasets'\n",
    "path_result   = '/home/adarsh-binny' + '/HULK_new/Counterspeech/Results'\n",
    "save_path     = '/home/adarsh-binny' + '/HULK_new/Counterspeech/metrics_results/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cell for Politeness evaluation\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn import metrics\n",
    "import transformers\n",
    "import torch\n",
    "from torch.utils.data import Dataset, DataLoader, RandomSampler, SequentialSampler\n",
    "from transformers import BertTokenizer, BertModel, BertConfig\n",
    "\n",
    "from torch import cuda\n",
    "\n",
    "def normalise_text(text):\n",
    "    text = text.strip()\n",
    "    text = text.lower() # lowercase\n",
    "    text = text.replace(r\"\\#\",\"\") # replaces hashtags\n",
    "    text = text.replace(r\"http\\S+\",\"URL\")  # remove URL addresses\n",
    "    text = text.replace(r\"@\",\"\")\n",
    "    text = text.replace(r\"[^A-Za-z0-9()!?\\'\\`\\\"]\", \" \")\n",
    "    text = text.replace(\"\\s{2,}\", \" \")\n",
    "    text = text.replace(r\"\\#\",\"\")\n",
    "    return text\n",
    "\n",
    "VALID_BATCH_SIZE = 16\n",
    "\n",
    "tokenizer = BertTokenizer.from_pretrained(\"bert-base-uncased\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.cuda.set_device(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = \"cuda\" if cuda.is_available() else \"cpu\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BERTClass(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super(BERTClass, self).__init__()\n",
    "        self.l1 = BertModel.from_pretrained(\"bert-base-uncased\", return_dict=False)\n",
    "        self.pre_classifier = torch.nn.Linear(768, 768)\n",
    "        self.dropout = torch.nn.Dropout(0.3)\n",
    "        self.classifier = torch.nn.Linear(768, 8)\n",
    "\n",
    "    def forward(self, input_ids, attention_mask):\n",
    "        output_1 = self.l1(input_ids=input_ids, attention_mask=attention_mask)\n",
    "        hidden_state = output_1[0]\n",
    "        pooler = hidden_state[:, 0]\n",
    "        pooler = self.pre_classifier(pooler)\n",
    "        pooler = torch.nn.ReLU()(pooler)\n",
    "        pooler = self.dropout(pooler)\n",
    "        output = self.classifier(pooler)\n",
    "        return output\n",
    "\n",
    "model = BERTClass()\n",
    "model.to(device);\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = \"/home/adarsh-binny/HULK_new/Counterspeech/Saved_Models/Metric_Models/Politeness/model_small.pt\"\n",
    "model.load_state_dict(torch.load(PATH));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class PDataset(Dataset):\n",
    "    def __init__(self, hypo, tokenizer, max_len=200):\n",
    "        self.len = len(hypo)\n",
    "        self.data = hypo\n",
    "        self.tokenizer = tokenizer\n",
    "        self.max_len = max_len\n",
    "        \n",
    "    def __getitem__(self, index):\n",
    "        text = self.data[index]\n",
    "        #print(text)\n",
    "        text = normalise_text(text)\n",
    "        text = \" \".join(text.split())\n",
    "        inputs = self.tokenizer.encode_plus(\n",
    "            text,\n",
    "            None,\n",
    "            add_special_tokens=True,\n",
    "            max_length=self.max_len,\n",
    "            pad_to_max_length=True,\n",
    "            return_token_type_ids=True,\n",
    "            truncation=True\n",
    "        )\n",
    "        ids = inputs['input_ids']\n",
    "        mask = inputs['attention_mask']\n",
    "        return {\n",
    "            'ids': torch.tensor(ids, dtype=torch.long),\n",
    "            'mask': torch.tensor(mask, dtype=torch.long)\n",
    "        } \n",
    "    \n",
    "    def __len__(self):\n",
    "        return self.len\n",
    "\n",
    "test_params = {'batch_size': VALID_BATCH_SIZE,\n",
    "                'shuffle': False,\n",
    "                'num_workers': 0\n",
    "                }\n",
    "\n",
    "    \n",
    "def calc_politeness(hypo):\n",
    "    N =len(hypo)\n",
    "    print(\"Length of Hypothesis: \", N)\n",
    "    hypo_ = PDataset(hypo, tokenizer, 200)\n",
    "    testing_loader = DataLoader(hypo_, **test_params)\n",
    "    model.eval()\n",
    "    tr_loss = 0\n",
    "    n_correct = 0 \n",
    "    n_wrong = 0\n",
    "    total = 0\n",
    "    n1 = N//8\n",
    "    if N%VALID_BATCH_SIZE!=0:\n",
    "        n1+=1\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        with tqdm(n1) as pbar:\n",
    "            for _, data in enumerate(testing_loader, 0):\n",
    "                #print(data)\n",
    "                ids = data['ids'].to(device, dtype = torch.long)\n",
    "                mask = data['mask'].to(device, dtype = torch.long)\n",
    "                outputs = model(ids, mask).squeeze()\n",
    "\n",
    "                outputs = torch.nn.Softmax(dim=1)(outputs)\n",
    "                \n",
    "                #print(outputs.shape)\n",
    "                #print(outputs[:,1])\n",
    "                output = torch.argmax(outputs, dim=1)\n",
    "                #print(output)\n",
    "                total+= sum(output)\n",
    "                pbar.update()\n",
    "    return total/N\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def metrics(trained_on,tested_on):\n",
    "    train_path = glob.glob(path_datasets+'/*'+trained_on+'*/*rain*')[0]\n",
    "    gen        = glob.glob(path_result+'/*'+trained_on+'*'+tested_on+'*1628*_updated')\n",
    "    gen += glob.glob(path_result+'/*'+trained_on+'*'+tested_on+'*162*_*')\n",
    "    ref        = glob.glob(path_result+'/*'+tested_on+'*'+'references'+'*')[0]\n",
    "    \n",
    "#     print(gen,ref)\n",
    "    with open(ref, 'r') as file:\n",
    "        ref_dict   = json.loads(file.read())\n",
    "        \n",
    "    scores = {}\n",
    "    for files in gen:\n",
    "        with open(files, 'r') as file:\n",
    "            gen_dict  = json.loads(file.read())\n",
    "        #emotion = gen_dict['params']['task_name'][0][1]\n",
    "        #emotion = gen_dict['params']['task_name'][0][1]\n",
    "        print(files)\n",
    "        emotion = \"politeness\"\n",
    "        print(gen_dict['params']['task_name'])\n",
    "        print(gen_dict['params']['disc_weight'])\n",
    "        if (\"polite\" in emotion or \"gedi\" not in files)==False:\n",
    "            continue\n",
    "        print(files)\n",
    "        gpu_id    = gen_dict['params']['gpu_id']\n",
    "        hypo = []\n",
    "        refs = []\n",
    "        signature = files.split(\"_\")[-1].split(\".\")[0]\n",
    "        for key in gen_dict['samples']:\n",
    "            for sentences in gen_dict['samples'][key]['counterspeech_model']:\n",
    "                hypo.append(sentences)\n",
    "                refs.append(ref_dict['samples'][key]['counterspeech_model'])\n",
    "        \n",
    "        train = pd.read_csv(train_path)\n",
    "        train_set = list(zip(train['initiator_message'].tolist(),train['reply_message'].tolist()))\n",
    "        params = hypo\n",
    "        average_score = calc_politeness(hypo)\n",
    "        key = trained_on+'_'+tested_on+'_'+emotion+'_'+signature+'_'+str(gpu_id)\n",
    "        scores[key] = average_score\n",
    "        logger.info(f'Key:{key}--Politeness_Score :{average_score}')\n",
    "    \n",
    "    json.dump(scores, open(save_path +tested_on+'_politeness.json','w'),indent = 4)    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "****Reddit****"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Reddit_DialoGPT-medium_on_Reddit_gedi_huggingface_1629484650_updated.json\n",
      "[['Politeness', 'polite']]\n",
      "[30]\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Reddit_DialoGPT-medium_on_Reddit_gedi_huggingface_1629484650_updated.json\n",
      "Length of Hypothesis:  100\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "250dc02b4d574b3a8dfd27e32b04b2fe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-21 18:58:59.345 | INFO     | __main__:metrics:39 - Key:Reddit_Reddit_politeness_updated_0--Politeness_Score :6\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Reddit_DialoGPT-medium_on_Reddit_gedi_huggingface_1629489580_single.json\n",
      "[['Toxicity', 'toxic']]\n",
      "[1]\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Reddit_DialoGPT-medium_on_Reddit_gedi_huggingface_1629489580_single.json\n",
      "Length of Hypothesis:  100\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "81fd454259f343de991e8a9ec6370849",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-21 18:59:00.382 | INFO     | __main__:metrics:39 - Key:Reddit_Reddit_politeness_single_0--Politeness_Score :4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Reddit_DialoGPT-medium_on_Reddit_gedi_huggingface_1629486310_updated.json\n",
      "[['Politeness', 'polite']]\n",
      "[1]\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Reddit_DialoGPT-medium_on_Reddit_gedi_huggingface_1629486310_updated.json\n",
      "Length of Hypothesis:  100\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9a57c73407d04e68a55ead3feca22022",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-21 18:59:01.309 | INFO     | __main__:metrics:39 - Key:Reddit_Reddit_politeness_updated_0--Politeness_Score :6\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Reddit_DialoGPT-medium_on_Reddit_gedi_huggingface_1629487718_multi.json\n",
      "[['Toxicity', 'toxic'], ['Emotion', 'joy']]\n",
      "[1, 1]\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Reddit_DialoGPT-medium_on_Reddit_gedi_huggingface_1629487718_multi.json\n",
      "Length of Hypothesis:  100\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a690f7c34d9f42278bde7b699c1be9ae",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-21 18:59:02.193 | INFO     | __main__:metrics:39 - Key:Reddit_Reddit_politeness_multi_0--Politeness_Score :4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Reddit_DialoGPT-medium_on_Reddit_huggingface_1629551746_base.json\n",
      "[]\n",
      "[]\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Reddit_DialoGPT-medium_on_Reddit_huggingface_1629551746_base.json\n",
      "Length of Hypothesis:  100\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bf0a0bdf04b34bab8a8d7ec80df1ae18",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-21 18:59:03.122 | INFO     | __main__:metrics:39 - Key:Reddit_Reddit_politeness_base_0--Politeness_Score :4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Reddit_DialoGPT-medium_on_Reddit_gedi_huggingface_1629483113_updated.json\n",
      "[['Emotion', 'joy']]\n",
      "[30]\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Reddit_DialoGPT-medium_on_Reddit_gedi_huggingface_1629483113_updated.json\n",
      "Length of Hypothesis:  100\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2b1b9963e4304637b713dd9c9ceee732",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-21 18:59:03.957 | INFO     | __main__:metrics:39 - Key:Reddit_Reddit_politeness_updated_0--Politeness_Score :5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Reddit_DialoGPT-medium_on_Reddit_gedi_huggingface_1629483851_updated.json\n",
      "[['Politeness', 'polite'], ['Emotion', 'joy']]\n",
      "[10, 10]\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Reddit_DialoGPT-medium_on_Reddit_gedi_huggingface_1629483851_updated.json\n",
      "Length of Hypothesis:  100\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e53761d800414a5b8b8b41e2637b6c18",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-21 18:59:04.804 | INFO     | __main__:metrics:39 - Key:Reddit_Reddit_politeness_updated_0--Politeness_Score :6\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Reddit_DialoGPT-medium_on_Reddit_gedi_huggingface_1629485585_updated.json\n",
      "[['Politeness', 'polite']]\n",
      "[5]\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Reddit_DialoGPT-medium_on_Reddit_gedi_huggingface_1629485585_updated.json\n",
      "Length of Hypothesis:  100\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c4999305d2ab41ec96bd9755a5305e1d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-21 18:59:05.647 | INFO     | __main__:metrics:39 - Key:Reddit_Reddit_politeness_updated_0--Politeness_Score :5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Reddit_DialoGPT-medium_on_Reddit_gedi_huggingface_1629550279_multi_2.json\n",
      "[['Emotion', 'joy'], ['Politeness', 'polite']]\n",
      "[1, 1]\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Reddit_DialoGPT-medium_on_Reddit_gedi_huggingface_1629550279_multi_2.json\n",
      "Length of Hypothesis:  100\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5319b40122864246a7a952d7b211b79b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-21 18:59:06.901 | INFO     | __main__:metrics:39 - Key:Reddit_Reddit_politeness_2_0--Politeness_Score :5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Reddit_DialoGPT-medium_on_Reddit_gedi_huggingface_1629488890_single.json\n",
      "[['Emotion', 'joy']]\n",
      "[1]\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Reddit_DialoGPT-medium_on_Reddit_gedi_huggingface_1629488890_single.json\n",
      "Length of Hypothesis:  100\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2ff7391765b547d38e8e9c6f86b6c130",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-21 18:59:07.820 | INFO     | __main__:metrics:39 - Key:Reddit_Reddit_politeness_single_0--Politeness_Score :5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "Object of type Tensor is not JSON serializable",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-20-562649cb15f8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmetrics\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Reddit'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'Reddit'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-19-079b03eef31b>\u001b[0m in \u001b[0;36mmetrics\u001b[0;34m(trained_on, tested_on)\u001b[0m\n\u001b[1;32m     39\u001b[0m         \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'Key:{key}--Politeness_Score :{average_score}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 41\u001b[0;31m     \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscores\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msave_path\u001b[0m \u001b[0;34m+\u001b[0m\u001b[0mtested_on\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'_politeness.json'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'w'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mindent\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/.conda/envs/csgen/lib/python3.7/json/__init__.py\u001b[0m in \u001b[0;36mdump\u001b[0;34m(obj, fp, skipkeys, ensure_ascii, check_circular, allow_nan, cls, indent, separators, default, sort_keys, **kw)\u001b[0m\n\u001b[1;32m    177\u001b[0m     \u001b[0;31m# could accelerate with writelines in some versions of Python, at\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    178\u001b[0m     \u001b[0;31m# a debuggability cost\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 179\u001b[0;31m     \u001b[0;32mfor\u001b[0m \u001b[0mchunk\u001b[0m \u001b[0;32min\u001b[0m \u001b[0miterable\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    180\u001b[0m         \u001b[0mfp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    181\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/csgen/lib/python3.7/json/encoder.py\u001b[0m in \u001b[0;36m_iterencode\u001b[0;34m(o, _current_indent_level)\u001b[0m\n\u001b[1;32m    429\u001b[0m             \u001b[0;32myield\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0m_iterencode_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_current_indent_level\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    430\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 431\u001b[0;31m             \u001b[0;32myield\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0m_iterencode_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_current_indent_level\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    432\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    433\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mmarkers\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/csgen/lib/python3.7/json/encoder.py\u001b[0m in \u001b[0;36m_iterencode_dict\u001b[0;34m(dct, _current_indent_level)\u001b[0m\n\u001b[1;32m    403\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    404\u001b[0m                     \u001b[0mchunks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_iterencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_current_indent_level\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 405\u001b[0;31m                 \u001b[0;32myield\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mchunks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    406\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mnewline_indent\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    407\u001b[0m             \u001b[0m_current_indent_level\u001b[0m \u001b[0;34m-=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/csgen/lib/python3.7/json/encoder.py\u001b[0m in \u001b[0;36m_iterencode\u001b[0;34m(o, _current_indent_level)\u001b[0m\n\u001b[1;32m    436\u001b[0m                     \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Circular reference detected\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    437\u001b[0m                 \u001b[0mmarkers\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmarkerid\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mo\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 438\u001b[0;31m             \u001b[0mo\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_default\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    439\u001b[0m             \u001b[0;32myield\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0m_iterencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_current_indent_level\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    440\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mmarkers\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/csgen/lib/python3.7/json/encoder.py\u001b[0m in \u001b[0;36mdefault\u001b[0;34m(self, o)\u001b[0m\n\u001b[1;32m    177\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    178\u001b[0m         \"\"\"\n\u001b[0;32m--> 179\u001b[0;31m         raise TypeError(f'Object of type {o.__class__.__name__} '\n\u001b[0m\u001b[1;32m    180\u001b[0m                         f'is not JSON serializable')\n\u001b[1;32m    181\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: Object of type Tensor is not JSON serializable"
     ]
    }
   ],
   "source": [
    "metrics('Reddit','Reddit')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "****CONAN****"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-CONAN_DialoGPT-medium_on_CONAN_gedi_huggingface_1629485151_updated.json\n",
      "[['Politeness', 'polite']]\n",
      "[30]\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-CONAN_DialoGPT-medium_on_CONAN_gedi_huggingface_1629485151_updated.json\n",
      "Length of Hypothesis:  86\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fe16a6398d9b4f6c9eb549b2e1d477db",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-21 19:02:01.468 | INFO     | __main__:metrics:39 - Key:CONAN_CONAN_politeness_updated_0--Politeness_Score :5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-CONAN_DialoGPT-medium_on_CONAN_gedi_huggingface_1629486713_updated.json\n",
      "[['Politeness', 'polite']]\n",
      "[1]\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-CONAN_DialoGPT-medium_on_CONAN_gedi_huggingface_1629486713_updated.json\n",
      "Length of Hypothesis:  86\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4cc22773fe134cedbf5558efb81de78f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-21 19:02:02.074 | INFO     | __main__:metrics:39 - Key:CONAN_CONAN_politeness_updated_0--Politeness_Score :4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-CONAN_DialoGPT-medium_on_CONAN_gedi_huggingface_1629489279_single.json\n",
      "[['Emotion', 'joy']]\n",
      "[1]\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-CONAN_DialoGPT-medium_on_CONAN_gedi_huggingface_1629489279_single.json\n",
      "Length of Hypothesis:  86\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bb5acf31a2ca4bcd8db2b546b740ee5b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-21 19:02:02.698 | INFO     | __main__:metrics:39 - Key:CONAN_CONAN_politeness_single_0--Politeness_Score :3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-CONAN_DialoGPT-medium_on_CONAN_gedi_huggingface_1629488258_multi.json\n",
      "[['Toxicity', 'toxic'], ['Emotion', 'joy']]\n",
      "[1, 1]\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-CONAN_DialoGPT-medium_on_CONAN_gedi_huggingface_1629488258_multi.json\n",
      "Length of Hypothesis:  86\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c45b8801c39f413d81baf30f363fac98",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-21 19:02:03.334 | INFO     | __main__:metrics:39 - Key:CONAN_CONAN_politeness_multi_0--Politeness_Score :3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-CONAN_DialoGPT-medium_on_CONAN_huggingface_1629552117_base.json\n",
      "[]\n",
      "[]\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-CONAN_DialoGPT-medium_on_CONAN_huggingface_1629552117_base.json\n",
      "Length of Hypothesis:  86\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5c1356ab757b49aba626ff971d49eb1a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-21 19:02:03.959 | INFO     | __main__:metrics:39 - Key:CONAN_CONAN_politeness_base_0--Politeness_Score :3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-CONAN_DialoGPT-medium_on_CONAN_gedi_huggingface_1629490070_single.json\n",
      "[['Toxicity', 'toxic']]\n",
      "[1]\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-CONAN_DialoGPT-medium_on_CONAN_gedi_huggingface_1629490070_single.json\n",
      "Length of Hypothesis:  86\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2e6f66f121df4e70af12c0f8a1bddd6a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-21 19:02:04.620 | INFO     | __main__:metrics:39 - Key:CONAN_CONAN_politeness_single_0--Politeness_Score :3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-CONAN_DialoGPT-medium_on_CONAN_gedi_huggingface_1629485999_updated.json\n",
      "[['Politeness', 'polite']]\n",
      "[5]\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-CONAN_DialoGPT-medium_on_CONAN_gedi_huggingface_1629485999_updated.json\n",
      "Length of Hypothesis:  86\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "855224b9e2a342858ae875f45c19dc33",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-21 19:02:05.282 | INFO     | __main__:metrics:39 - Key:CONAN_CONAN_politeness_updated_0--Politeness_Score :4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-CONAN_DialoGPT-medium_on_CONAN_gedi_huggingface_1629550796_multi_2.json\n",
      "[['Emotion', 'joy'], ['Politeness', 'polite']]\n",
      "[1, 1]\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-CONAN_DialoGPT-medium_on_CONAN_gedi_huggingface_1629550796_multi_2.json\n",
      "Length of Hypothesis:  86\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "736a14bf565a49d3ac95ed1b55ae504d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-21 19:02:05.890 | INFO     | __main__:metrics:39 - Key:CONAN_CONAN_politeness_2_0--Politeness_Score :4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "Object of type Tensor is not JSON serializable",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-21-8d687e8c3365>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmetrics\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'CONAN'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'CONAN'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-19-079b03eef31b>\u001b[0m in \u001b[0;36mmetrics\u001b[0;34m(trained_on, tested_on)\u001b[0m\n\u001b[1;32m     39\u001b[0m         \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'Key:{key}--Politeness_Score :{average_score}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 41\u001b[0;31m     \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscores\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msave_path\u001b[0m \u001b[0;34m+\u001b[0m\u001b[0mtested_on\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'_politeness.json'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'w'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mindent\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/.conda/envs/csgen/lib/python3.7/json/__init__.py\u001b[0m in \u001b[0;36mdump\u001b[0;34m(obj, fp, skipkeys, ensure_ascii, check_circular, allow_nan, cls, indent, separators, default, sort_keys, **kw)\u001b[0m\n\u001b[1;32m    177\u001b[0m     \u001b[0;31m# could accelerate with writelines in some versions of Python, at\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    178\u001b[0m     \u001b[0;31m# a debuggability cost\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 179\u001b[0;31m     \u001b[0;32mfor\u001b[0m \u001b[0mchunk\u001b[0m \u001b[0;32min\u001b[0m \u001b[0miterable\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    180\u001b[0m         \u001b[0mfp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    181\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/csgen/lib/python3.7/json/encoder.py\u001b[0m in \u001b[0;36m_iterencode\u001b[0;34m(o, _current_indent_level)\u001b[0m\n\u001b[1;32m    429\u001b[0m             \u001b[0;32myield\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0m_iterencode_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_current_indent_level\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    430\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 431\u001b[0;31m             \u001b[0;32myield\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0m_iterencode_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_current_indent_level\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    432\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    433\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mmarkers\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/csgen/lib/python3.7/json/encoder.py\u001b[0m in \u001b[0;36m_iterencode_dict\u001b[0;34m(dct, _current_indent_level)\u001b[0m\n\u001b[1;32m    403\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    404\u001b[0m                     \u001b[0mchunks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_iterencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_current_indent_level\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 405\u001b[0;31m                 \u001b[0;32myield\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mchunks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    406\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mnewline_indent\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    407\u001b[0m             \u001b[0m_current_indent_level\u001b[0m \u001b[0;34m-=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/csgen/lib/python3.7/json/encoder.py\u001b[0m in \u001b[0;36m_iterencode\u001b[0;34m(o, _current_indent_level)\u001b[0m\n\u001b[1;32m    436\u001b[0m                     \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Circular reference detected\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    437\u001b[0m                 \u001b[0mmarkers\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmarkerid\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mo\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 438\u001b[0;31m             \u001b[0mo\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_default\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    439\u001b[0m             \u001b[0;32myield\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0m_iterencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_current_indent_level\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    440\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mmarkers\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/csgen/lib/python3.7/json/encoder.py\u001b[0m in \u001b[0;36mdefault\u001b[0;34m(self, o)\u001b[0m\n\u001b[1;32m    177\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    178\u001b[0m         \"\"\"\n\u001b[0;32m--> 179\u001b[0;31m         raise TypeError(f'Object of type {o.__class__.__name__} '\n\u001b[0m\u001b[1;32m    180\u001b[0m                         f'is not JSON serializable')\n\u001b[1;32m    181\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: Object of type Tensor is not JSON serializable"
     ]
    }
   ],
   "source": [
    "metrics('CONAN','CONAN')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "****Gab****"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Gab_DialoGPT-medium_on_Gab_gedi_huggingface_1629485810_updated.json\n",
      "[['Politeness', 'polite']]\n",
      "[5]\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Gab_DialoGPT-medium_on_Gab_gedi_huggingface_1629485810_updated.json\n",
      "Length of Hypothesis:  100\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "04db686ae7f24aec820d3510d7be3a01",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-21 19:03:27.134 | INFO     | __main__:metrics:39 - Key:Gab_Gab_politeness_updated_0--Politeness_Score :6\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Gab_DialoGPT-medium_on_Gab_gedi_huggingface_1629489109_single.json\n",
      "[['Emotion', 'joy']]\n",
      "[1]\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Gab_DialoGPT-medium_on_Gab_gedi_huggingface_1629489109_single.json\n",
      "Length of Hypothesis:  100\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d438de76f2a9414fb16691ff544a2422",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-21 19:03:28.399 | INFO     | __main__:metrics:39 - Key:Gab_Gab_politeness_single_0--Politeness_Score :5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Gab_DialoGPT-medium_on_Gab_gedi_huggingface_1629484260_updated.json\n",
      "[['Politeness', 'polite'], ['Emotion', 'joy']]\n",
      "[10, 10]\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Gab_DialoGPT-medium_on_Gab_gedi_huggingface_1629484260_updated.json\n",
      "Length of Hypothesis:  100\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2a865c5cd7ba43059ae5057dadc0bc1f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-21 19:03:29.350 | INFO     | __main__:metrics:39 - Key:Gab_Gab_politeness_updated_0--Politeness_Score :6\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Gab_DialoGPT-medium_on_Gab_gedi_huggingface_1629486537_updated.json\n",
      "[['Politeness', 'polite']]\n",
      "[1]\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Gab_DialoGPT-medium_on_Gab_gedi_huggingface_1629486537_updated.json\n",
      "Length of Hypothesis:  100\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1218073f625b4e04b26d5122288e8bdc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-21 19:03:30.521 | INFO     | __main__:metrics:39 - Key:Gab_Gab_politeness_updated_0--Politeness_Score :5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Gab_DialoGPT-medium_on_Gab_gedi_huggingface_1629489852_single.json\n",
      "[['Toxicity', 'toxic']]\n",
      "[1]\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Gab_DialoGPT-medium_on_Gab_gedi_huggingface_1629489852_single.json\n",
      "Length of Hypothesis:  100\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1c3514c095c3419493d5d9f252c62aae",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-21 19:03:31.623 | INFO     | __main__:metrics:39 - Key:Gab_Gab_politeness_single_0--Politeness_Score :4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Gab_DialoGPT-medium_on_Gab_huggingface_1629551952_base.json\n",
      "[]\n",
      "[]\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Gab_DialoGPT-medium_on_Gab_huggingface_1629551952_base.json\n",
      "Length of Hypothesis:  100\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f47ecd13f6034c9a9a53d13de45342ce",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-21 19:03:32.479 | INFO     | __main__:metrics:39 - Key:Gab_Gab_politeness_base_0--Politeness_Score :4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Gab_DialoGPT-medium_on_Gab_gedi_huggingface_1629488002_multi.json\n",
      "[['Toxicity', 'toxic'], ['Emotion', 'joy']]\n",
      "[1, 1]\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Gab_DialoGPT-medium_on_Gab_gedi_huggingface_1629488002_multi.json\n",
      "Length of Hypothesis:  100\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bca3b79ddd224c9cb5ea299f41cd44c7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-21 19:03:33.482 | INFO     | __main__:metrics:39 - Key:Gab_Gab_politeness_multi_0--Politeness_Score :4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Gab_DialoGPT-medium_on_Gab_gedi_huggingface_1629484920_updated.json\n",
      "[['Politeness', 'polite']]\n",
      "[30]\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Gab_DialoGPT-medium_on_Gab_gedi_huggingface_1629484920_updated.json\n",
      "Length of Hypothesis:  100\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "90ba81226c33465c93f1e153a04ac52a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-21 19:03:34.431 | INFO     | __main__:metrics:39 - Key:Gab_Gab_politeness_updated_0--Politeness_Score :6\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Gab_DialoGPT-medium_on_Gab_gedi_huggingface_1629550555_multi_2.json\n",
      "[['Emotion', 'joy'], ['Politeness', 'polite']]\n",
      "[1, 1]\n",
      "/home/adarsh-binny/HULK_new/Counterspeech/Results/Generator-Gab_DialoGPT-medium_on_Gab_gedi_huggingface_1629550555_multi_2.json\n",
      "Length of Hypothesis:  100\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0a29984ab97d437dafde6b9801ffc82d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-21 19:03:35.321 | INFO     | __main__:metrics:39 - Key:Gab_Gab_politeness_2_0--Politeness_Score :6\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "Object of type Tensor is not JSON serializable",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-22-0e80bb7396d0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmetrics\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Gab'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'Gab'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-19-079b03eef31b>\u001b[0m in \u001b[0;36mmetrics\u001b[0;34m(trained_on, tested_on)\u001b[0m\n\u001b[1;32m     39\u001b[0m         \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'Key:{key}--Politeness_Score :{average_score}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 41\u001b[0;31m     \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscores\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msave_path\u001b[0m \u001b[0;34m+\u001b[0m\u001b[0mtested_on\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'_politeness.json'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'w'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mindent\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/.conda/envs/csgen/lib/python3.7/json/__init__.py\u001b[0m in \u001b[0;36mdump\u001b[0;34m(obj, fp, skipkeys, ensure_ascii, check_circular, allow_nan, cls, indent, separators, default, sort_keys, **kw)\u001b[0m\n\u001b[1;32m    177\u001b[0m     \u001b[0;31m# could accelerate with writelines in some versions of Python, at\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    178\u001b[0m     \u001b[0;31m# a debuggability cost\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 179\u001b[0;31m     \u001b[0;32mfor\u001b[0m \u001b[0mchunk\u001b[0m \u001b[0;32min\u001b[0m \u001b[0miterable\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    180\u001b[0m         \u001b[0mfp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    181\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/csgen/lib/python3.7/json/encoder.py\u001b[0m in \u001b[0;36m_iterencode\u001b[0;34m(o, _current_indent_level)\u001b[0m\n\u001b[1;32m    429\u001b[0m             \u001b[0;32myield\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0m_iterencode_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_current_indent_level\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    430\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 431\u001b[0;31m             \u001b[0;32myield\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0m_iterencode_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_current_indent_level\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    432\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    433\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mmarkers\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/csgen/lib/python3.7/json/encoder.py\u001b[0m in \u001b[0;36m_iterencode_dict\u001b[0;34m(dct, _current_indent_level)\u001b[0m\n\u001b[1;32m    403\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    404\u001b[0m                     \u001b[0mchunks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_iterencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_current_indent_level\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 405\u001b[0;31m                 \u001b[0;32myield\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mchunks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    406\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mnewline_indent\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    407\u001b[0m             \u001b[0m_current_indent_level\u001b[0m \u001b[0;34m-=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/csgen/lib/python3.7/json/encoder.py\u001b[0m in \u001b[0;36m_iterencode\u001b[0;34m(o, _current_indent_level)\u001b[0m\n\u001b[1;32m    436\u001b[0m                     \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Circular reference detected\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    437\u001b[0m                 \u001b[0mmarkers\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmarkerid\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mo\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 438\u001b[0;31m             \u001b[0mo\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_default\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    439\u001b[0m             \u001b[0;32myield\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0m_iterencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_current_indent_level\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    440\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mmarkers\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/csgen/lib/python3.7/json/encoder.py\u001b[0m in \u001b[0;36mdefault\u001b[0;34m(self, o)\u001b[0m\n\u001b[1;32m    177\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    178\u001b[0m         \"\"\"\n\u001b[0;32m--> 179\u001b[0;31m         raise TypeError(f'Object of type {o.__class__.__name__} '\n\u001b[0m\u001b[1;32m    180\u001b[0m                         f'is not JSON serializable')\n\u001b[1;32m    181\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: Object of type Tensor is not JSON serializable"
     ]
    }
   ],
   "source": [
    "metrics('Gab','Gab')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "****Toxicity_Score****"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def toxicity(trained_on,tested_on):\n",
    "    train_path = glob.glob(path_datasets+'/*'+trained_on+'*/*rain*')[0]\n",
    "    gen        = glob.glob(path_result+'/*'+trained_on+'*'+tested_on+'*1628*')\n",
    "    ref        = glob.glob(path_result+'/*'+tested_on+'*'+'references'+'*')[0]\n",
    "    \n",
    "#     print(gen,ref)\n",
    "    with open(ref, 'r') as file:\n",
    "        ref_dict   = json.loads(file.read())\n",
    "        \n",
    "    scores = {}\n",
    "    for files in gen:\n",
    "        with open(files, 'r') as file:\n",
    "            gen_dict  = json.loads(file.read())\n",
    "        emotion   = gen_dict['params']['task_name'][0][1]\n",
    "        gpu_id    = gen_dict['params']['gpu_id']\n",
    "        \n",
    "        detox = 0.0\n",
    "        tot   = 0.0\n",
    "        \n",
    "        for key in gen_dict['samples']:\n",
    "            for sentence in gen_dict['samples'][key]['counterspeech_model']:\n",
    "                detox += get_non_toxicity_score(sentence)\n",
    "                tot   += 1\n",
    "        \n",
    "        detox /= tot\n",
    "        key = trained_on+'_'+tested_on+'_'+emotion+'_'+str(gpu_id)\n",
    "        logger.info(f'Key -> {Key}-- Detox -> {detox}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "toxicity('Reddit','Reddit')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "toxicity('Gab','Gab')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "toxicity('CONAN','CONAN')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}