02-compute-metrics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     /Users/Egor.Bogomolov/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": "True"
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "from tqdm.auto import tqdm\n",
    "import nltk\n",
    "nltk.download(\"wordnet\")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from metrics_evaluation.metrics import (\n",
    "    codebleu,\n",
    "    ruby,\n",
    "    sentence_chrf,\n",
    "    sentence_bleu,\n",
    "    meteor,\n",
    "    rouge,\n",
    "    tokenize_tranx\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "def compute_metric_all(model_dictionary, model_list, metric_name, metric_function):\n",
    "    print(f\"Computing {metric_name}...\")\n",
    "    for item in tqdm(model_dictionary):\n",
    "        for field in model_list:\n",
    "            max_score = 0\n",
    "            hypothesis = item[field]\n",
    "            for snippet in item[\"snippet\"]:\n",
    "                cur_score = round(metric_function(snippet, hypothesis), 3)\n",
    "                max_score = max(cur_score, max_score)\n",
    "            item[f\"{metric_name}-{field}\"] = max_score"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [],
   "source": [
    "# ROUGE-L\n",
    "def rougel_all(model_dictionary, model_list):\n",
    "    compute_metric_all(\n",
    "        model_dictionary,\n",
    "        model_list,\n",
    "        \"rougel\",\n",
    "        lambda snippet, hypothesis: rouge._score_lcs(tokenize_tranx(snippet), tokenize_tranx(hypothesis)).fmeasure\n",
    "    )"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [],
   "source": [
    "# codebleu\n",
    "def codebleu_all(model_dictionary, model_list):\n",
    "    compute_metric_all(\n",
    "        model_dictionary,\n",
    "        model_list,\n",
    "        \"codebleu\",\n",
    "        lambda snippet, hypothesis: codebleu(snippet, hypothesis)\n",
    "    )"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [],
   "source": [
    "# ruby\n",
    "def ruby_all(model_dictionary, model_list):\n",
    "    compute_metric_all(\n",
    "        model_dictionary,\n",
    "        model_list,\n",
    "        \"ruby\",\n",
    "        lambda snippet, hypothesis: ruby(hypothesis, snippet)[0]\n",
    "    )"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [],
   "source": [
    "# METEOR\n",
    "def meteor_all(model_dictionary, model_list):\n",
    "    compute_metric_all(\n",
    "        model_dictionary,\n",
    "        model_list,\n",
    "        \"meteor\",\n",
    "        lambda snippet, hypothesis: meteor(tokenize_tranx(snippet), tokenize_tranx(hypothesis))\n",
    "    )"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [],
   "source": [
    "# chrF\n",
    "def chrf_all(model_dictionary, model_list):\n",
    "    print(\"Computing chrf...\")\n",
    "    for item in tqdm(model_dictionary):\n",
    "        for field in model_list:\n",
    "            item[f\"chrf-{field}\"] = round(sentence_chrf(item[field], item[\"snippet\"]).score / 100, 3)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [],
   "source": [
    "# BLEU\n",
    "def bleu_all(model_dictionary, model_list):\n",
    "    print(\"Computing bleu...\")\n",
    "    for item in tqdm(model_dictionary):\n",
    "        for field in model_list:\n",
    "            item[f\"bleu-{field}\"] = round(sentence_bleu(item[field], item[\"snippet\"]).score / 100, 3)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [],
   "source": [
    "def run_all_metrics(model_dictionary, model_list):\n",
    "    rougel_all(model_dictionary, model_list)\n",
    "    bleu_all(model_dictionary, model_list)\n",
    "    chrf_all(model_dictionary, model_list)\n",
    "    meteor_all(model_dictionary, model_list)\n",
    "    codebleu_all(model_dictionary, model_list)\n",
    "    # Ruby is slow as it computes graph edit distances which might take a while\n",
    "    ruby_all(model_dictionary, model_list)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing rougel...\n"
     ]
    },
    {
     "data": {
      "text/plain": "  0%|          | 0/472 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "c62b1fc61e8c4fe9bc8b11b26db4fb9c"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing bleu...\n"
     ]
    },
    {
     "data": {
      "text/plain": "  0%|          | 0/472 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "47cd8335430a4c04a838908a792acc10"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing chrf...\n"
     ]
    },
    {
     "data": {
      "text/plain": "  0%|          | 0/472 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "b1484544d7944eb09711447c0310f226"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing meteor...\n"
     ]
    },
    {
     "data": {
      "text/plain": "  0%|          | 0/472 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "e7f805371cca4f2d9cd51ade8284c53a"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing codebleu...\n"
     ]
    },
    {
     "data": {
      "text/plain": "  0%|          | 0/472 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "7d36a2fcd4cc4283b0af2100cbb9454d"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Symbol \"int\"@1:1 Not Found!\n",
      "WARNING:root:Symbol \"d\"@1:5 Not Found!\n",
      "WARNING:root:Symbol \"d\"@1:12 Not Found!\n",
      "WARNING:root:Symbol \"s\"@1:18 Not Found!\n",
      "WARNING:root:Symbol \"i\"@1:28 Not Found!\n",
      "WARNING:root:Symbol \"i\"@1:34 Not Found!\n",
      "WARNING:root:Symbol \"s\"@1:37 Not Found!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing ruby...\n"
     ]
    },
    {
     "data": {
      "text/plain": "  0%|          | 0/472 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "e9cf03f0267942b194eb22b9514ff1b1"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Symbol \"int\"@1:1 Not Found!\n",
      "WARNING:root:Symbol \"d\"@1:5 Not Found!\n",
      "WARNING:root:Symbol \"d\"@1:12 Not Found!\n",
      "WARNING:root:Symbol \"s\"@1:18 Not Found!\n",
      "WARNING:root:Symbol \"i\"@1:28 Not Found!\n",
      "WARNING:root:Symbol \"i\"@1:34 Not Found!\n",
      "WARNING:root:Symbol \"s\"@1:37 Not Found!\n"
     ]
    }
   ],
   "source": [
    "conala_models_list = ['baseline', 'tranx-annot', 'best-tranx', 'best-tranx-rerank', 'codex']\n",
    "conala_data = json.load(open(\"data/to-grade/conala/conala-aggregated-grades.json\"))\n",
    "run_all_metrics(conala_data, conala_models_list)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [],
   "source": [
    "json.dump(conala_data, open(\"data/to-grade/conala/conala-all-grades.json\", \"w\"))"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing rougel...\n"
     ]
    },
    {
     "data": {
      "text/plain": "  0%|          | 0/66 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "2ebdf5b0a73b4f6f89b01d40ca864a4a"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing bleu...\n"
     ]
    },
    {
     "data": {
      "text/plain": "  0%|          | 0/66 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "1d8b82c68ac04a789ed6087e74bd2418"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing chrf...\n"
     ]
    },
    {
     "data": {
      "text/plain": "  0%|          | 0/66 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "dc27aa7a518448ec93fed2094a6ec4bd"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing meteor...\n"
     ]
    },
    {
     "data": {
      "text/plain": "  0%|          | 0/66 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "0a0625f9c4b54a7dbd427642188ce33d"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing codebleu...\n"
     ]
    },
    {
     "data": {
      "text/plain": "  0%|          | 0/66 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "7538df4269564c62b736679f6d34ac69"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing ruby...\n"
     ]
    },
    {
     "data": {
      "text/plain": "  0%|          | 0/66 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "dd66ec478e474d6d8d637877017e05ea"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "hs_models_list = ['gcnn', 'nl2code']\n",
    "hs_data = json.load(open(\"data/to-grade/hs/hs-aggregated-grades.json\"))\n",
    "run_all_metrics(hs_data, hs_models_list)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "outputs": [],
   "source": [
    "json.dump(hs_data, open(\"data/to-grade/hs/hs-all-grades.json\", \"w\"))"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}