EVAL_VAD.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ecc39024",
   "metadata": {},
   "outputs": [],
   "source": [
    "from inaGVAD.vad_metrics import VadEval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f57db379",
   "metadata": {},
   "outputs": [],
   "source": [
    "fref = './annotations/vad/tv-TF1-20220218T133551-60.csv'\n",
    "fpred = './annotations/baseline_iss_vad/tv-TF1-20220218T133551-60.csv'\n",
    "dref = './annotations/vad/'\n",
    "dpred = './annotations/baseline_iss_vad/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a3c55f28",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'true negative': 4.879999999999999,\n",
       " 'true positive': 48.96499999999999,\n",
       " 'false negative': 0.18399999999998684,\n",
       " 'false positive': 6.195000000000016,\n",
       " 'retrieved': 55.160000000000004,\n",
       " 'relevant': 49.14899999999997,\n",
       " 'relevant retrieved': 48.96499999999999,\n",
       " 'basename': 'tv-TF1-20220218T133551-60',\n",
       " 'accuracy': 0.8940787725823592,\n",
       " 'precision': 0.887690355329949,\n",
       " 'recall': 0.9962562819182489,\n",
       " 'fmeasure': 0.9388451619706832}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# evaluation without collar (very strict)\n",
    "ve = VadEval(collar=0)\n",
    "ve.compare_csv(fref, fpred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7844b0db",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'true negative': 4.430000000000003,\n",
       " 'true positive': 44.649,\n",
       " 'false negative': 0.0,\n",
       " 'false positive': 2.4450000000000367,\n",
       " 'retrieved': 47.09400000000004,\n",
       " 'relevant': 44.649,\n",
       " 'relevant retrieved': 44.649,\n",
       " 'basename': 'tv-TF1-20220218T133551-60',\n",
       " 'accuracy': 0.952546386150143,\n",
       " 'precision': 0.9480825582876793,\n",
       " 'recall': 1.0,\n",
       " 'fmeasure': 0.9733494653543046}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# evaluation with 300ms collar (zones around reference boundaries are ignored)\n",
    "ve = VadEval(collar=.3)\n",
    "ve.compare_csv(fref, fpred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f1c94649",
   "metadata": {},
   "outputs": [],
   "source": [
    "# compare reference and hypothesis directories\n",
    "details, summary = ve.compare_directories(dref, dpred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8f797018",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.9316051037305464,\n",
       " 'precision': 0.9201954612806083,\n",
       " 'recall': 0.97343292945581,\n",
       " 'fmeasure': 0.9460658362837207}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# see result summary\n",
    "summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "16d5abdc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>true negative</th>\n",
       "      <th>true positive</th>\n",
       "      <th>false negative</th>\n",
       "      <th>false positive</th>\n",
       "      <th>retrieved</th>\n",
       "      <th>relevant</th>\n",
       "      <th>relevant retrieved</th>\n",
       "      <th>basename</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>fmeasure</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>47.721</td>\n",
       "      <td>5.352</td>\n",
       "      <td>4.350</td>\n",
       "      <td>0.338</td>\n",
       "      <td>5.690</td>\n",
       "      <td>9.702</td>\n",
       "      <td>5.352</td>\n",
       "      <td>radio-FIP-20220423T220055-60</td>\n",
       "      <td>0.918838</td>\n",
       "      <td>0.940598</td>\n",
       "      <td>0.551639</td>\n",
       "      <td>0.695426</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8.822</td>\n",
       "      <td>39.974</td>\n",
       "      <td>3.536</td>\n",
       "      <td>2.920</td>\n",
       "      <td>42.894</td>\n",
       "      <td>43.510</td>\n",
       "      <td>39.974</td>\n",
       "      <td>tv-FR2-20221206T050843-60</td>\n",
       "      <td>0.883154</td>\n",
       "      <td>0.931925</td>\n",
       "      <td>0.918731</td>\n",
       "      <td>0.925281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000</td>\n",
       "      <td>57.608</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.303</td>\n",
       "      <td>57.911</td>\n",
       "      <td>57.608</td>\n",
       "      <td>57.608</td>\n",
       "      <td>tv-BFT-20210409T165932-60</td>\n",
       "      <td>0.994768</td>\n",
       "      <td>0.994768</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.997377</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>60.032</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>radio-FUN-20220823T013906-60</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3.710</td>\n",
       "      <td>38.039</td>\n",
       "      <td>0.890</td>\n",
       "      <td>8.756</td>\n",
       "      <td>46.795</td>\n",
       "      <td>38.929</td>\n",
       "      <td>38.039</td>\n",
       "      <td>tv-TF1-20220709T021706-60</td>\n",
       "      <td>0.812316</td>\n",
       "      <td>0.812886</td>\n",
       "      <td>0.977138</td>\n",
       "      <td>0.887476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>280</th>\n",
       "      <td>24.529</td>\n",
       "      <td>29.232</td>\n",
       "      <td>0.822</td>\n",
       "      <td>0.580</td>\n",
       "      <td>29.812</td>\n",
       "      <td>30.054</td>\n",
       "      <td>29.232</td>\n",
       "      <td>tv-LCI-20210624T142245-60</td>\n",
       "      <td>0.974584</td>\n",
       "      <td>0.980545</td>\n",
       "      <td>0.972649</td>\n",
       "      <td>0.976581</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>281</th>\n",
       "      <td>0.000</td>\n",
       "      <td>57.578</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.354</td>\n",
       "      <td>57.932</td>\n",
       "      <td>57.578</td>\n",
       "      <td>57.578</td>\n",
       "      <td>radio-RMC-20220727T161225-60</td>\n",
       "      <td>0.993889</td>\n",
       "      <td>0.993889</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.996935</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282</th>\n",
       "      <td>27.014</td>\n",
       "      <td>27.262</td>\n",
       "      <td>3.794</td>\n",
       "      <td>0.867</td>\n",
       "      <td>28.129</td>\n",
       "      <td>31.056</td>\n",
       "      <td>27.262</td>\n",
       "      <td>tv-T5M-20210712T040130-60</td>\n",
       "      <td>0.920916</td>\n",
       "      <td>0.969178</td>\n",
       "      <td>0.877834</td>\n",
       "      <td>0.921247</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>283</th>\n",
       "      <td>3.560</td>\n",
       "      <td>41.584</td>\n",
       "      <td>9.261</td>\n",
       "      <td>0.868</td>\n",
       "      <td>42.452</td>\n",
       "      <td>50.845</td>\n",
       "      <td>41.584</td>\n",
       "      <td>tv-C+_-20221206T181000-60</td>\n",
       "      <td>0.816746</td>\n",
       "      <td>0.979553</td>\n",
       "      <td>0.817858</td>\n",
       "      <td>0.891433</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>284</th>\n",
       "      <td>9.280</td>\n",
       "      <td>33.654</td>\n",
       "      <td>0.000</td>\n",
       "      <td>4.648</td>\n",
       "      <td>38.302</td>\n",
       "      <td>33.654</td>\n",
       "      <td>33.654</td>\n",
       "      <td>radio-FCR-20220313T171139-60</td>\n",
       "      <td>0.902316</td>\n",
       "      <td>0.878649</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.935405</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>285 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     true negative  true positive  false negative  false positive  retrieved   \n",
       "0           47.721          5.352           4.350           0.338      5.690  \\\n",
       "1            8.822         39.974           3.536           2.920     42.894   \n",
       "2            0.000         57.608           0.000           0.303     57.911   \n",
       "3           60.032          0.000           0.000           0.000      0.000   \n",
       "4            3.710         38.039           0.890           8.756     46.795   \n",
       "..             ...            ...             ...             ...        ...   \n",
       "280         24.529         29.232           0.822           0.580     29.812   \n",
       "281          0.000         57.578           0.000           0.354     57.932   \n",
       "282         27.014         27.262           3.794           0.867     28.129   \n",
       "283          3.560         41.584           9.261           0.868     42.452   \n",
       "284          9.280         33.654           0.000           4.648     38.302   \n",
       "\n",
       "     relevant  relevant retrieved                      basename  accuracy   \n",
       "0       9.702               5.352  radio-FIP-20220423T220055-60  0.918838  \\\n",
       "1      43.510              39.974     tv-FR2-20221206T050843-60  0.883154   \n",
       "2      57.608              57.608     tv-BFT-20210409T165932-60  0.994768   \n",
       "3       0.000               0.000  radio-FUN-20220823T013906-60  1.000000   \n",
       "4      38.929              38.039     tv-TF1-20220709T021706-60  0.812316   \n",
       "..        ...                 ...                           ...       ...   \n",
       "280    30.054              29.232     tv-LCI-20210624T142245-60  0.974584   \n",
       "281    57.578              57.578  radio-RMC-20220727T161225-60  0.993889   \n",
       "282    31.056              27.262     tv-T5M-20210712T040130-60  0.920916   \n",
       "283    50.845              41.584     tv-C+_-20221206T181000-60  0.816746   \n",
       "284    33.654              33.654  radio-FCR-20220313T171139-60  0.902316   \n",
       "\n",
       "     precision    recall  fmeasure  \n",
       "0     0.940598  0.551639  0.695426  \n",
       "1     0.931925  0.918731  0.925281  \n",
       "2     0.994768  1.000000  0.997377  \n",
       "3     1.000000  1.000000  1.000000  \n",
       "4     0.812886  0.977138  0.887476  \n",
       "..         ...       ...       ...  \n",
       "280   0.980545  0.972649  0.976581  \n",
       "281   0.993889  1.000000  0.996935  \n",
       "282   0.969178  0.877834  0.921247  \n",
       "283   0.979553  0.817858  0.891433  \n",
       "284   0.878649  1.000000  0.935405  \n",
       "\n",
       "[285 rows x 12 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# details per file\n",
    "details"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "68842906",
   "metadata": {},
   "outputs": [],
   "source": [
    "# evaluate based on AVGBEST3_DIFFICULTY criterion\n",
    "df, summary = ve.compare_category(dpred, 'channel_category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8b70a20b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>fmeasure</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>generalist_radio</td>\n",
       "      <td>0.951689</td>\n",
       "      <td>0.952204</td>\n",
       "      <td>0.997257</td>\n",
       "      <td>0.974210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>generalist_tv</td>\n",
       "      <td>0.883284</td>\n",
       "      <td>0.873171</td>\n",
       "      <td>0.943247</td>\n",
       "      <td>0.906857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>music_radio</td>\n",
       "      <td>0.978965</td>\n",
       "      <td>0.925055</td>\n",
       "      <td>0.957480</td>\n",
       "      <td>0.940988</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>news_tv</td>\n",
       "      <td>0.947949</td>\n",
       "      <td>0.949738</td>\n",
       "      <td>0.992457</td>\n",
       "      <td>0.970627</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           category  accuracy  precision    recall  fmeasure\n",
       "0  generalist_radio  0.951689   0.952204  0.997257  0.974210\n",
       "1     generalist_tv  0.883284   0.873171  0.943247  0.906857\n",
       "2       music_radio  0.978965   0.925055  0.957480  0.940988\n",
       "3           news_tv  0.947949   0.949738  0.992457  0.970627"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# summary per category\n",
    "summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b938b85f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>true negative</th>\n",
       "      <th>true positive</th>\n",
       "      <th>false negative</th>\n",
       "      <th>false positive</th>\n",
       "      <th>retrieved</th>\n",
       "      <th>relevant</th>\n",
       "      <th>relevant retrieved</th>\n",
       "      <th>basename</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>fmeasure</th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>12.131</td>\n",
       "      <td>29.893</td>\n",
       "      <td>0.248</td>\n",
       "      <td>7.939</td>\n",
       "      <td>37.832</td>\n",
       "      <td>30.141</td>\n",
       "      <td>29.893</td>\n",
       "      <td>radio-FCR-20210309T031913-60</td>\n",
       "      <td>0.836948</td>\n",
       "      <td>0.790151</td>\n",
       "      <td>0.991772</td>\n",
       "      <td>0.879555</td>\n",
       "      <td>generalist_radio</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000</td>\n",
       "      <td>46.914</td>\n",
       "      <td>0.000</td>\n",
       "      <td>3.818</td>\n",
       "      <td>50.732</td>\n",
       "      <td>46.914</td>\n",
       "      <td>46.914</td>\n",
       "      <td>radio-FCR-20210627T083707-60</td>\n",
       "      <td>0.924742</td>\n",
       "      <td>0.924742</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.960900</td>\n",
       "      <td>generalist_radio</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.832</td>\n",
       "      <td>51.494</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.438</td>\n",
       "      <td>52.932</td>\n",
       "      <td>51.494</td>\n",
       "      <td>51.494</td>\n",
       "      <td>radio-FCR-20211004T074934-60</td>\n",
       "      <td>0.973253</td>\n",
       "      <td>0.972833</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.986229</td>\n",
       "      <td>generalist_radio</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6.386</td>\n",
       "      <td>38.849</td>\n",
       "      <td>0.000</td>\n",
       "      <td>2.497</td>\n",
       "      <td>41.346</td>\n",
       "      <td>38.849</td>\n",
       "      <td>38.849</td>\n",
       "      <td>radio-FCR-20211012T032010-60</td>\n",
       "      <td>0.947687</td>\n",
       "      <td>0.939607</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.968863</td>\n",
       "      <td>generalist_radio</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.000</td>\n",
       "      <td>48.313</td>\n",
       "      <td>0.000</td>\n",
       "      <td>5.398</td>\n",
       "      <td>53.711</td>\n",
       "      <td>48.313</td>\n",
       "      <td>48.313</td>\n",
       "      <td>radio-FCR-20211030T225430-60</td>\n",
       "      <td>0.899499</td>\n",
       "      <td>0.899499</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.947091</td>\n",
       "      <td>generalist_radio</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>1.128</td>\n",
       "      <td>55.713</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.640</td>\n",
       "      <td>56.353</td>\n",
       "      <td>55.713</td>\n",
       "      <td>55.713</td>\n",
       "      <td>tv-LCI-20220123T050538-60</td>\n",
       "      <td>0.988866</td>\n",
       "      <td>0.988643</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.994289</td>\n",
       "      <td>news_tv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>10.342</td>\n",
       "      <td>33.966</td>\n",
       "      <td>0.057</td>\n",
       "      <td>9.916</td>\n",
       "      <td>43.882</td>\n",
       "      <td>34.023</td>\n",
       "      <td>33.966</td>\n",
       "      <td>tv-LCI-20220214T061740-60</td>\n",
       "      <td>0.816271</td>\n",
       "      <td>0.774030</td>\n",
       "      <td>0.998325</td>\n",
       "      <td>0.871985</td>\n",
       "      <td>news_tv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>0.000</td>\n",
       "      <td>56.559</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.618</td>\n",
       "      <td>57.177</td>\n",
       "      <td>56.559</td>\n",
       "      <td>56.559</td>\n",
       "      <td>tv-LCI-20220301T100827-60</td>\n",
       "      <td>0.989191</td>\n",
       "      <td>0.989191</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.994566</td>\n",
       "      <td>news_tv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>0.000</td>\n",
       "      <td>56.299</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.667</td>\n",
       "      <td>56.966</td>\n",
       "      <td>56.299</td>\n",
       "      <td>56.299</td>\n",
       "      <td>tv-LCI-20220517T213519-60</td>\n",
       "      <td>0.988291</td>\n",
       "      <td>0.988291</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.994111</td>\n",
       "      <td>news_tv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>0.352</td>\n",
       "      <td>58.529</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.414</td>\n",
       "      <td>58.943</td>\n",
       "      <td>58.529</td>\n",
       "      <td>58.529</td>\n",
       "      <td>tv-LCI-20220926T174051-60</td>\n",
       "      <td>0.993018</td>\n",
       "      <td>0.992976</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.996476</td>\n",
       "      <td>news_tv</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>285 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    true negative  true positive  false negative  false positive  retrieved   \n",
       "0          12.131         29.893           0.248           7.939     37.832  \\\n",
       "1           0.000         46.914           0.000           3.818     50.732   \n",
       "2           0.832         51.494           0.000           1.438     52.932   \n",
       "3           6.386         38.849           0.000           2.497     41.346   \n",
       "4           0.000         48.313           0.000           5.398     53.711   \n",
       "..            ...            ...             ...             ...        ...   \n",
       "57          1.128         55.713           0.000           0.640     56.353   \n",
       "58         10.342         33.966           0.057           9.916     43.882   \n",
       "59          0.000         56.559           0.000           0.618     57.177   \n",
       "60          0.000         56.299           0.000           0.667     56.966   \n",
       "61          0.352         58.529           0.000           0.414     58.943   \n",
       "\n",
       "    relevant  relevant retrieved                      basename  accuracy   \n",
       "0     30.141              29.893  radio-FCR-20210309T031913-60  0.836948  \\\n",
       "1     46.914              46.914  radio-FCR-20210627T083707-60  0.924742   \n",
       "2     51.494              51.494  radio-FCR-20211004T074934-60  0.973253   \n",
       "3     38.849              38.849  radio-FCR-20211012T032010-60  0.947687   \n",
       "4     48.313              48.313  radio-FCR-20211030T225430-60  0.899499   \n",
       "..       ...                 ...                           ...       ...   \n",
       "57    55.713              55.713     tv-LCI-20220123T050538-60  0.988866   \n",
       "58    34.023              33.966     tv-LCI-20220214T061740-60  0.816271   \n",
       "59    56.559              56.559     tv-LCI-20220301T100827-60  0.989191   \n",
       "60    56.299              56.299     tv-LCI-20220517T213519-60  0.988291   \n",
       "61    58.529              58.529     tv-LCI-20220926T174051-60  0.993018   \n",
       "\n",
       "    precision    recall  fmeasure          category  \n",
       "0    0.790151  0.991772  0.879555  generalist_radio  \n",
       "1    0.924742  1.000000  0.960900  generalist_radio  \n",
       "2    0.972833  1.000000  0.986229  generalist_radio  \n",
       "3    0.939607  1.000000  0.968863  generalist_radio  \n",
       "4    0.899499  1.000000  0.947091  generalist_radio  \n",
       "..        ...       ...       ...               ...  \n",
       "57   0.988643  1.000000  0.994289           news_tv  \n",
       "58   0.774030  0.998325  0.871985           news_tv  \n",
       "59   0.989191  1.000000  0.994566           news_tv  \n",
       "60   0.988291  1.000000  0.994111           news_tv  \n",
       "61   0.992976  1.000000  0.996476           news_tv  \n",
       "\n",
       "[285 rows x 13 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# details per file\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ce3f4d09",
   "metadata": {},
   "outputs": [],
   "source": [
    "# evaluate based on AVGBEST3_DIFFICULTY criterion\n",
    "df, summary = ve.compare_category(dpred, 'AVGBEST3_DIFFICULTY')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "de0c7c94",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>fmeasure</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>EASY</td>\n",
       "      <td>0.979542</td>\n",
       "      <td>0.976897</td>\n",
       "      <td>0.994955</td>\n",
       "      <td>0.985843</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>HARD</td>\n",
       "      <td>0.842536</td>\n",
       "      <td>0.715697</td>\n",
       "      <td>0.848754</td>\n",
       "      <td>0.776567</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>MEDIUM</td>\n",
       "      <td>0.917601</td>\n",
       "      <td>0.903889</td>\n",
       "      <td>0.975984</td>\n",
       "      <td>0.938554</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  category  accuracy  precision    recall  fmeasure\n",
       "0     EASY  0.979542   0.976897  0.994955  0.985843\n",
       "1     HARD  0.842536   0.715697  0.848754  0.776567\n",
       "2   MEDIUM  0.917601   0.903889  0.975984  0.938554"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# summary per category\n",
    "summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "767a270f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>true negative</th>\n",
       "      <th>true positive</th>\n",
       "      <th>false negative</th>\n",
       "      <th>false positive</th>\n",
       "      <th>retrieved</th>\n",
       "      <th>relevant</th>\n",
       "      <th>relevant retrieved</th>\n",
       "      <th>basename</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>fmeasure</th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>60.032</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>radio-FBL-20210131T132313-60</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>EASY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>60.032</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>radio-FBL-20210404T234419-60</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>EASY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>37.436</td>\n",
       "      <td>18.489</td>\n",
       "      <td>0.322</td>\n",
       "      <td>1.535</td>\n",
       "      <td>20.024</td>\n",
       "      <td>18.811</td>\n",
       "      <td>18.489</td>\n",
       "      <td>radio-FBL-20210418T174018-60</td>\n",
       "      <td>0.967862</td>\n",
       "      <td>0.923342</td>\n",
       "      <td>0.982882</td>\n",
       "      <td>0.952182</td>\n",
       "      <td>EASY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>60.011</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>radio-FBL-20210502T035452-60</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>EASY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.734</td>\n",
       "      <td>56.049</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.528</td>\n",
       "      <td>56.577</td>\n",
       "      <td>56.049</td>\n",
       "      <td>56.049</td>\n",
       "      <td>radio-FBL-20210702T150454-60</td>\n",
       "      <td>0.990787</td>\n",
       "      <td>0.990668</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.995312</td>\n",
       "      <td>EASY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>3.710</td>\n",
       "      <td>38.039</td>\n",
       "      <td>0.890</td>\n",
       "      <td>8.756</td>\n",
       "      <td>46.795</td>\n",
       "      <td>38.929</td>\n",
       "      <td>38.039</td>\n",
       "      <td>tv-TF1-20220709T021706-60</td>\n",
       "      <td>0.812316</td>\n",
       "      <td>0.812886</td>\n",
       "      <td>0.977138</td>\n",
       "      <td>0.887476</td>\n",
       "      <td>MEDIUM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>29.101</td>\n",
       "      <td>18.252</td>\n",
       "      <td>1.579</td>\n",
       "      <td>5.456</td>\n",
       "      <td>23.708</td>\n",
       "      <td>19.831</td>\n",
       "      <td>18.252</td>\n",
       "      <td>tv-TF1-20220709T031751-60</td>\n",
       "      <td>0.870652</td>\n",
       "      <td>0.769867</td>\n",
       "      <td>0.920377</td>\n",
       "      <td>0.838421</td>\n",
       "      <td>MEDIUM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>12.554</td>\n",
       "      <td>35.729</td>\n",
       "      <td>0.000</td>\n",
       "      <td>3.105</td>\n",
       "      <td>38.834</td>\n",
       "      <td>35.729</td>\n",
       "      <td>35.729</td>\n",
       "      <td>tv-TF1-20220810T110550-60</td>\n",
       "      <td>0.939577</td>\n",
       "      <td>0.920044</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.958357</td>\n",
       "      <td>MEDIUM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>6.832</td>\n",
       "      <td>34.252</td>\n",
       "      <td>0.000</td>\n",
       "      <td>12.447</td>\n",
       "      <td>46.699</td>\n",
       "      <td>34.252</td>\n",
       "      <td>34.252</td>\n",
       "      <td>tv-TFX-20220724T050203-60</td>\n",
       "      <td>0.767481</td>\n",
       "      <td>0.733463</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.846240</td>\n",
       "      <td>MEDIUM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>40.720</td>\n",
       "      <td>7.775</td>\n",
       "      <td>0.000</td>\n",
       "      <td>7.436</td>\n",
       "      <td>15.211</td>\n",
       "      <td>7.775</td>\n",
       "      <td>7.775</td>\n",
       "      <td>tv-TFX-20221018T231251-60</td>\n",
       "      <td>0.867050</td>\n",
       "      <td>0.511143</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.676499</td>\n",
       "      <td>MEDIUM</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>285 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     true negative  true positive  false negative  false positive  retrieved   \n",
       "0           60.032          0.000           0.000           0.000      0.000  \\\n",
       "1           60.032          0.000           0.000           0.000      0.000   \n",
       "2           37.436         18.489           0.322           1.535     20.024   \n",
       "3           60.011          0.000           0.000           0.000      0.000   \n",
       "4            0.734         56.049           0.000           0.528     56.577   \n",
       "..             ...            ...             ...             ...        ...   \n",
       "99           3.710         38.039           0.890           8.756     46.795   \n",
       "100         29.101         18.252           1.579           5.456     23.708   \n",
       "101         12.554         35.729           0.000           3.105     38.834   \n",
       "102          6.832         34.252           0.000          12.447     46.699   \n",
       "103         40.720          7.775           0.000           7.436     15.211   \n",
       "\n",
       "     relevant  relevant retrieved                      basename  accuracy   \n",
       "0       0.000               0.000  radio-FBL-20210131T132313-60  1.000000  \\\n",
       "1       0.000               0.000  radio-FBL-20210404T234419-60  1.000000   \n",
       "2      18.811              18.489  radio-FBL-20210418T174018-60  0.967862   \n",
       "3       0.000               0.000  radio-FBL-20210502T035452-60  1.000000   \n",
       "4      56.049              56.049  radio-FBL-20210702T150454-60  0.990787   \n",
       "..        ...                 ...                           ...       ...   \n",
       "99     38.929              38.039     tv-TF1-20220709T021706-60  0.812316   \n",
       "100    19.831              18.252     tv-TF1-20220709T031751-60  0.870652   \n",
       "101    35.729              35.729     tv-TF1-20220810T110550-60  0.939577   \n",
       "102    34.252              34.252     tv-TFX-20220724T050203-60  0.767481   \n",
       "103     7.775               7.775     tv-TFX-20221018T231251-60  0.867050   \n",
       "\n",
       "     precision    recall  fmeasure category  \n",
       "0     1.000000  1.000000  1.000000     EASY  \n",
       "1     1.000000  1.000000  1.000000     EASY  \n",
       "2     0.923342  0.982882  0.952182     EASY  \n",
       "3     1.000000  1.000000  1.000000     EASY  \n",
       "4     0.990668  1.000000  0.995312     EASY  \n",
       "..         ...       ...       ...      ...  \n",
       "99    0.812886  0.977138  0.887476   MEDIUM  \n",
       "100   0.769867  0.920377  0.838421   MEDIUM  \n",
       "101   0.920044  1.000000  0.958357   MEDIUM  \n",
       "102   0.733463  1.000000  0.846240   MEDIUM  \n",
       "103   0.511143  1.000000  0.676499   MEDIUM  \n",
       "\n",
       "[285 rows x 13 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# details per file\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b92cb34",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}