fake-news-net-zero-shot.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-24T18:47:10.676855Z",
     "start_time": "2024-01-24T18:47:05.053216Z"
    }
   },
   "outputs": [],
   "source": [
    "! pip install transformers datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-24T18:46:45.206067Z",
     "start_time": "2024-01-24T18:46:41.878668Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "V4vJIZJeclwP",
    "outputId": "b20006dd-8811-4d0c-b81e-faad120f1a29",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'space-model'...\r\n",
      "remote: Enumerating objects: 151, done.\u001b[K\r\n",
      "remote: Counting objects: 100% (151/151), done.\u001b[K\r\n",
      "remote: Compressing objects: 100% (126/126), done.\u001b[K\r\n",
      "remote: Total 151 (delta 45), reused 125 (delta 22), pack-reused 0\u001b[K\r\n",
      "Receiving objects: 100% (151/151), 29.68 MiB | 26.09 MiB/s, done.\r\n",
      "Resolving deltas: 100% (45/45), done.\r\n"
     ]
    }
   ],
   "source": [
    "! rm -rf space-model\n",
    "! git clone https://github.com/StepanTita/space-model.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-24T18:56:46.114875Z",
     "start_time": "2024-01-24T18:56:46.113837Z"
    },
    "id": "LFX6x0OIc6uP",
    "tags": []
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "sys.path.append('space-model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-24T18:56:51.903728Z",
     "start_time": "2024-01-24T18:56:49.561278Z"
    },
    "id": "EzQpePJ4cXPG",
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "from transformers import AutoModel, AutoTokenizer, AutoModelForSequenceClassification\n",
    "\n",
    "from datasets import Dataset, DatasetDict\n",
    "\n",
    "from space_model.model import *\n",
    "from space_model.loss import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-24T18:56:51.903893Z",
     "start_time": "2024-01-24T18:56:51.903670Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "device_id = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-24T18:56:53.438166Z",
     "start_time": "2024-01-24T18:56:53.437709Z"
    },
    "id": "dPxAI9XscXPH",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device(f'cuda:{device_id}' if torch.cuda.is_available() else 'cpu')\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-24T18:56:54.147151Z",
     "start_time": "2024-01-24T18:56:54.081266Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "torch.cuda.set_device(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-24T18:57:09.710980Z",
     "start_time": "2024-01-24T18:57:09.555125Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>news_url</th>\n",
       "      <th>title</th>\n",
       "      <th>tweet_ids</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>speedtalk.com/forum/viewtopic.php?t=51650</td>\n",
       "      <td>BREAKING: First NFL Team Declares Bankruptcy O...</td>\n",
       "      <td>937349434668498944\\t937379378006282240\\t937380...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>politics2020.info/index.php/2018/03/13/court-o...</td>\n",
       "      <td>Court Orders Obama To Pay $400 Million In Rest...</td>\n",
       "      <td>972666281441878016\\t972678396575559680\\t972827...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>www.nscdscamps.org/blog/category/parenting/467...</td>\n",
       "      <td>UPDATE: Second Roy Moore Accuser Works For Mic...</td>\n",
       "      <td>929405740732870656\\t929439450400264192\\t929439...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://howafrica.com/oscar-pistorius-attempts...</td>\n",
       "      <td>Oscar Pistorius Attempts To Commit Suicide</td>\n",
       "      <td>886941526458347521\\t887011300278194176\\t887023...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>http://washingtonsources.org/trump-votes-for-d...</td>\n",
       "      <td>Trump Votes For Death Penalty For Being Gay</td>\n",
       "      <td>915205698212040704\\t915242076681506816\\t915249...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1051</th>\n",
       "      <td>https://www.flake.senate.gov/public/index.cfm/...</td>\n",
       "      <td>Flake: “Religious tests should have no place i...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1052</th>\n",
       "      <td>https://web.archive.org/web/20080131000131/htt...</td>\n",
       "      <td>Change We Can Believe In</td>\n",
       "      <td>634287923135909888\\t946743411100536832\\t946816...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1053</th>\n",
       "      <td>http://www.youtube.com/watch?v=4O8CxZ1OD58</td>\n",
       "      <td>deputy director of national health statistics ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1054</th>\n",
       "      <td>http://www.youtube.com/watch?v=EhyMplwY6HY</td>\n",
       "      <td>Romneys ProLife Conversion Myth or Reality Jun...</td>\n",
       "      <td>188871706637647874</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1055</th>\n",
       "      <td>https://web.archive.org/web/20071102131244/htt...</td>\n",
       "      <td>Interest Group Ratings</td>\n",
       "      <td>1002208963239337984\\t1024651239697666048</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1056 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               news_url  \\\n",
       "0             speedtalk.com/forum/viewtopic.php?t=51650   \n",
       "1     politics2020.info/index.php/2018/03/13/court-o...   \n",
       "2     www.nscdscamps.org/blog/category/parenting/467...   \n",
       "3     https://howafrica.com/oscar-pistorius-attempts...   \n",
       "4     http://washingtonsources.org/trump-votes-for-d...   \n",
       "...                                                 ...   \n",
       "1051  https://www.flake.senate.gov/public/index.cfm/...   \n",
       "1052  https://web.archive.org/web/20080131000131/htt...   \n",
       "1053         http://www.youtube.com/watch?v=4O8CxZ1OD58   \n",
       "1054         http://www.youtube.com/watch?v=EhyMplwY6HY   \n",
       "1055  https://web.archive.org/web/20071102131244/htt...   \n",
       "\n",
       "                                                  title  \\\n",
       "0     BREAKING: First NFL Team Declares Bankruptcy O...   \n",
       "1     Court Orders Obama To Pay $400 Million In Rest...   \n",
       "2     UPDATE: Second Roy Moore Accuser Works For Mic...   \n",
       "3            Oscar Pistorius Attempts To Commit Suicide   \n",
       "4           Trump Votes For Death Penalty For Being Gay   \n",
       "...                                                 ...   \n",
       "1051  Flake: “Religious tests should have no place i...   \n",
       "1052                           Change We Can Believe In   \n",
       "1053  deputy director of national health statistics ...   \n",
       "1054  Romneys ProLife Conversion Myth or Reality Jun...   \n",
       "1055                             Interest Group Ratings   \n",
       "\n",
       "                                              tweet_ids  label  \n",
       "0     937349434668498944\\t937379378006282240\\t937380...      1  \n",
       "1     972666281441878016\\t972678396575559680\\t972827...      1  \n",
       "2     929405740732870656\\t929439450400264192\\t929439...      1  \n",
       "3     886941526458347521\\t887011300278194176\\t887023...      1  \n",
       "4     915205698212040704\\t915242076681506816\\t915249...      1  \n",
       "...                                                 ...    ...  \n",
       "1051                                                NaN      0  \n",
       "1052  634287923135909888\\t946743411100536832\\t946816...      0  \n",
       "1053                                                NaN      0  \n",
       "1054                                 188871706637647874      0  \n",
       "1055           1002208963239337984\\t1024651239697666048      0  \n",
       "\n",
       "[1056 rows x 4 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# goss_fake_df = pd.read_csv('data/gossipcop_fake.csv', index_col=0)\n",
    "# goss_fake_df['label'] = 1\n",
    "# goss_fake_df['label'] = goss_fake_df['label'].astype(int)\n",
    "#\n",
    "# goss_real_df = pd.read_csv('data/gossipcop_real.csv', index_col=0)\n",
    "# goss_real_df['label'] = 0\n",
    "# goss_real_df['label'] = goss_real_df['label'].astype(int)\n",
    "\n",
    "politi_fake_df = pd.read_csv('data/politifact_fake.csv', index_col=0)\n",
    "politi_fake_df['label'] = 1\n",
    "politi_fake_df['label'] = politi_fake_df['label'].astype(int)\n",
    "\n",
    "politi_real_df = pd.read_csv('data/politifact_real.csv', index_col=0)\n",
    "politi_real_df['label'] = 0\n",
    "politi_real_df['label'] = politi_real_df['label'].astype(int)\n",
    "\n",
    "train_df = pd.concat([\n",
    "    # goss_fake_df,\n",
    "    # goss_real_df,\n",
    "    politi_fake_df,\n",
    "    politi_real_df\n",
    "], ignore_index=True, axis=0)\n",
    "\n",
    "train_df = train_df[train_df['title'].notnull()]\n",
    "train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-24T18:56:56.081282Z",
     "start_time": "2024-01-24T18:56:55.939678Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: >"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAGFCAYAAAASI+9IAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/OQEPoAAAACXBIWXMAAA9hAAAPYQGoP6dpAAApiUlEQVR4nO3dd3iUZb4+8PudmWRm0htJSAIkFGlCkCo2RLGgckBd13Xt5eh6POyu7Dnq+rNt0XXP2nYVXXRRiqjYFsSCAiqIDQhdek1vpE8yfX5/DA5EWsrMfN9yf64rF2ZMhps29zzP8z7PqwQCgQCIiIgAmKQDEBGRerAUiIgohKVAREQhLAUiIgphKRARUQhLgYiIQlgKREQUwlIgIqIQlgIREYWwFIiIKISlQEREISwFIiIKYSkQEVEIS4GIiEJYCkREFMJSICKiEJYCERGFsBSIiCiEpUBERCEsBSIiCmEpEBFRCEuBiIhCWApERBTCUiAiohCWAhERhbAUiIgohKVAREQhLAUiIgphKRARUQhLgYiIQlgKREQUwlIgIqIQlgJ12cyZM5Gfnw+bzYZx48ZhzZo10pGIqJtYCtQlCxcuxIwZM/Doo49i/fr1KCwsxCWXXILq6mrpaETUDUogEAhIhyDtGTduHMaMGYMXXngBAOD3+9GrVy9Mnz4dDzzwgHA6IuoqjhSo09xuN4qKijBp0qTQYyaTCZMmTcK3334rmIyIuoulQJ1WW1sLn8+HrKysdo9nZWWhsrJSKBURhQNLgYiIQlgK1GkZGRkwm82oqqpq93hVVRWys7OFUhFROLAUqNNiY2MxatQorFixIvSY3+/HihUrMH78eMFkRNRdFukApE0zZszAzTffjNGjR2Ps2LF47rnn4HA4cOutt0pHI6JuYClQl1x77bWoqanBI488gsrKSowYMQJLly49ZvGZiLSF+xSIiCiEawpERBTCUiAiohCuKZDuOD0+VDU50dDqgcPlRbPLixanFy2uoz6cXri8PgCASVGgKAAQ/FEBoCiAWVFgj7UgyW5Bki0GSfYYJNksSLTFIPnwYylxsYi18L0V6QdLgTSnstGJ/bUOVDS2oaLRGfyxwRn67/pWT9SyKArQI8GK3FQ7clPsyE21I+/wj7kpcchNtSPByn9mpB1caCbVqne4saOyGbuqjv5oQWNb9F70wyEz0YqB2Yk4LSsRp2UlYHDPJAzMToTVYpaORnQMlgKpgsPlxcaSBqw7UI+i4npsK29CbYtLOlbEWEwK+vVIwNCcJAzLS8aY/DQM6ZkEk0mRjkYGx1IgEZWNTqw7WId1B+qx7mAdtlc0w+c39l/FRJsFo/ukYkxBGsYVpGF4XgpizFyvoOhiKVBUNDs9+HpPLVbuqsHqPbUoqWuTjqR6thgTRvRKwbiCdEwclInCvGQoCkcSFFksBYqYfTUtWLatCp/vqMb64np4fPyr1h2ZiVZcODgTkwZn4ez+GbDFcE2Cwo+lQGG1saQBn2ypwLJtVdhX65COo1v2GDPO7p+Bi4Zk4sLBWchIsEpHIp1gKVC3HTzkwL83lGHxxnLsZxFEnUkBxvdLx1Vn5GHysGzExfISWOo6lgJ1SZ3DjSWbyrFoYxk2FDdIx6HD4mLNuHRoNq4elYfxfdN5NRN1GkuBOsznD2DZtiosXFuMr3bXwmvwq4XULifZhqln5OLqkXnon5kgHYc0gqVAp1TvcOPNtcVY8F0xyhp41ZAWndk3DbeeXYCLBmdx9EAnxVKgE9pa1oi53xzAB5vK4fL6peNQGPROi8NN4/vg2jG9kGiLkY5DKsRSoHa8Pj8+2VqJud8cwLqD9dJxKEISrBb8bFQebjkrH/kZ8dJxSEVYCgQgWAbvry/DzC/34OChVuk4FCWKAlwwMBPTLxyAEb1SpOOQCrAUDM7j8+PdolK8+OUe7jI2uAsGZeLeSadhWF6ydBQSxFIwKLfXj3eKSvDiF3u5eEztTBqchXsvGoChOSwHI2IpGIzX58fCdSwDOjlFAS4ekoV7LzoNg7KTpONQFLEUDGTlrhr8+cNt2F3dIh2FNEJRgCnDc/DA5EHISbFLx6EoYCkYwJ7qZvz5o+34cmeNdBTSKHuMGb+a0A93TejLg/h0jqWgYw2tbjy7bBcWfF/M3ccUFrkpdjx42WBcPryndBSKEJaCDnl9fsz79iD+vmK35m5dSdpwZt80PDplKAb35HqD3rAUdGZrWSP+993N2F7RJB2FdM5sUnDd2F6479JBSOLuaN1gKeiE0+PDs8t3YfZX+zlVRFHVM9mGJ64chomDMqWjUBiwFHTg+32H8Pv3t/CmNiTq6pF5eGTKECTbOWrQMpaChrW4vHjyk+1Y8H0x+KdIapCZaMUTVw7DpCFZ0lGoi1gKGvXV7hrc/+5mlDc6paMQHWPaiBw89h9DkRIXKx2FOomloDFenx9PfbYLs1bt5eiAVC0jwYqnrhmO8wdyrUFLWAoaUlLXil+/tYG3vyTNUBTgjnMKcN+lgxBjNknHoQ5gKWjEx1sqcP97m9Hs9EpHIeq0wrxkPH/dSPROj5OOQqfAUlA5p8eHP364DW98XywdhahbEm0WPHVNIS4Zmi0dhU6CpaBi+2pa8F8L1mNHZbN0FKKw+c9zC3D/pYNg4XSSKrEUVGrlrhpMf2M9mjhdRDo0Jj8VL90wChkJVuko9BMsBRWavXo/nvh4O3zcmUw6lptix6u3jMHA7ETpKHQUloKKeHx+PLxoK95aWyIdhSgqEqwWPH/dGTwiQ0VYCipR53DjV68XYc3+OukoRFFlNin4f5cNxm3nFEhHIbAUVGFHZRPumLsOpfW8PSYZ1w1n9sZjU4ZyAVoYS0HY13tqcdf8IrS4uKBMdO6ADMy8fiSP4hbEUhD00eYK3LtwI9w+v3QUItUYlJ2I+bePQ49EXpkkgaUg5PXvDuKRxVvBC4yIjtU3Ix6v3zEOOSl26SiGw1IQMPOLPfjbpzulYxCpWm6KHQvuGIf8jHjpKIbCUoiyvy7dgZe+3Csdg0gTeiRa8frt47iXIYpYClESCATw2Ac/YO63B6WjEGlKalwM5t42FsPzUqSjGAJLIUoeWbwV81gIRF2SaLVg9i1jMLYgTTqK7vGC4Cj4yyfbWQhE3dDs8uLW19ZgQ3G9dBTdYylE2N+X78aslfukYxBpnsPtwy2vrcWOyibpKLrGUoigV1btw7PLd0nHINKNxjYPbpy9BgdqHdJRdIulECHzvzuIxz/eLh2DSHdqml24/l/fo6KRx8JEAkshAt4rKsUji7dKxyDSrbKGNtzwr+9xqMUlHUV3WAph9vmOKtz33mbwmi6iyNpb48BNr65Bk9MjHUVXWAphtK28CdPf2MCb4xBFyQ/lTbhz3jp4eH5Y2LAUwqS6yYk75q6Fw+2TjkJkKN/tq8Mji3+QjqEbLIUwaHP7cMe8dShvdEpHITKkN9cUY/bq/dIxdIGl0E2BQAD3LtyIzaWN0lGIDO2Jj7fji53V0jE0j6XQTX9duhNLf6iUjkFkeD5/AL9+YwN2VzVLR9E0lkI3vL2uBP9cyRNPidSi2eXF7XPXoc7hlo6iWSyFLvqhvBEPL+JeBCK1Ka5rxa/mF/GKpC5iKXRBs9ODexash8vLv3REarTmQB1vZNVFFukAWvTAe1tw4FCrdAzVa1i9AI1fv9nuMUtaHnL/858AAE99Beq/mA1X6TYEfB7YC0Yh7aK7YI5PPenzNq//EI3fvw+fox6xmQVIm3QXrDkDQ/+/q89L+vLKV/swvm86Jg7KlI6iKbyfQifN/eYAHv2A10R3RMPqBWjd+TWyrn38yIMmE8xxyfC7nah47b8Rk1mAlHOuD379V6/D13II2Tc+DUU5/iDWsX0Vaj96BukX34PYnIFoXrcYrTtWI+c/Z8Ecn9Ll5yV9SouPxce/PhfZyTbpKJrBfyGdsLm0AY9/xEPuOsVkhjkh9chHXDIAwFW2Dd7GamRcdi9ie+Qjtkc+Mi6/F+6KPXAe3HzCp2tauwiJhZcgYfhFiM3ojbRL7oESY0XLlmXdel7SpzqHG79+i6cMdAZLoYMa2zy45431cHPxqlO89eUonXkTyv55O2qW/A3epuB15AFf8LwaxRwT+lrFHAsoClylxx+JBXweuCv3wNZnxJHvUUyw5Y+Aq2xHl5+X9G3N/jr8nUfYdxhLoYPuf3czSup4VG9nWHsORPpl9yLzmj8g7eL/gq+hCpUL7off1QprziAoMTbUf/ka/B4n/G4n6r+YDQT88LUc/+5avtYmIOCHOT6l3ePmuBT4HMHv6crzkv698MUefLOnVjqGJnChuQP+vaGUG9S6wN5v9JFPMgtgzRmI0pdug2PHaiQWXowe0x5A3WcvorloCaAoiB8yAbFZ/QBF6fLPaY5Ljsjzkrb5A8BvFm7E0t+ci/QEq3QcVWMpnEJ1kxOPfbBNOoYumGwJiEnLhbehHABgLxiJ3Lv+BV9rIxSTGSZbAkpeuAFxKdnH/X5zXBKgmOBzNLR73Nfa0O7Kos4+LxlDTbMLjy3ZhuevO0M6iqpx+ugUHvz3FjS28bz2cPC72+BtqIA5Pq3d4+a4ZJhsCWg7uAl+RyPi+o877vcr5hjEZveH8+Cm0GOBgB/OA5tgzR10zNd39HnJOJZsKsdnHPWfFEcKJ/FeUSmWb+cBW11V//ls2PuPhSU5E97mOjSuXgAoJsQPmQAAaNm8DDHpvWCKS4arfAfql7+MxDFTEZOeF3qOqrcehH3AeCSNmgIASBozDbUfPYvY7AGw9jwNTesWI+BxImHYpND3dOR5ybgeWrQV4/qmI9kec+ovNiCWwglUNTnxhyW8WqU7vM21qF3yN/jammC2J8OaNwTZNz4duizVU1eG+lVz4W9rgSU5E8njf47EMdPaPYenvhLWtqbQ5/GDz4OvtRENq18/vHmtLzJ//sd200cdeV4yrupmF/704TY8dU2hdBRV4ua1E7htzlp8voOjBCK9mnvbWEw4rYd0DNXhmsJxvFdUykIg0rkH39+CFpdXOobqsBR+osnpwV8+4a5lIr0ra2jDk/y3fgyWwk88t2w3alt4FjuREbzxfTG2lvGuiUdjKRxld1Uz5n17QDoGEUWJPwD8cQn3IR2NpXCUPyzZBi8PziIylDUH6vDR5grpGKrBUjhs6dYKrObZKESG9MTH2+H0+KRjqAJLAYDT48OfPuSCE5FRlTW04ZVV+6RjqAJLAcA/V+5FWQNPQCUyspdW7kVVk1M6hjjDl0JNswuzVvIdApHRtbp9ePKTHdIxxBm+FF76ci/aOJdIRAAWbSwz/CWqhi6FykYnFnx/UDoGEalEIAA8/dlO6RiiDF0KM7/YA5eXt9ckoiO+2FmD9cXGvUufYUuhrKENC9eWSMcgIhV65jPj3tPZsKXw/IrdcPs4SiCiY63eU4u1B+qkY4gwZCkcPOTAu0Wl0jGISMWe/3yPdAQRhiyFf6zYw+MsiOikVu2qwaaSBukYUWe4UqhqcuKDTWXSMYhIA57/fLd0hKgzXCnM/eYAPD6OEojo1FbsqMbemhbpGFFlqFJoc/vwxppi6RhEpBGBQPCNpJEYqhTeW1+KhlaPdAwi0pD3ikrR5DTO64ZhSiEQCODVr/dLxyAijXG4fXjbQHuaDFMKX+6swb4ah3QMItKg+d8dhN8gVywaphQ4SiCirjp4qBWf76iWjhEVhiiF3VXN+Go376pGRF03xyALzoYohbcMNB9IRJGxek8t9lQ3S8eION2Xgsfnx6IN3KxGRN33zjr9H4+j+1JYvq0Khxxu6RhEpAOLN5brfsFZ96XwDg++I6IwqWxy4pu9h6RjRJSuS6G2xYVVu2qkYxCRjry/Qd9vNHVdCks2lfM0VCIKq0+3VqLNrd/7uuu6FP7NBWYiCjOH24dPf6iUjhExui2FfTUt2FzaKB2DiHRIz284dVsKn/5QJR2BiHRq9Z5aVDc7pWNEhG5LYdk2/Q7viEiWzx/A8m36PPZCl6VQ0+zCRgPeRo+IomfFdn3ORuiyFFZsrwIvOiKiSPp6by2cHv1dhaTLUli2TZ8NTkTq4fT48c1e/R20qbtSaHP78LUO/6CISH2Wb9ffuoLuSmHV7ho4PX7pGERkAF/o8B4LuiuF5Zw6IqIoqWh04odyfe2H0l0prN7DqSMiip4VOptC0lUplNS1oqJRnxtKiEidVuvsro66KoXv9un7SFsiUp9NpQ1we/WzjqmrUvh+f510BCIyGJfXj82lDdIxwkZnpcCRAhFF39oD9dIRwkY3pVDe0IaSujbpGERkQGsP6GeWQjelwFECEUkpOliPQEAfZ+vopxT26aepiUhbGts82FnVLB0jLHRTCuuL9TOnR0Tao5d1BV2UgtPjw74ah3QMIjKw9QdZCqqxu6oFXp6VTUSCtlc0SUcIC12Ugl7+MIhIu/bVOOD1aX8Tmy5KYRtLgYiEuX1+7KvV/jQ2S4GIKEx2VGr/CiRdlMIOlgIRqcDOSu2/Fmm+FErrW9Hk9ErHICLCTo4U5OnhD4GI9IHTRypw8FCrdAQiIgBAWUMbWlzanrnQfCmU1LMUiEgdAgGgWONvVDVfCqX1PBmViNSjvEHbr0ksBSKiMCpv1PZrkg5KQdtDNSLSlzKOFOQ0tnnQzMtRiUhFKhqc0hG6RdOlwFECEakN1xQEcT2BiNSGpSCoutklHYGIqJ2qZhd8Gj7KX9Ol0NTmkY5ARNSOzx9AbYt237BquhQaWt3SEYiIjqHlN6yaLoVGDf/GE5F+afmQTpYCEVGYNTu1+9rEUiAiCjMtH4qn6VJoaGUpEJH6aHlTraZLQcuLOUSkX5w+EqLlNiYi/dLya5OmS8Ht80tHICI6BktBSEC7mwaJSMdcXp90hC7TdCn42QpEpEJ+DU9iaLoUfCwFIlIhLb82aboUNPz7TkQ6puVZDIt0gK7ya/gUQlKXXJsLc3p9jLzW7dJRSCfcCRcBGCEdo0s0WwpaHp6Revy+zy7c0fISzCVV0lFIR+y9CqUjdJlmS0HLwzOSNzihFa9lLkR2+TLpKKRHJrN0gi7TbClYTJpeDiEhihLA03034sraWVDKm6TjkF4pLIWoM5sUWC0muLwavvaLouqctEbMTJyL5LLvpKOQ3pk0+9Kq3VIAgHirBS4vb7RDJ2c1+TGr3zeYUPEqlCqndBwyAk4fyYi3mlHnkE5BajY1qxpPWv4Fe8lW6ShkJBabdIIu03YpxGo6PkVQcowXc/OXo7B0AZSAdo8cII2Kz5BO0GWaflWNt2o6PkXIHXkluN/zEmJKDkhHIaOKYymIiIvV7rwdhV+uzYX5eR+gb+m/paOQ0XGkICOBIwU67MH8nbi9+SWYS6uloxABcWnSCbpM06+qnD6ioYkOvNpjIbLKl0tHITqC00cy0hNipSOQEG5CI1Xj9JGMzETtXvZFXXdOWiNeTJyDpLLvpaMQHctiB2LjpVN0mcZLwSodgaKIm9BIE+LSpRN0C0uBNGFaVjX+YnkF9pIfpKMQnVw8S0FMz2S7dASKsOQYL+YVLMPwkje4CY20QcOLzIDGSyE72QZF4R3Y9Cq0Ca34gHQUoo7j9JGcWIsJGQlW1DS7pKNQGOXZXJiftxgFpYukoxB1XnKedIJu0XQpAEBOip2loCP/L38nbuMmNNKyHoOkE3SL5kuhd1ocNpU0SMegbuImNNKNHgOlE3SL5kthQGaCdATqBkUJ4Jm+GzCt9mVuQiMdUICM06RDdIvmS+G0LJaCVp2X1oAXEucgqWyNdBSi8EjpBcTGSafoFs2XwoCsROkI1ElWkx8v9/sa51W8xk1opC8Z2p46AnRQCvnp8Yg1m+D28V7NWjAtqxpPWl6GrWSbdBSi8NP4egKgg1IwmxT07RGPHZXN0lHoJFJjvJiTvwzDS7kJjXRMB6Vgkg4QDpxCUrc784qxJvVhFJbMZyGQvmn8clRAByMFgFcgqRU3oZHh6GCkoItSOI0jBdV5KH8nbmt6EabSGukoRNGRkA3YkqVTdJsuSmFErxTpCHTY0EQHXuvxFjLLV0hHIYquTO1PHQE6KYXsZBtykm0ob+TljVIUJYBn+27AVG5CI6PKGyudICx0UQoAMLJPKso3V0jHMCRuQiMCkH+2dIKw0MXVRwAwuk+qdATDsZr8mDfgK8x1z0BSFQuBDMwUw5GC2ozqkyYdwVCuyqrGE9yERhSUO1Lzx1v8SDelMLhnIuJizWh18zr4SEqN8WJu/jIM4yY0oiP66GPqCNDR9JHFbMLwPO1fDqZmP25CG85NaETt6WQ9AdDRSAEARvVJxXf76qRj6A43oRGdhMkC9DpTOkXY6GakAADj+2r7htlq9HD+DqyKu4+FQHQiPQsBq35OVdDVSGFsQRrXFcJkWKIDs7kJjejUdLSeAOhspBBrMeGsfhwtdIeiBPD3fuvxgWkGC4GoI/LPkU4QVroqBQCYOKiHdATNmpBej029nsPUsqeguHgUOdEpKSagt37WEwCdTR8BwMSBmdIRNMdq8uOVfl/j3PJXoThc0nGItCNnpC4OwTua7kohJ8WOgVmJ2FnFd7odcVVWNZ4wz4KtZLt0FCLtGTpNOkHY6a4UAOD8QT1YCqeQGuPFvILPcHrJm9xzQNRVQ6ZJJwg73a0pAJxCOpW7Dm9CG1b8OguBqKtyRwMpvaRThJ0uRwqj+6QiNS4G9a0e6Siq0tvuxNzcD7jngCgcdDh1BOh0pGAxmzB5WE/pGKrycMEOfGm/n4VAFC5DpkoniAhdlgIATC3MkY6gCsMSHVjTdzZur/gjTK28NSZRWOSOAlJ6S6eICF1OHwHB3c1GvhubogTwXN/1+I/al6GUc9GdKKx0uMD8I92OFBRFwZQRxhwtHNmE9jQ3oRFFgk7XEwAdlwIATC3MlY4QVXazD/MHrMIc5wwkVa+VjkOkTzqeOgJ0PH0EAENyknBaVgJ2VbVIR4m4q7Oq8Lj5ZW5CI4o0HU8dATofKQDA1BH6Hi2kxnixZMBHeKrpf2CrYyEQRZRiAoZeKZ0iogxQCjkwKdIpIuPuXgexNvUhDCtZwE1oRNEw4BJdblg7mq6njwAgLzUOEwdmYsWOaukoYdPb7sS83MXIL10sHYXIWMbdKZ0g4nQ/UgCAm87Kl44QNo8UbMeX9vtYCETRlnEa0HeidIqI0/1IAQDOG5CBgox47K91SEfpsmGJDrza4030KP9cOgqRMY29E1B0Ohd9FEOMFBRFwQ1n9pGO0SWKEsA/+q/DB6YZLAQiKdYkoPA66RRRYYhSAICfjcqDPcYsHaNTJqTXY3OvZ/Efpc9wExqRpMLrAGuCdIqoMEwpJNtjMO0Mbexwtpt9eH3ASsxxzkBi9TrpOEQGpwSnjgzCMKUAADeNz5eOcEpXZ1VhQ9YTOKdkFhQfb41JJK7fRCCjv3SKqDHEQvOPBvdMwvi+6fh23yHpKMdIjfFiXv6nOL30TSgBv3QcIvrR2LukE0SVoUYKADD9AvU1fvtNaCwEItVIzQcGXCydIqoMNVIAgLP6Z2BUn1QUHayXjnJ4E9oi5Jd+IB2FiI5n3N2AyVjvnY31qz1MDaOFR/N/3ITGQiBSpaQ8YPSt0imiznAjBQA4f2AmCvOSsam0Meo/9/CkFszOeIt7DojUbsJ9gMUqnSLqDDlSAIDpFwyI6s/34ya0xcrvWAhEapfWDxhxvXQKEYYcKQDApCFZGNIzCdsqmiL+c52fVo/nE15DYin3HBBpwsQHAbMxXx4NO1IAIr+28OMmtNdc3IRGpBlZpwOnXy2dQowxq/CwS0/Pjtjaws+yq/Bn0yzYSnaE/bmJKIIueMgQB9+diKFHCoqi4KErhoT1OdNjPfhwwEf4W+PvYKtjIRBpSt4YYOBk6RSiDF0KADAmPw2TT88Oy3Pd3esAvk95CKdzExqRNl34iHQCcYaePvrR7ycPxort1XD7uvZC3tvuxPzcRejDPQdE2lUwASg4TzqFOMOPFACgd3ocbj6ra/dbePTwndBYCEQax1ECAJZCyPQLByAtPrbDXz88qQVr+76CWyv+BFNrbQSTEVHEFf4SyBstnUIVWAqHJdli8NtJp97QpigBPN9vHRZjBnqUfxGFZEQUUXHpwCWPS6dQDZbCUX45tjdOyzrx3ZXOTwveCW1K2TNQ3C1RTEZEEXPJE0BcmnQK1WApHMViNuGJK4cdc4my3ezDAm5CI9KfvucDhb+QTqEqLIWfGJ2fhuvG9g59fk12JTZm/Rln805oRPpisQGXPyOdQnV4Sepx3H/pIKzfXYqn0j/A0NK3uOeASI/O+18gvZ90CtXhSOE4ku0x+OgKH04veYOFQMd4crULyh+a8NulztBjTm8A93zUhvT/a0bCE024+u1WVLWc/O9OVYsftyxqQ87TzYh7vAmXvu7A7kO+dl/zcpEb589xIOkvTVD+0IQGZyAivybDyRwCnP0b6RSqxFI4AfOQK4Ah06RjkMqsLfNhVpEbw7Pa/9O5d6kTS3Z58c41dqy8JR7lzQFc9XbbCZ8nEAhg2sI27Kv3Y/Ev4rDhrnj0STZh0vxWONxHXvhbPQFc2t+CB8813rn+kaMAU/4OmGOkg6gSS+FkLn86eLkaEYAWdwDXv9+GV6bYkWo7cjVCozOA2Rs8eOYSGy4osGBUjhmvTbXhmxIfviv1Hve5dtf58V2pDy9dbsOYXDMGZpjx0hU2tHmAN7d6Ql/32zOteOAcK87MM0f812cYo28Deo2VTqFaLIWTic8ALvubdApSiXs+duLyARZM6tt+Ka6owgePH+0eH5RhRu9kBd+W+H76NAAA1+GusFmOlItJUWC1AKuLj/89FAYJ2cCkR6VTqBpL4VROvxoYPEU6BQl7a6sH6yt8+MukY6dxKlsCiDUDKbb21zJnxSuobDn+GsCgDBN6Jyv4/Qon6tsCcPsC+OtqF0qbAqg4xVoEdZUCTJ0J2JKlg6gaS6EjrngOSMyRTkFCShr9+M1SJxZcZW/3zr47YswK3v95HHYd8iPt/5oR93gzvjjgxeT+FpiMe5R/ZI2/BxgwSTqF6vGS1I6IzwCueQ2YczngP/4cMelXUYUP1Y4ARs5yhB7zBYBVB314YY0bn94QB7cPaHAG2o0WqhwBZCec+BV+VI4ZG3+VgEZncKTQI96Ecf9qweieXD8Iu54jgAs5bdQRLIWO6n0mMOkx4LOHpJNQlF1YYMGWu+PbPXbr4jYMyjDj/rNj0SvJhBgTsGKfF1cPCV7RsrPWh+LGAMb3OvULfLJNAaBg9yEf1pX78aeJtkj8MowrNgH42auApeMHXhoZS6EzzpoOFH8H7PhQOglFUaJVwemZ7V/c42MUpNuPPH77GTGY8ZkTaXYFSVYF0z9xYnyeGWfmHbX4/EIL/nKhFVcODhbHOz940CNeQe9kE7ZU+fCbpU5MG2TBxf2OfE9lix+VLQHsqQuuM2yp8iHRGvyeNDvnmTrk8qe5Sa0TWAqdNe1FYNZWoP6AdBJSkWcvtcH0qRNXv90Klw+4pJ8FL17e/h3/zkN+NLqOLDxXtPgx4zM3qloC6Jmo4KbhMXh4QvuF7H+uc+MPK92hz8+b0woAeG2qDbeM4DvfUzrjRp5t1ElKIBDgFsnOqtgEzL4Y8DpP/bVEJCN7OHD7MiCG03GdwauPuqJnIXDpk9IpiOhEbMnAz+exELqApdBVo28FhnNYSqQ+CjDtn0BagXQQTWIpdMcVzwYP1iIi9Th3BjDoMukUmsVS6I7YuOAQ1codkkSqMOwa4IKHpVNoGkuhuzIGANe9AZh5iiWRqPxzgakv4phbJ1KnsBTCIf8c4KqXAYW/nUQiMocAv1jADWphwFexcBk6Dbj0r9IpiIwnMQe4/l0edBcmLIVwGncncM4M6RRExmFNAm54F0jOlU6iGyyFcJv0KDDieukURPpnigGufR3IGiqdRFdYCpEw5R/AgEukUxDp2OF7I/SdIB1Ed1gKkWC2ANfMAXJHSych0qcLHwYKr5VOoUsshUiJjQOufwdIHyCdhEhfxt0NnPs76RS6xVKIpLg04Mb3gZTe0kmI9OGsXwOTee5YJPGU1GhoLAPmTQUO7ZZOQqRdE+4HJj4onUL3WArR0lIDvH4lULlFOgmR9lz4aPBMI4o4lkI0tTUAC34GlK6VTkKkEUrwmPozfyUdxDBYCtHmagHeug7Yv0o6CZHKKcGTiEffKh3EUFgKEjxO4J2bgV1LpZMQqZNiDu5DGHGddBLDYSlI8XmA9+8EfnhfOgmRuphigKtfAYZeKZ3EkFgKkvx+YMmvgQ3zpZMQqYPFDvzsVd4kRxBLQVogACx7BPjmH9JJiGQl5QaPv845QzqJobEU1GLTW8CS3wBep3QSoujLGwNcuwBIzJJOYngsBTUpWw8svAFoKpNOQhQ9hb8EpjwHWHj3QjVgKahNSw3w9k1A8TfSSYgiSzEDF/0ROOu/pZPQUVgKauTzAJ/cD6ybLZ2EKDJsycEF5f6TpJPQT7AU1KxoLvDx/wA+t3QSovBJ7w9c9xaQwROE1YiloHYla4CFNwItldJJiLqv/yTg6tmAPUU6CZ0AS0ELmiqCC9Bl66STEHWNyQKcdx9w3v8AJrN0GjoJloJW+DzAyr8CXz0DBHzSaYg6Ln0AcNXLQO5I6STUASwFrSktAhb9CqjdJZ2E6BQUYMwdwMV/AmLs0mGog1gKWuRpA1b8EfjuJQD84yMVSuwZPNCu/4XSSaiTWApadmA1sOhuoKFYOgnREUOvBC5/Jng7WtIcloLWuZqBTx8E1s+TTkJGZ0sGLnsKGP5z6STUDSwFvdj1GfDBdF66SjIKzgOmvQQk50knoW5iKehJax2w9PfA5oXgWgNFRWLP4FEVHB3oBktBj0rWAksf4L4GihyzNXhm0bm/A2LjpdNQGLEU9CoQALa8Ayx/jKeuUngNugK4+M9AWoF0EooAloLeuVuDN/D5+u+Ap1U6DWlZj0HApU8C/SZKJ6EIYikYRVN5cNSw+W1wvYE6xZYMnP9gcCOa2SKdhiKMpWA0pUXB9YbSNdJJSO0UMzDyJuCCh4H4dOk0FCUsBaPa8i6w8v+A2p3SSUhtzLFA4S+As38LpPeTTkNRxlIwskAA2PkxsPpZoHStdBqSFhMHjLwZOGs6kJwrnYaEsBQo6MDqYDnsWS6dhKLNmgyMvQM487+A+AzpNCSMpUDtVW4BVj8H/PBvHtGtd3HpwJl3A2PvDC4mE4GlQCdStx/45nlg4wLA65ROQ+GUmBOcIhp1CxAbJ52GVIalQCfXUgOsmQWsn89zlbRMMQePsT7jRmDgZMAcI52IVIqlQB3j8wK7PwM2zA/+6PdKJ6KOSOsHnHE9UPhLIKmndBrSAJYCdV5zJbDxjeDHod3SaeinYuKBIVOBM24A8s+WTkMaw1Kg7ilbH9wlvfU9wFEtncbY8sYGi+D0qwBronQa0iiWAoWH3wfs+wLY/A6w+1OgrV46kTH0LAQGXg4MnQb0GCidhnSApUDh5/cBZUXBPQ+7lwEVG4GAXzqVPphigIJzgYGXBT+4yYzCjKVAkeeoBfZ+HiyIvZ8DrbXSibQlpTfQ70Kg/6TgHc5sSdKJSMdYChRdgQBQvuHIKKKsiJvkfiohC8gdBRRMCF5GmjFAOhEZCEuBZLkdwV3U5RuBik3BqaaancYpCmsykDMCyB0J5IwMlgGnhEgQS4HUx90KVG39SVHs0P7eCIsNyB7evgDS+wGKIp2MKISlQNrgcQaL4tBeoLEEaCw98tFUBriapBMCUIJTPym9j/PRB0jN501qSPVYCqQPzsb2RdFYGrzbnLsF8LQFz2/ytAbLxdvW/kefq/1zmSzBDWCxccHjpGPjgx8//W97CpDc68iLfkovwGIV+eUThQtLgcjvDxaEzx18secLOxkYS4GIiEJM0gGIiEg9WApEOrBq1SpMmTIFOTk5UBQFixYtko5EGsVSINIBh8OBwsJCzJw5UzoKaRyvjyPSgcmTJ2Py5MnSMUgHOFIgIqIQlgIREYWwFIiIKISlQEREISwFIiIK4dVHRDrQ0tKCPXv2hD7fv38/Nm7ciLS0NPTu3VswGWkNj7kg0oEvv/wSEydOPObxm2++GXPmzIl+INIslgIREYVwTYGIiEJYCkREFMJSICKiEJYCERGFsBSIiCiEpUBERCEsBSIiCmEpEBFRCEuBiIhCWApERBTCUiAiohCWAhERhbAUiIgohKVAREQhLAUiIgphKRARUQhLgYiIQlgKREQUwlIgIqIQlgIREYWwFIiIKISlQEREISwFIiIKYSkQEVEIS4GIiEJYCkREFMJSICKiEJYCERGFsBSIiCiEpUBERCEsBSIiCmEpEBFRCEuBiIhC/j+YwfJAR00YOAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_df.groupby('label').size().plot(kind='pie', autopct='%.2f')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-24T19:02:10.108117Z",
     "start_time": "2024-01-24T19:02:10.106274Z"
    },
    "id": "5b6HjLMEcXPJ",
    "tags": []
   },
   "outputs": [],
   "source": [
    "MODEL_NAME = 'bert-base-cased'\n",
    "MODELS_PATH = 'models'\n",
    "DATASET_NAME = 'fake-news-net'\n",
    "\n",
    "# SEED = \n",
    "NUM_EPOCHS = 50\n",
    "BATCH_SIZE = 256\n",
    "MAX_SEQ_LEN = 512\n",
    "N_LATENT = 128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-24T18:57:16.405471Z",
     "start_time": "2024-01-24T18:57:16.362741Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    test: Dataset({\n",
       "        features: ['title', 'label'],\n",
       "        num_rows: 1056\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_dataset = DatasetDict({\n",
    "    'test': Dataset.from_pandas(train_df[['title', 'label']]),\n",
    "})\n",
    "val_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-24T18:57:18.665526Z",
     "start_time": "2024-01-24T18:57:18.522428Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "7CdJl4UEcXPJ",
    "outputId": "ee737554-14a9-4e2c-86c9-35c190b1ec64",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertTokenizerFast(name_or_path='bert-base-cased', vocab_size=28996, model_max_length=512, is_fast=True, padding_side='right', truncation_side='right', special_tokens={'unk_token': '[UNK]', 'sep_token': '[SEP]', 'pad_token': '[PAD]', 'cls_token': '[CLS]', 'mask_token': '[MASK]'}, clean_up_tokenization_spaces=True),  added_tokens_decoder={\n",
       "\t0: AddedToken(\"[PAD]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t100: AddedToken(\"[UNK]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t101: AddedToken(\"[CLS]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t102: AddedToken(\"[SEP]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t103: AddedToken(\"[MASK]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)\n",
    "tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-24T18:57:29.995589Z",
     "start_time": "2024-01-24T18:57:29.397224Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0cb4e7b9a8c74948955643edc44b5f2b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/1056 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    test: Dataset({\n",
       "        features: ['title', 'label', 'input_ids', 'token_type_ids', 'attention_mask'],\n",
       "        num_rows: 1056\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_dataset = val_dataset.map(lambda x: tokenizer(x['title'], truncation=True, padding='max_length', max_length=MAX_SEQ_LEN, return_tensors='pt'), batched=True)\n",
    "val_dataset.set_format('torch', device=device)\n",
    "val_dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Base Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-24T18:57:23.853411Z",
     "start_time": "2024-01-24T18:57:21.862268Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "state_dict = torch.load(f'{MODELS_PATH}/{DATASET_NAME}_{MODEL_NAME}_{NUM_EPOCHS}.bin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-24T18:57:24.260940Z",
     "start_time": "2024-01-24T18:57:23.853097Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "7LtAcsA8cXPK",
    "outputId": "363eca6c-f4a9-42fa-d817-05d9d2362878",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertForSequenceClassification(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(28996, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0-11): 12 x BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.1, inplace=False)\n",
       "  (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base_model = AutoModelForSequenceClassification.from_pretrained(MODEL_NAME, state_dict=state_dict).to(device)\n",
    "base_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-24T18:57:35.042454Z",
     "start_time": "2024-01-24T18:57:35.042301Z"
    },
    "id": "cgyRO91Ng17v"
   },
   "outputs": [],
   "source": [
    "def eval(f):\n",
    "    def wrapper(model, *args, **kwargs):\n",
    "        model.eval()\n",
    "        return f(model, *args, **kwargs)\n",
    "    return wrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-24T18:57:36.504479Z",
     "start_time": "2024-01-24T18:57:36.501544Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def concept_space_to_preds(concept_spaces):\n",
    "    tensor_concept_spaces = torch.cat([cs.unsqueeze(0) for cs in concept_spaces], dim=0)\n",
    "    concept_space_dist = tensor_concept_spaces.permute(1, 0, 2, 3).mean(dim=(2, 3)) # (B, n)\n",
    "    return torch.argmax(concept_space_dist, dim=1).detach().cpu().tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-24T18:57:37.349126Z",
     "start_time": "2024-01-24T18:57:37.348996Z"
    }
   },
   "outputs": [],
   "source": [
    "@eval\n",
    "def eval_epoch(model, val_dataloader):\n",
    "    val_loss = 0.0\n",
    "    val_preds = []\n",
    "    cs_val_preds = []\n",
    "    val_labels = []\n",
    "\n",
    "    with torch.no_grad():\n",
    "\n",
    "        for step, batch in enumerate(tqdm(val_dataloader, total=len(val_dataloader))):\n",
    "            ids = batch['input_ids'].to(device, dtype = torch.long)\n",
    "            mask = batch['attention_mask'].to(device, dtype = torch.long)\n",
    "            targets = batch['label'].to(device, dtype = torch.long)\n",
    "\n",
    "            outputs = model(input_ids=ids, attention_mask=mask, labels=targets)\n",
    "\n",
    "            loss, logits = outputs.loss, outputs.logits\n",
    "\n",
    "            probs = F.softmax(logits, dim=-1).cpu()\n",
    "            pred = torch.argmax(probs, dim=-1) # (B)\n",
    "            val_preds += pred.detach().tolist()\n",
    "            val_labels += [l.item() for l in targets.cpu()]\n",
    "\n",
    "            ### Distance Based Classification\n",
    "            # out.concept_spaces (n, B, seq_len, n_latent)\n",
    "            if hasattr(outputs, 'concept_spaces'):\n",
    "                cs_val_preds += concept_space_to_preds(outputs.concept_spaces)\n",
    "            ### END\n",
    "\n",
    "            val_loss += loss.item()\n",
    "    return val_loss, val_preds, val_labels, cs_val_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-24T18:57:38.253756Z",
     "start_time": "2024-01-24T18:57:38.242682Z"
    }
   },
   "outputs": [],
   "source": [
    "val_dataloader = torch.utils.data.DataLoader(val_dataset['test'], batch_size=2 * BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-24T18:57:57.854353Z",
     "start_time": "2024-01-24T18:57:38.878197Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3/3 [00:18<00:00,  6.30s/it]\n"
     ]
    }
   ],
   "source": [
    "val_loss, val_preds, val_labels, _ = eval_epoch(base_model, val_dataloader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-24T18:58:08.052065Z",
     "start_time": "2024-01-24T18:58:08.040907Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "val_acc = accuracy_score(val_labels, val_preds)\n",
    "val_f1 = f1_score(val_labels, val_preds, average='macro')\n",
    "val_precision = precision_score(val_labels, val_preds, average='macro')\n",
    "val_recall = recall_score(val_labels, val_preds, average='macro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-24T18:58:08.581883Z",
     "start_time": "2024-01-24T18:58:08.537104Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val loss: 0.5989654759565989\n",
      "Val acc: 0.5909090909090909\n",
      "Val f1: 0.37142857142857144\n",
      "Val precision: 0.29545454545454547\n",
      "Val recall: 0.5\n"
     ]
    }
   ],
   "source": [
    "print(f'Val loss: {val_loss / len(val_dataloader)}')\n",
    "print(f'Val acc: {val_acc}')\n",
    "print(f'Val f1: {val_f1}')\n",
    "print(f'Val precision: {val_precision}')\n",
    "print(f'Val recall: {val_recall}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Space Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-24T19:02:14.820899Z",
     "start_time": "2024-01-24T19:02:14.695073Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertModel(\n",
       "  (embeddings): BertEmbeddings(\n",
       "    (word_embeddings): Embedding(28996, 768, padding_idx=0)\n",
       "    (position_embeddings): Embedding(512, 768)\n",
       "    (token_type_embeddings): Embedding(2, 768)\n",
       "    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "  )\n",
       "  (encoder): BertEncoder(\n",
       "    (layer): ModuleList(\n",
       "      (0-11): 12 x BertLayer(\n",
       "        (attention): BertAttention(\n",
       "          (self): BertSelfAttention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (output): BertSelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (intermediate): BertIntermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          (intermediate_act_fn): GELUActivation()\n",
       "        )\n",
       "        (output): BertOutput(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (pooler): BertPooler(\n",
       "    (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "    (activation): Tanh()\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base_model = AutoModel.from_pretrained(MODEL_NAME)\n",
    "base_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-24T19:02:16.971640Z",
     "start_time": "2024-01-24T19:02:15.363091Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SpaceModelForSequenceClassification(\n",
       "  (base_model): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(28996, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0-11): 12 x BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (space_model): SpaceModel(\n",
       "    (concept_spaces): ModuleList(\n",
       "      (0-1): 2 x Sequential(\n",
       "        (0): Linear(in_features=768, out_features=128, bias=False)\n",
       "        (1): Tanh()\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (classifier): Linear(in_features=256, out_features=2, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "space_model = SpaceModelForSequenceClassification(base_model, n_embed=768, n_latent=N_LATENT, n_concept_spaces=2, l1=0, l2=0).to(device)\n",
    "space_model.load_state_dict(torch.load(f'{MODELS_PATH}/{DATASET_NAME}_space-{MODEL_NAME}-({N_LATENT})_{NUM_EPOCHS}.bin'))\n",
    "space_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-24T19:02:35.561108Z",
     "start_time": "2024-01-24T19:02:16.965607Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3/3 [00:18<00:00,  6.18s/it]\n"
     ]
    }
   ],
   "source": [
    "val_loss, val_preds, val_labels, cs_val_preds = eval_epoch(space_model, val_dataloader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-24T19:02:37.125463Z",
     "start_time": "2024-01-24T19:02:37.112174Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "cs_val_acc = accuracy_score(val_labels, cs_val_preds)\n",
    "cs_val_f1 = f1_score(val_labels, cs_val_preds, average='macro')\n",
    "\n",
    "val_acc = accuracy_score(val_labels, val_preds)\n",
    "val_f1 = f1_score(val_labels, val_preds, average='macro')\n",
    "val_precision = precision_score(val_labels, val_preds, average='macro')\n",
    "val_recall = recall_score(val_labels, val_preds, average='macro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-24T19:02:37.516323Z",
     "start_time": "2024-01-24T19:02:37.472244Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val loss: 0.531201163927714\n",
      "Val acc: 0.6941287878787878\n",
      "CS Val acc: 0.646780303030303\n",
      "Val f1: 0.6395237868771897\n",
      "CS Val f1: 0.6155446129219597\n",
      "Val precision: 0.7171988037358195\n",
      "Val recall: 0.6446759259259259\n"
     ]
    }
   ],
   "source": [
    "print(f'Val loss: {val_loss / len(val_dataloader)}')\n",
    "print(f'Val acc: {val_acc}')\n",
    "print(f'CS Val acc: {cs_val_acc}')\n",
    "print(f'Val f1: {val_f1}')\n",
    "print(f'CS Val f1: {cs_val_f1}')\n",
    "print(f'Val precision: {val_precision}')\n",
    "print(f'Val recall: {val_recall}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "T4",
   "provenance": []
  },
  "environment": {
   "kernel": "python3",
   "name": ".m114",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/:m114"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}