demo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "source": [
    "! rm -rf nano-BERT\n",
    "! git clone https://github.com/StepanTita/nano-BERT.git"
   ],
   "metadata": {
    "id": "V4vJIZJeclwP",
    "outputId": "b20006dd-8811-4d0c-b81e-faad120f1a29",
    "colab": {
     "base_uri": "https://localhost:8080/"
    }
   },
   "execution_count": 23,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Cloning into 'nano-BERT'...\n",
      "remote: Enumerating objects: 13, done.\u001B[K\n",
      "remote: Counting objects: 100% (13/13), done.\u001B[K\n",
      "remote: Compressing objects: 100% (9/9), done.\u001B[K\n",
      "remote: Total 13 (delta 2), reused 13 (delta 2), pack-reused 0\u001B[K\n",
      "Receiving objects: 100% (13/13), 2.08 MiB | 4.27 MiB/s, done.\n",
      "Resolving deltas: 100% (2/2), done.\n"
     ]
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "import sys\n",
    "\n",
    "sys.path.append('/content/nano-BERT')"
   ],
   "metadata": {
    "id": "LFX6x0OIc6uP"
   },
   "execution_count": 1,
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "start_time": "2023-10-17T21:21:52.683897Z",
     "end_time": "2023-10-17T21:21:56.394657Z"
    },
    "id": "EzQpePJ4cXPG"
   },
   "outputs": [],
   "source": [
    "import json\n",
    "from collections import Counter\n",
    "\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "from sklearn.metrics import accuracy_score, f1_score\n",
    "from tqdm import tqdm\n",
    "\n",
    "from nano_bert.model import NanoBertForClassification\n",
    "from nano_bert.tokenizer import WordTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
   ],
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-10-17T21:21:57.485837Z",
     "end_time": "2023-10-17T21:21:57.492835Z"
    },
    "id": "dPxAI9XscXPH"
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "data = None\n",
    "with open('nano-BERT/data/dataset.json') as f:\n",
    "    data = json.loads(f.read())"
   ],
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-10-17T21:21:58.202759Z",
     "end_time": "2023-10-17T21:21:58.663667Z"
    },
    "id": "KQb635bAcXPH"
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [],
   "source": [
    "vocab = set()\n",
    "for id in data:\n",
    "    vocab |= set(data[id]['post_tokens'])"
   ],
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-10-17T21:21:59.290429Z",
     "end_time": "2023-10-17T21:21:59.380681Z"
    },
    "id": "MnzxttcxcXPI"
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [],
   "source": [
    "ids_split = None\n",
    "with open('nano-BERT/data/post_id_divisions.json') as f:\n",
    "    ids_split = json.loads(f.read())"
   ],
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-10-17T21:21:59.785847Z",
     "end_time": "2023-10-17T21:21:59.795588Z"
    },
    "id": "m5CMy4NmcXPI"
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [],
   "source": [
    "def encode_label(label):\n",
    "    if label == 'hatespeech':\n",
    "        return 0\n",
    "    elif label == 'normal':\n",
    "        return 1\n",
    "    elif label == 'offensive':\n",
    "        return 0  # 2\n",
    "    raise Exception(f'Unknown Label: {label}!')\n",
    "\n",
    "\n",
    "class HateDataloader:\n",
    "    def __init__(self, data, ids_split, tokenizer, label_encoder, batch_size):\n",
    "        def get_label(annotators):\n",
    "            return Counter([anno['label'] for anno in annotators]).most_common(1)[0][0]\n",
    "\n",
    "        self.splits = {\n",
    "            'train': [data[post_id]['post_tokens'] for post_id in tqdm(data, desc='Train') if\n",
    "                      post_id in ids_split['train']],\n",
    "            'test': [data[post_id]['post_tokens'] for post_id in tqdm(data, desc='Test') if\n",
    "                     post_id in ids_split['test']],\n",
    "            'val': [data[post_id]['post_tokens'] for post_id in tqdm(data, desc='Val') if post_id in ids_split['val']]\n",
    "        }\n",
    "\n",
    "        self.labels = {\n",
    "            'train': [get_label(data[post_id]['annotators']) for post_id in tqdm(data, desc='Train Labels') if\n",
    "                      post_id in ids_split['train']],\n",
    "            'test': [get_label(data[post_id]['annotators']) for post_id in tqdm(data, desc='Test Labels') if\n",
    "                     post_id in ids_split['test']],\n",
    "            'val': [get_label(data[post_id]['annotators']) for post_id in tqdm(data, desc='Val Labels') if\n",
    "                    post_id in ids_split['val']]\n",
    "        }\n",
    "\n",
    "        self.tokenized = {\n",
    "            'train': [tokenizer(record).unsqueeze(0) for record in\n",
    "                      tqdm(self.splits['train'], desc='Train Tokenization')],\n",
    "            'test': [tokenizer(record).unsqueeze(0) for record in tqdm(self.splits['test'], desc='Test Tokenization')],\n",
    "            'val': [tokenizer(record).unsqueeze(0) for record in tqdm(self.splits['val'], desc='Val Tokenization')],\n",
    "        }\n",
    "\n",
    "        self.encoded_labels = {\n",
    "            'train': [label_encoder(label) for label in tqdm(self.labels['train'], desc='Train Label Encoding')],\n",
    "            'test': [label_encoder(label) for label in tqdm(self.labels['test'], desc='Test Label Encoding')],\n",
    "            'val': [label_encoder(label) for label in tqdm(self.labels['val'], desc='Val Label Encoding')],\n",
    "        }\n",
    "\n",
    "        self.curr_batch = 0\n",
    "        self.batch_size = batch_size\n",
    "        self.iterate_split = None\n",
    "\n",
    "    def peek(self, split):\n",
    "        return {\n",
    "            'input_ids': self.splits[split][self.batch_size * self.curr_batch:self.batch_size * (self.curr_batch + 1)],\n",
    "            'label_ids': self.labels[split][self.batch_size * self.curr_batch:self.batch_size * (self.curr_batch + 1)],\n",
    "        }\n",
    "\n",
    "    def take(self, split):\n",
    "        batch = self.splits[split][self.batch_size * self.curr_batch:self.batch_size * (self.curr_batch + 1)]\n",
    "        labels = self.labels[split][self.batch_size * self.curr_batch:self.batch_size * (self.curr_batch + 1)]\n",
    "        self.curr_batch += 1\n",
    "        return {\n",
    "            'input_ids': batch,\n",
    "            'label_ids': labels,\n",
    "        }\n",
    "\n",
    "    def peek_tokenized(self, split):\n",
    "        return {\n",
    "            'input_ids': torch.cat(\n",
    "                self.tokenized[split][self.batch_size * self.curr_batch:self.batch_size * (self.curr_batch + 1)],\n",
    "                dim=0),\n",
    "            'label_ids': torch.tensor(\n",
    "                self.encoded_labels[split][self.batch_size * self.curr_batch:self.batch_size * (self.curr_batch + 1)],\n",
    "                dtype=torch.long),\n",
    "        }\n",
    "\n",
    "    def take_tokenized(self, split):\n",
    "        batch = self.tokenized[split][self.batch_size * self.curr_batch:self.batch_size * (self.curr_batch + 1)]\n",
    "        labels = self.encoded_labels[split][self.batch_size * self.curr_batch:self.batch_size * (self.curr_batch + 1)]\n",
    "        self.curr_batch += 1\n",
    "        return {\n",
    "            'input_ids': torch.cat(batch, dim=0),\n",
    "            'label_ids': torch.tensor(labels, dtype=torch.long),\n",
    "        }\n",
    "\n",
    "    def get_split(self, split):\n",
    "        self.iterate_split = split\n",
    "        return self\n",
    "\n",
    "    def steps(self, split):\n",
    "        return len(self.tokenized[split]) // self.batch_size\n",
    "\n",
    "    def __iter__(self):\n",
    "        self.reset()\n",
    "        return self\n",
    "\n",
    "    def __next__(self):\n",
    "        if self.batch_size * self.curr_batch < len(self.splits[self.iterate_split]):\n",
    "            return self.take_tokenized(self.iterate_split)\n",
    "        else:\n",
    "            raise StopIteration\n",
    "\n",
    "    def reset(self):\n",
    "        self.curr_batch = 0"
   ],
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-10-17T21:22:01.646787Z",
     "end_time": "2023-10-17T21:22:01.652524Z"
    },
    "id": "V1EiqscxcXPI"
   }
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "outputs": [],
   "source": [
    "NUM_EPOCHS = 100\n",
    "BATCH_SIZE = 32\n",
    "MAX_SEQ_LEN = 128\n",
    "LEARNING_RATE = 1e-2"
   ],
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-10-17T21:22:18.368560Z",
     "end_time": "2023-10-17T21:22:18.371680Z"
    },
    "id": "5b6HjLMEcXPJ"
   }
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "Tokenizer[vocab=28842,self.special_tokens=['[PAD]', '[CLS]', '[SEP]', '[UNK]'],self.sep=' ',self.max_seq_len=128]"
      ]
     },
     "metadata": {},
     "execution_count": 34
    }
   ],
   "source": [
    "tokenizer = WordTokenizer(vocab=vocab, max_seq_len=MAX_SEQ_LEN)\n",
    "tokenizer"
   ],
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-10-17T21:26:23.403439Z",
     "end_time": "2023-10-17T21:26:23.407128Z"
    },
    "id": "7CdJl4UEcXPJ",
    "outputId": "ee737554-14a9-4e2c-86c9-35c190b1ec64",
    "colab": {
     "base_uri": "https://localhost:8080/"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Train: 100%|██████████| 20148/20148 [00:04<00:00, 4483.92it/s]\n",
      "Test: 100%|██████████| 20148/20148 [00:00<00:00, 32883.00it/s]\n",
      "Val: 100%|██████████| 20148/20148 [00:00<00:00, 25917.49it/s]\n",
      "Train Labels: 100%|██████████| 20148/20148 [00:03<00:00, 5449.81it/s]\n",
      "Test Labels: 100%|██████████| 20148/20148 [00:00<00:00, 31697.45it/s]\n",
      "Val Labels: 100%|██████████| 20148/20148 [00:00<00:00, 31642.22it/s]\n",
      "Train Tokenization: 100%|██████████| 15383/15383 [00:00<00:00, 24469.71it/s]\n",
      "Test Tokenization: 100%|██████████| 1924/1924 [00:00<00:00, 25211.32it/s]\n",
      "Val Tokenization: 100%|██████████| 1922/1922 [00:00<00:00, 26442.78it/s]\n",
      "Train Label Encoding: 100%|██████████| 15383/15383 [00:00<00:00, 1500522.77it/s]\n",
      "Test Label Encoding: 100%|██████████| 1924/1924 [00:00<00:00, 970866.33it/s]\n",
      "Val Label Encoding: 100%|██████████| 1922/1922 [00:00<00:00, 1065624.89it/s]\n"
     ]
    }
   ],
   "source": [
    "dataloader = HateDataloader(data, ids_split, tokenizer, encode_label, batch_size=BATCH_SIZE)"
   ],
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-10-17T21:26:24.941273Z",
     "end_time": "2023-10-17T21:26:37.278112Z"
    },
    "id": "iQMDcK01cXPK",
    "outputId": "80c6627f-5a72-4b25-8e2b-fbfa7babbee9",
    "colab": {
     "base_uri": "https://localhost:8080/"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "NanoBertForClassification(\n",
       "  (nano_bert): NanoBERT(\n",
       "    (embedding): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(28842, 3)\n",
       "      (pos_embeddings): Embedding(128, 3)\n",
       "      (layer_norm): LayerNorm((3,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layers): ModuleList(\n",
       "        (0): BertLayer(\n",
       "          (self_attention): BertSelfAttention(\n",
       "            (heads): ModuleList(\n",
       "              (0): BertAttentionHead(\n",
       "                (query): Linear(in_features=3, out_features=3, bias=True)\n",
       "                (key): Linear(in_features=3, out_features=3, bias=True)\n",
       "                (values): Linear(in_features=3, out_features=3, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (proj): Linear(in_features=3, out_features=3, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (layer_norm1): LayerNorm((3,), eps=1e-05, elementwise_affine=True)\n",
       "          (layer_norm2): LayerNorm((3,), eps=1e-05, elementwise_affine=True)\n",
       "          (feed_forward): FeedForward(\n",
       "            (ffwd): Sequential(\n",
       "              (0): Linear(in_features=3, out_features=12, bias=True)\n",
       "              (1): GELU(approximate='none')\n",
       "              (2): Linear(in_features=12, out_features=3, bias=True)\n",
       "              (3): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=3, out_features=3, bias=True)\n",
       "      (activation): GELU(approximate='none')\n",
       "    )\n",
       "  )\n",
       "  (classifier): Linear(in_features=3, out_features=2, bias=True)\n",
       ")"
      ]
     },
     "metadata": {},
     "execution_count": 36
    }
   ],
   "source": [
    "bert = NanoBertForClassification(\n",
    "    vocab_size=len(tokenizer.vocab),\n",
    "    n_layers=1,\n",
    "    n_heads=1,\n",
    "    max_seq_len=MAX_SEQ_LEN,\n",
    "    n_classes=2\n",
    ").to(device)\n",
    "bert"
   ],
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-10-17T21:26:39.490237Z",
     "end_time": "2023-10-17T21:26:39.504430Z"
    },
    "id": "7LtAcsA8cXPK",
    "outputId": "363eca6c-f4a9-42fa-d817-05d9d2362878",
    "colab": {
     "base_uri": "https://localhost:8080/"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "outputs": [],
   "source": [
    "def count_parameters(model):\n",
    "    return sum(p.numel() for p in model.parameters() if p.requires_grad)"
   ],
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-10-17T21:26:40.211284Z",
     "end_time": "2023-10-17T21:26:40.220320Z"
    },
    "id": "H64pDk8AcXPK"
   }
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "87083"
      ]
     },
     "metadata": {},
     "execution_count": 43
    }
   ],
   "source": [
    "count_parameters(bert)"
   ],
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-10-17T21:26:40.956953Z",
     "end_time": "2023-10-17T21:26:40.969226Z"
    },
    "id": "i-Bhh81ScXPK",
    "outputId": "217d91d4-40b3-4211-e8d7-e21befe0699d",
    "colab": {
     "base_uri": "https://localhost:8080/"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 187.55it/s]                         \n",
      "61it [00:00, 235.13it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.6532830839976669 | Val loss: 0.6448634972174963\n",
      "Train acc: 0.5944224143535071 | Val acc: 0.6233090530697191\n",
      "Train f1: 0.34758966851406464 | Val f1: 0.4084967320261438\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 212.14it/s]                         \n",
      "61it [00:00, 168.70it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.5837939226999879 | Val loss: 0.5605499883492787\n",
      "Train acc: 0.6954430215172593 | Val acc: 0.7148803329864725\n",
      "Train f1: 0.6167689161554193 | Val f1: 0.6509554140127388\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 179.94it/s]                         \n",
      "61it [00:00, 229.79it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.5106981079404552 | Val loss: 0.47607098892331123\n",
      "Train acc: 0.7576545537281415 | Val acc: 0.7804370447450573\n",
      "Train f1: 0.6999839047159182 | Val f1: 0.7365792759051186\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 218.56it/s]                         \n",
      "61it [00:00, 227.20it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.4543498909411331 | Val loss: 0.38817669364313284\n",
      "Train acc: 0.7978287720210622 | Val acc: 0.8413111342351717\n",
      "Train f1: 0.7527821939586645 | Val f1: 0.8106765983860955\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 169.05it/s]                         \n",
      "61it [00:00, 220.73it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.41864503944913545 | Val loss: 0.3444541578491529\n",
      "Train acc: 0.8187609699018397 | Val acc: 0.8652445369406868\n",
      "Train f1: 0.7783431388138018 | Val f1: 0.8386292834890966\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 200.33it/s]                         \n",
      "61it [00:00, 207.72it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.3831731419855108 | Val loss: 0.3035725925117731\n",
      "Train acc: 0.842553468114152 | Val acc: 0.8928199791883454\n",
      "Train f1: 0.8082039911308204 | Val f1: 0.8710888610763455\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 215.02it/s]                         \n",
      "61it [00:00, 222.72it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.3573876689032962 | Val loss: 0.2727705121040344\n",
      "Train acc: 0.8547097445231749 | Val acc: 0.9027055150884495\n",
      "Train f1: 0.8230263678834429 | Val f1: 0.8836341008089607\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 212.68it/s]                         \n",
      "61it [00:00, 223.66it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.3305578186331938 | Val loss: 0.2515181009968122\n",
      "Train acc: 0.8660859390235974 | Val acc: 0.9141519250780437\n",
      "Train f1: 0.8388862818707962 | Val f1: 0.896421845574388\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 193.49it/s]                         \n",
      "61it [00:00, 171.30it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.3239343744935468 | Val loss: 0.2507672702272733\n",
      "Train acc: 0.8694662939608659 | Val acc: 0.9073881373569199\n",
      "Train f1: 0.8426578906127566 | Val f1: 0.8876262626262627\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 197.25it/s]                         \n",
      "61it [00:00, 225.58it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.3071245421345035 | Val loss: 0.19970860959341127\n",
      "Train acc: 0.8808424884612884 | Val acc: 0.9386056191467221\n",
      "Train f1: 0.8550644421601961 | Val f1: 0.9256926952141059\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 216.94it/s]                         \n",
      "61it [00:00, 228.40it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.2898086346608276 | Val loss: 0.1964961914345622\n",
      "Train acc: 0.889618409933043 | Val acc: 0.9360041623309053\n",
      "Train f1: 0.8664884415788646 | Val f1: 0.9225928256765261\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 223.15it/s]                         \n",
      "61it [00:00, 232.91it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.28608230777705707 | Val loss: 0.1652445367537439\n",
      "Train acc: 0.8897484235844764 | Val acc: 0.951092611862643\n",
      "Train f1: 0.8661404893449092 | Val f1: 0.9407313997477932\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 214.80it/s]                         \n",
      "61it [00:00, 217.49it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.27832723564933987 | Val loss: 0.16319900842693943\n",
      "Train acc: 0.8926737307417278 | Val acc: 0.9505723204994797\n",
      "Train f1: 0.8695995576968644 | Val f1: 0.9400630914826499\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 173.12it/s]                         \n",
      "61it [00:00, 234.14it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.27137257354334 | Val loss: 0.14836932715649406\n",
      "Train acc: 0.8985243450562309 | Val acc: 0.9568158168574402\n",
      "Train f1: 0.876669036896579 | Val f1: 0.9476340694006311\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 219.46it/s]                         \n",
      "61it [00:00, 225.97it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.266493749715543 | Val loss: 0.14406100227497518\n",
      "Train acc: 0.8983943314047975 | Val acc: 0.9562955254942768\n",
      "Train f1: 0.8769000551311333 | Val f1: 0.9469696969696969\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 197.12it/s]                         \n",
      "61it [00:00, 236.23it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.26402977340233824 | Val loss: 0.14186121540454527\n",
      "Train acc: 0.9015146590391991 | Val acc: 0.9562955254942768\n",
      "Train f1: 0.8800949742777998 | Val f1: 0.9468354430379745\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 214.69it/s]                         \n",
      "61it [00:00, 226.49it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.2560979838793476 | Val loss: 0.15501793526733915\n",
      "Train acc: 0.904374959370734 | Val acc: 0.9573361082206036\n",
      "Train f1: 0.8842005825395577 | Val f1: 0.9480354879594424\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 197.90it/s]                         \n",
      "61it [00:00, 172.15it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.25406430249956125 | Val loss: 0.14105653315782546\n",
      "Train acc: 0.9049600208021842 | Val acc: 0.959417273673257\n",
      "Train f1: 0.8848818897637795 | Val f1: 0.9510664993726475\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 194.00it/s]                         \n",
      "61it [00:00, 231.71it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.24920952453588446 | Val loss: 0.1400976948129634\n",
      "Train acc: 0.9056100890593513 | Val acc: 0.9588969823100937\n",
      "Train f1: 0.8852173913043478 | Val f1: 0.9500316255534471\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 221.91it/s]                         \n",
      "61it [00:00, 238.82it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.2405256699770689 | Val loss: 0.1282794436439872\n",
      "Train acc: 0.9096405122537866 | Val acc: 0.9640998959417274\n",
      "Train f1: 0.8905511811023623 | Val f1: 0.9563014566181127\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 217.32it/s]                         \n",
      "61it [00:00, 233.49it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.2438654977440213 | Val loss: 0.12652083911622564\n",
      "Train acc: 0.9099005395566534 | Val acc: 0.9646201873048907\n",
      "Train f1: 0.890521327014218 | Val f1: 0.9567430025445292\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 212.69it/s]                         \n",
      "61it [00:00, 230.43it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.22859723906925258 | Val loss: 0.12601286626886576\n",
      "Train acc: 0.9152310992654229 | Val acc: 0.9672216441207075\n",
      "Train f1: 0.8968191169488843 | Val f1: 0.9599491417673237\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 177.26it/s]                         \n",
      "61it [00:00, 173.83it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.23176546569059914 | Val loss: 0.1253157380502671\n",
      "Train acc: 0.9136059286225053 | Val acc: 0.9651404786680541\n",
      "Train f1: 0.895411977650114 | Val f1: 0.9574603174603175\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 217.41it/s]                         \n",
      "61it [00:00, 227.51it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.2234981488979732 | Val loss: 0.12879162789322435\n",
      "Train acc: 0.9184164337255412 | Val acc: 0.9646201873048907\n",
      "Train f1: 0.9010954369926708 | Val f1: 0.957016434892541\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 222.04it/s]                         \n",
      "61it [00:00, 228.35it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.2292554115372089 | Val loss: 0.11389813033553461\n",
      "Train acc: 0.9144510173568224 | Val acc: 0.9698231009365245\n",
      "Train f1: 0.8961161982949163 | Val f1: 0.9632911392405064\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 223.09it/s]                         \n",
      "61it [00:00, 227.74it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.22546718122127155 | Val loss: 0.10912590237955253\n",
      "Train acc: 0.9165312357797569 | Val acc: 0.9729448491155047\n",
      "Train f1: 0.8984819734345352 | Val f1: 0.9670886075949368\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 222.29it/s]                         \n",
      "61it [00:00, 203.11it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.22503940491393828 | Val loss: 0.11513398839936902\n",
      "Train acc: 0.9182214132483911 | Val acc: 0.9698231009365245\n",
      "Train f1: 0.9008980620765716 | Val f1: 0.9632911392405064\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 177.44it/s]                         \n",
      "61it [00:00, 225.33it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.22703385815645258 | Val loss: 0.10909955167832473\n",
      "Train acc: 0.9162062016511734 | Val acc: 0.9719042663891779\n",
      "Train f1: 0.8977876457061296 | Val f1: 0.9657794676806083\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 223.07it/s]                         \n",
      "61it [00:00, 218.55it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.22034678693938378 | Val loss: 0.11500145920241872\n",
      "Train acc: 0.9210167067542092 | Val acc: 0.9739854318418314\n",
      "Train f1: 0.9032258064516129 | Val f1: 0.9683544303797468\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 209.42it/s]                         \n",
      "61it [00:00, 230.66it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.22325932374612117 | Val loss: 0.11298281924488644\n",
      "Train acc: 0.9197165702398752 | Val acc: 0.9713839750260146\n",
      "Train f1: 0.9018984828024466 | Val f1: 0.9649904519414386\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 213.00it/s]                         \n",
      "61it [00:00, 231.90it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.21046273469304044 | Val loss: 0.13183375828278562\n",
      "Train acc: 0.9242670480400442 | Val acc: 0.9625390218522373\n",
      "Train f1: 0.907444188448399 | Val f1: 0.9541984732824428\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 192.32it/s]                         \n",
      "61it [00:00, 173.88it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.21214651759558667 | Val loss: 0.11824854308894525\n",
      "Train acc: 0.921276734057076 | Val acc: 0.9687825182101977\n",
      "Train f1: 0.9043368354530373 | Val f1: 0.9621689785624211\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 198.13it/s]                         \n",
      "61it [00:00, 232.12it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.2164294785160261 | Val loss: 0.11397734109001854\n",
      "Train acc: 0.9238120002600273 | Val acc: 0.967741935483871\n",
      "Train f1: 0.9073371283997469 | Val f1: 0.9604086845466157\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 221.99it/s]\n",
      "61it [00:00, 225.23it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.21000818730487178 | Val loss: 0.11838347064331174\n",
      "Train acc: 0.9255671845543783 | Val acc: 0.9667013527575442\n",
      "Train f1: 0.9089028562335906 | Val f1: 0.9593908629441623\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 222.73it/s]                         \n",
      "61it [00:00, 229.50it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.20183917635198062 | Val loss: 0.11050760140642524\n",
      "Train acc: 0.9268023142429955 | Val acc: 0.9745057232049948\n",
      "Train f1: 0.9109318145862997 | Val f1: 0.969046114971573\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 203.40it/s]                         \n",
      "61it [00:00, 226.08it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.20506785865873098 | Val loss: 0.1176809636875987\n",
      "Train acc: 0.9267373074172788 | Val acc: 0.9703433922996878\n",
      "Train f1: 0.9106193988420969 | Val f1: 0.9639468690702088\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 169.86it/s]                         \n",
      "61it [00:00, 228.57it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.20278751777174572 | Val loss: 0.11835956939806541\n",
      "Train acc: 0.9253721640772281 | Val acc: 0.967741935483871\n",
      "Train f1: 0.9089322544819928 | Val f1: 0.9607594936708861\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 219.69it/s]\n",
      "61it [00:00, 231.51it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.20745944862331575 | Val loss: 0.10515600297755251\n",
      "Train acc: 0.9256971982058116 | Val acc: 0.9708636836628513\n",
      "Train f1: 0.9088734752451566 | Val f1: 0.9644670050761421\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 216.73it/s]                         \n",
      "61it [00:00, 220.62it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.20102795362084483 | Val loss: 0.10744992992840707\n",
      "Train acc: 0.9275173893258792 | Val acc: 0.9687825182101977\n",
      "Train f1: 0.9113461079748747 | Val f1: 0.9618805590851335\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 224.94it/s]                         \n",
      "61it [00:00, 233.62it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.21907877982982124 | Val loss: 0.10791657139391948\n",
      "Train acc: 0.9217967886628096 | Val acc: 0.9713839750260146\n",
      "Train f1: 0.9044403844626262 | Val f1: 0.9652118912080961\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 213.66it/s]                         \n",
      "61it [00:00, 158.06it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.20255782891763374 | Val loss: 0.11097638049007703\n",
      "Train acc: 0.9259572255086784 | Val acc: 0.9698231009365245\n",
      "Train f1: 0.9100102709962866 | Val f1: 0.9632911392405064\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 182.20it/s]                         \n",
      "61it [00:00, 227.81it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.19720048557113234 | Val loss: 0.10115086368750781\n",
      "Train acc: 0.9305727101345641 | Val acc: 0.9734651404786681\n",
      "Train f1: 0.9153724247226624 | Val f1: 0.9676190476190476\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 217.97it/s]                         \n",
      "61it [00:00, 237.79it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.20049065247488518 | Val loss: 0.10409921382864316\n",
      "Train acc: 0.9260222323343951 | Val acc: 0.9739854318418314\n",
      "Train f1: 0.9099541066624465 | Val f1: 0.9682337992376112\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 218.96it/s]                         \n",
      "61it [00:00, 225.81it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.20044611947766194 | Val loss: 0.10741392102402945\n",
      "Train acc: 0.9275823961515959 | Val acc: 0.9698231009365245\n",
      "Train f1: 0.9116013331217268 | Val f1: 0.9630573248407642\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 221.08it/s]                         \n",
      "61it [00:00, 236.28it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.1997323496073174 | Val loss: 0.09863884324052681\n",
      "Train acc: 0.9286875121887799 | Val acc: 0.9760665972944849\n",
      "Train f1: 0.9128465877492652 | Val f1: 0.970959595959596\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 190.96it/s]                         \n",
      "61it [00:00, 177.52it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.19346533531594712 | Val loss: 0.12092838982741037\n",
      "Train acc: 0.9307027237859975 | Val acc: 0.9656607700312175\n",
      "Train f1: 0.9153565189772909 | Val f1: 0.9582278481012658\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 202.70it/s]                         \n",
      "61it [00:00, 226.54it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.19589960058025704 | Val loss: 0.10378381469442198\n",
      "Train acc: 0.9301826691802639 | Val acc: 0.9734651404786681\n",
      "Train f1: 0.9149239543726236 | Val f1: 0.967741935483871\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 222.36it/s]                         \n",
      "61it [00:00, 233.44it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.18671276018333932 | Val loss: 0.09530866582257053\n",
      "Train acc: 0.9338230514203991 | Val acc: 0.9771071800208116\n",
      "Train f1: 0.9190264078905505 | Val f1: 0.9722222222222222\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 222.34it/s]                         \n",
      "61it [00:00, 229.92it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.2008969905747411 | Val loss: 0.10323020362605652\n",
      "Train acc: 0.9274523825001625 | Val acc: 0.9729448491155047\n",
      "Train f1: 0.9116529449018367 | Val f1: 0.9671717171717171\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 218.62it/s]                         \n",
      "61it [00:00, 231.89it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.2008704162959475 | Val loss: 0.109211961288626\n",
      "Train acc: 0.9271923551972957 | Val acc: 0.9703433922996878\n",
      "Train f1: 0.9110122358175752 | Val f1: 0.9638095238095239\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 173.68it/s]                         \n",
      "61it [00:00, 233.21it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.19259907852780694 | Val loss: 0.11022020406089723\n",
      "Train acc: 0.9307027237859975 | Val acc: 0.9682622268470343\n",
      "Train f1: 0.9151814131126671 | Val f1: 0.9611217335882728\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 214.97it/s]                         \n",
      "61it [00:00, 239.15it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.19976645377270566 | Val loss: 0.1056692126672715\n",
      "Train acc: 0.9279074302801794 | Val acc: 0.9719042663891779\n",
      "Train f1: 0.9120050781559945 | Val f1: 0.9657794676806083\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 218.69it/s]\n",
      "61it [00:00, 228.22it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.2009770498300592 | Val loss: 0.10716276944925388\n",
      "Train acc: 0.9294675940973802 | Val acc: 0.9719042663891779\n",
      "Train f1: 0.9139639996828166 | Val f1: 0.9659090909090909\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 223.98it/s]                         \n",
      "61it [00:00, 230.78it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.201461807184387 | Val loss: 0.10990433757348607\n",
      "Train acc: 0.9286225053630631 | Val acc: 0.9703433922996878\n",
      "Train f1: 0.9129538607895988 | Val f1: 0.963855421686747\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 213.69it/s]                         \n",
      "61it [00:00, 167.21it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.18689269472379239 | Val loss: 0.10431429715827108\n",
      "Train acc: 0.9327179353832152 | Val acc: 0.9745057232049948\n",
      "Train f1: 0.9176938369781311 | Val f1: 0.969046114971573\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 163.15it/s]                         \n",
      "61it [00:00, 236.89it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.18932005447374348 | Val loss: 0.09748699673606703\n",
      "Train acc: 0.9323278944289151 | Val acc: 0.9771071800208116\n",
      "Train f1: 0.9172824791418355 | Val f1: 0.9722222222222222\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 219.49it/s]                         \n",
      "61it [00:00, 224.27it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.1844379194973347 | Val loss: 0.10294445535788933\n",
      "Train acc: 0.9325229149060651 | Val acc: 0.9745057232049948\n",
      "Train f1: 0.9176843774781919 | Val f1: 0.9691241335853813\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 216.74it/s]                         \n",
      "61it [00:00, 233.87it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.19039424426834253 | Val loss: 0.10553689242806286\n",
      "Train acc: 0.9316128193460314 | Val acc: 0.9729448491155047\n",
      "Train f1: 0.9164681594410036 | Val f1: 0.9670468948035489\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 219.40it/s]                         \n",
      "61it [00:00, 223.03it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.18900541296461598 | Val loss: 0.1086679886172836\n",
      "Train acc: 0.931482805694598 | Val acc: 0.9724245577523413\n",
      "Train f1: 0.9166666666666667 | Val f1: 0.9663064208518755\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 180.63it/s]                         \n",
      "61it [00:00, 169.08it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.18413754718882652 | Val loss: 0.09917023496236652\n",
      "Train acc: 0.934928167457583 | Val acc: 0.9734651404786681\n",
      "Train f1: 0.9206374375644177 | Val f1: 0.9675778766687857\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 216.73it/s]                         \n",
      "61it [00:00, 238.83it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.18320910048593456 | Val loss: 0.10729085537604988\n",
      "Train acc: 0.9340830787232659 | Val acc: 0.9698231009365245\n",
      "Train f1: 0.9195365814950007 | Val f1: 0.9631979695431472\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 222.82it/s]                         \n",
      "61it [00:00, 235.99it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.1874785387054241 | Val loss: 0.10061505758203566\n",
      "Train acc: 0.9357732561919001 | Val acc: 0.9734651404786681\n",
      "Train f1: 0.9216121866074262 | Val f1: 0.9676601141407736\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 219.62it/s]\n",
      "61it [00:00, 224.25it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.18277203401861092 | Val loss: 0.1035675894934684\n",
      "Train acc: 0.9375284404862511 | Val acc: 0.9708636836628513\n",
      "Train f1: 0.9239414325286901 | Val f1: 0.9644670050761421\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 216.96it/s]                         \n",
      "61it [00:00, 228.05it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.19000893946892272 | Val loss: 0.1044841297979777\n",
      "Train acc: 0.9329129558603654 | Val acc: 0.9708636836628513\n",
      "Train f1: 0.9180301826846704 | Val f1: 0.9646017699115045\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 169.69it/s]                         \n",
      "61it [00:00, 223.50it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.18790583172522018 | Val loss: 0.10493984842517724\n",
      "Train acc: 0.9339530650718325 | Val acc: 0.9724245577523413\n",
      "Train f1: 0.9192753853488003 | Val f1: 0.9664344521849272\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 215.32it/s]                         \n",
      "61it [00:00, 230.14it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.18417121153712893 | Val loss: 0.09992110367553929\n",
      "Train acc: 0.9361632971462004 | Val acc: 0.9713839750260146\n",
      "Train f1: 0.9223715415019763 | Val f1: 0.9651678277390754\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 199.89it/s]                         \n",
      "61it [00:00, 176.33it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.1870551909261849 | Val loss: 0.09726271622348577\n",
      "Train acc: 0.9331079763375154 | Val acc: 0.9760665972944849\n",
      "Train f1: 0.9181319118466067 | Val f1: 0.9708121827411167\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 197.61it/s]                         \n",
      "61it [00:00, 233.20it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.18434540424107884 | Val loss: 0.10108330342142532\n",
      "Train acc: 0.9344081128518494 | Val acc: 0.9745057232049948\n",
      "Train f1: 0.9198506632774645 | Val f1: 0.969046114971573\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 185.35it/s]                         \n",
      "61it [00:00, 172.26it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.1916799707939693 | Val loss: 0.10754438803996891\n",
      "Train acc: 0.9320028603003315 | Val acc: 0.9693028095733611\n",
      "Train f1: 0.9168389251073303 | Val f1: 0.9625871908687381\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 211.82it/s]                         \n",
      "61it [00:00, 226.33it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.1826603361677068 | Val loss: 0.09629707351171722\n",
      "Train acc: 0.9359032698433335 | Val acc: 0.9765868886576483\n",
      "Train f1: 0.9214718063077413 | Val f1: 0.9715729627289955\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 223.03it/s]                         \n",
      "61it [00:00, 238.35it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.1840208721482971 | Val loss: 0.11440038837802907\n",
      "Train acc: 0.9362933107976338 | Val acc: 0.9687825182101977\n",
      "Train f1: 0.921875 | Val f1: 0.9618320610687023\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 218.01it/s]                         \n",
      "61it [00:00, 223.52it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.18121301576805612 | Val loss: 0.10444751603839299\n",
      "Train acc: 0.9351881947604499 | Val acc: 0.9724245577523413\n",
      "Train f1: 0.9207661130096162 | Val f1: 0.9663064208518755\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 219.59it/s]                         \n",
      "61it [00:00, 233.95it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.1835172301022491 | Val loss: 0.10305121555769195\n",
      "Train acc: 0.9372034063576675 | Val acc: 0.9719042663891779\n",
      "Train f1: 0.9233333333333333 | Val f1: 0.9655172413793103\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 168.62it/s]                         \n",
      "61it [00:00, 235.19it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.17795587675645946 | Val loss: 0.11017884278359512\n",
      "Train acc: 0.9379184814405512 | Val acc: 0.9713839750260146\n",
      "Train f1: 0.9237037628824799 | Val f1: 0.9651678277390754\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 222.08it/s]                         \n",
      "61it [00:00, 238.46it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.17699415406095795 | Val loss: 0.09379881221490602\n",
      "Train acc: 0.9384385360462849 | Val acc: 0.9750260145681582\n",
      "Train f1: 0.9245959073174617 | Val f1: 0.9696586599241467\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 224.43it/s]                         \n",
      "61it [00:00, 233.87it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.18672858056767533 | Val loss: 0.09916011776464681\n",
      "Train acc: 0.934928167457583 | Val acc: 0.9739854318418314\n",
      "Train f1: 0.9202453987730062 | Val f1: 0.9683544303797468\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 222.49it/s]                         \n",
      "61it [00:00, 230.97it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.17451553408949014 | Val loss: 0.09887803182161102\n",
      "Train acc: 0.936683351751934 | Val acc: 0.9755463059313215\n",
      "Train f1: 0.9226493011435833 | Val f1: 0.9702343255224826\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 219.71it/s]                         \n",
      "61it [00:00, 174.87it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.17582366987868833 | Val loss: 0.09310911600477993\n",
      "Train acc: 0.9386335565234349 | Val acc: 0.9771071800208116\n",
      "Train f1: 0.9246487867177522 | Val f1: 0.9722222222222222\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 186.47it/s]                         \n",
      "61it [00:00, 237.20it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.17636407836107537 | Val loss: 0.1042994815390557\n",
      "Train acc: 0.9388285770005851 | Val acc: 0.9750260145681582\n",
      "Train f1: 0.9252996745256807 | Val f1: 0.9695044472681068\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 227.39it/s]                         \n",
      "61it [00:00, 235.13it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.1761016653579039 | Val loss: 0.09984565339206407\n",
      "Train acc: 0.9389585906520185 | Val acc: 0.9739854318418314\n",
      "Train f1: 0.9255411941955436 | Val f1: 0.9684343434343434\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 222.29it/s]                         \n",
      "61it [00:00, 236.88it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.1824927913354865 | Val loss: 0.10605312103871256\n",
      "Train acc: 0.9374634336605343 | Val acc: 0.9724245577523413\n",
      "Train f1: 0.9232120051085567 | Val f1: 0.9664344521849272\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 220.66it/s]\n",
      "61it [00:00, 241.88it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.17829691703276088 | Val loss: 0.09703030227683485\n",
      "Train acc: 0.9375284404862511 | Val acc: 0.9771071800208116\n",
      "Train f1: 0.9236149749622447 | Val f1: 0.9722222222222222\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 194.79it/s]                         \n",
      "61it [00:00, 168.03it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.1765913582018887 | Val loss: 0.0978253709928443\n",
      "Train acc: 0.9388935838263017 | Val acc: 0.9750260145681582\n",
      "Train f1: 0.9251949705554671 | Val f1: 0.9695044472681068\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:03, 140.76it/s]                         \n",
      "61it [00:00, 234.96it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.1697953065779681 | Val loss: 0.10594214864540845\n",
      "Train acc: 0.9409738022492362 | Val acc: 0.9719042663891779\n",
      "Train f1: 0.9276954929128843 | Val f1: 0.9657794676806083\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 223.21it/s]                         \n",
      "61it [00:00, 224.04it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.18372289067289482 | Val loss: 0.10944516807018469\n",
      "Train acc: 0.9372684131833843 | Val acc: 0.9719042663891779\n",
      "Train f1: 0.9232238045986155 | Val f1: 0.9656488549618321\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 218.86it/s]\n",
      "61it [00:00, 234.09it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.16891003779213254 | Val loss: 0.09245080875698478\n",
      "Train acc: 0.9399986998634857 | Val acc: 0.9765868886576483\n",
      "Train f1: 0.926635402591209 | Val f1: 0.9715370018975331\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 225.68it/s]                         \n",
      "61it [00:00, 225.97it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.17980269869440235 | Val loss: 0.09170502185200652\n",
      "Train acc: 0.9365533381005006 | Val acc: 0.9781477627471384\n",
      "Train f1: 0.922119374401532 | Val f1: 0.9734513274336283\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 174.80it/s]                         \n",
      "61it [00:00, 226.01it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.18580899625473346 | Val loss: 0.09673639118360976\n",
      "Train acc: 0.9342130923746993 | Val acc: 0.9771071800208116\n",
      "Train f1: 0.9196442750516118 | Val f1: 0.9722222222222222\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 213.30it/s]                         \n",
      "61it [00:00, 237.70it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.18368328613772367 | Val loss: 0.10011541426647455\n",
      "Train acc: 0.9359682766690502 | Val acc: 0.9771071800208116\n",
      "Train f1: 0.9219307283823412 | Val f1: 0.9720812182741115\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 221.95it/s]                         \n",
      "61it [00:00, 229.32it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.17416184533503837 | Val loss: 0.09847547643973181\n",
      "Train acc: 0.9397386725606188 | Val acc: 0.9745057232049948\n",
      "Train f1: 0.9267135741955885 | Val f1: 0.9689677010766309\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 224.65it/s]                         \n",
      "61it [00:00, 233.41it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.17916204271023162 | Val loss: 0.09764288409302632\n",
      "Train acc: 0.9373334200091009 | Val acc: 0.9734651404786681\n",
      "Train f1: 0.9234191293295201 | Val f1: 0.967741935483871\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 217.52it/s]                         \n",
      "61it [00:00, 170.62it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.16830784873842883 | Val loss: 0.0899587502858291\n",
      "Train acc: 0.9410388090749529 | Val acc: 0.977627471383975\n",
      "Train f1: 0.9277348418452712 | Val f1: 0.9726984126984126\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 175.45it/s]                         \n",
      "61it [00:00, 234.04it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.179478679317981 | Val loss: 0.10117465837392955\n",
      "Train acc: 0.9353182084118833 | Val acc: 0.9734651404786681\n",
      "Train f1: 0.920937624155741 | Val f1: 0.9676601141407736\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 219.50it/s]                         \n",
      "61it [00:00, 230.30it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.18631013691192494 | Val loss: 0.09843253786675632\n",
      "Train acc: 0.9334980172918157 | Val acc: 0.9750260145681582\n",
      "Train f1: 0.9185574396942918 | Val f1: 0.9695817490494296\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 220.95it/s]                         \n",
      "61it [00:00, 241.30it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.16472862845500155 | Val loss: 0.09204595017557343\n",
      "Train acc: 0.9417538841578366 | Val acc: 0.9786680541103018\n",
      "Train f1: 0.928388746803069 | Val f1: 0.9740998104864181\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 221.95it/s]                         \n",
      "61it [00:00, 223.56it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.1786410267445414 | Val loss: 0.0884368696094801\n",
      "Train acc: 0.9365533381005006 | Val acc: 0.9791883454734651\n",
      "Train f1: 0.9220322735261223 | Val f1: 0.9746514575411914\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 191.02it/s]                         \n",
      "61it [00:00, 163.64it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.16807617930074534 | Val loss: 0.09002333539538085\n",
      "Train acc: 0.9422739387635701 | Val acc: 0.9760665972944849\n",
      "Train f1: 0.9293443666454488 | Val f1: 0.97084917617237\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 201.62it/s]                         \n",
      "61it [00:00, 229.46it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.17004232505375208 | Val loss: 0.090723361990725\n",
      "Train acc: 0.9410388090749529 | Val acc: 0.9760665972944849\n",
      "Train f1: 0.9279415269722731 | Val f1: 0.97084917617237\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 223.89it/s]                         \n",
      "61it [00:00, 227.23it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.18334991042890275 | Val loss: 0.09674443867988884\n",
      "Train acc: 0.9364233244490672 | Val acc: 0.9745057232049948\n",
      "Train f1: 0.9223069590085795 | Val f1: 0.9689283449587826\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "481it [00:02, 221.68it/s]                         \n",
      "61it [00:00, 222.79it/s]                        \n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Train loss: 0.16519838607443185 | Val loss: 0.09134297098498792\n",
      "Train acc: 0.9446141844893714 | Val acc: 0.9781477627471384\n",
      "Train f1: 0.9323057365326554 | Val f1: 0.9734177215189873\n"
     ]
    }
   ],
   "source": [
    "optimizer = torch.optim.Adam(bert.parameters(), lr=LEARNING_RATE)\n",
    "\n",
    "for i in range(NUM_EPOCHS):\n",
    "    print(f'Epoch: {i + 1}')\n",
    "    train_loss = 0.0\n",
    "    train_preds = []\n",
    "    train_labels = []\n",
    "\n",
    "    bert.train()\n",
    "    for step, batch in enumerate(tqdm(dataloader.get_split('train'), total=dataloader.steps('train'))):\n",
    "        logits = bert(batch['input_ids'].to(device)) # (B, Seq_Len, 2)\n",
    "\n",
    "        probs = F.softmax(logits[:, 0, :], dim=-1).cpu()\n",
    "        pred = torch.argmax(probs, dim=-1) # (B)\n",
    "        train_preds += pred.detach().tolist()\n",
    "        train_labels += [l.item() for l in batch['label_ids']]\n",
    "\n",
    "        loss = F.cross_entropy(logits[:, 0, :].cpu(), batch['label_ids'])\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        train_loss += loss.item()\n",
    "\n",
    "    val_loss = 0.0\n",
    "    val_preds = []\n",
    "    val_labels = []\n",
    "\n",
    "    bert.eval()\n",
    "    for step, batch in enumerate(tqdm(dataloader.get_split('val'), total=dataloader.steps('val'))):\n",
    "        logits = bert(batch['input_ids'].to(device))\n",
    "\n",
    "        probs = F.softmax(logits[:, 0, :], dim=-1).cpu()\n",
    "        pred = torch.argmax(probs, dim=-1) # (B)\n",
    "        val_preds += pred.detach().tolist()\n",
    "        val_labels += [l.item() for l in batch['label_ids']]\n",
    "\n",
    "        loss = F.cross_entropy(logits[:, 0, :].cpu(), batch['label_ids'])\n",
    "\n",
    "        val_loss += loss.item()\n",
    "\n",
    "    print()\n",
    "    print(f'Train loss: {train_loss / dataloader.steps(\"train\")} | Val loss: {val_loss / dataloader.steps(\"val\")}')\n",
    "    print(f'Train acc: {accuracy_score(train_labels, train_preds)} | Val acc: {accuracy_score(val_labels, val_preds)}')\n",
    "    print(f'Train f1: {f1_score(train_labels, train_preds)} | Val f1: {f1_score(val_labels, val_preds)}')"
   ],
   "metadata": {
    "id": "dlLiYGbjcXPL",
    "outputId": "ed4c55f0-e331-4fb4-f457-67261138ab2a",
    "colab": {
     "base_uri": "https://localhost:8080/"
    }
   }
  },
  {
   "cell_type": "code",
   "source": [],
   "metadata": {
    "id": "cgyRO91Ng17v"
   },
   "execution_count": null,
   "outputs": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  },
  "colab": {
   "provenance": [],
   "gpuType": "T4"
  },
  "accelerator": "GPU"
 },
 "nbformat": 4,
 "nbformat_minor": 0
}