QnA_Classifier.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/hishamp3/MasterThesis-Lies-DeceptiveText/blob/main/QnA_Classifier.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KqxJRDBp_iqc",
        "outputId": "74aa8a80-935e-4fc8-e62e-178f8cdabd0b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: transformers==4.20.0 in /usr/local/lib/python3.10/dist-packages (4.20.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers==4.20.0) (3.12.4)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in /usr/local/lib/python3.10/dist-packages (from transformers==4.20.0) (0.18.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.20.0) (1.23.5)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers==4.20.0) (23.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers==4.20.0) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.20.0) (2023.6.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers==4.20.0) (2.31.0)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.13,>=0.11.1 in /usr/local/lib/python3.10/dist-packages (from transformers==4.20.0) (0.12.1)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers==4.20.0) (4.66.1)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers==4.20.0) (2023.6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers==4.20.0) (4.5.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.20.0) (3.3.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.20.0) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.20.0) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.20.0) (2023.7.22)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (2.1.0+cu118)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.10/dist-packages (0.16.0+cu118)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch) (3.12.4)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch) (4.5.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch) (3.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch) (3.1.2)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch) (2023.6.0)\n",
            "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch) (2.1.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from torchvision) (1.23.5)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from torchvision) (2.31.0)\n",
            "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.10/dist-packages (from torchvision) (9.4.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch) (2.1.3)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->torchvision) (3.3.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->torchvision) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->torchvision) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->torchvision) (2023.7.22)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch) (1.3.0)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (1.5.3)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas) (2023.3.post1)\n",
            "Requirement already satisfied: numpy>=1.21.0 in /usr/local/lib/python3.10/dist-packages (from pandas) (1.23.5)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas) (1.16.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (1.23.5)\n"
          ]
        }
      ],
      "source": [
        "# Install Transformers\n",
        "!pip install transformers==4.20.0\n",
        "!pip install torch torchvision\n",
        "!pip install pandas\n",
        "!pip install numpy"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yaTpwePCNGX6",
        "outputId": "a573c1ef-82a5-41c7-f30d-e555f4ee3aa4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.10/dist-packages (0.1.99)\n"
          ]
        }
      ],
      "source": [
        "!pip install sentencepiece"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jvDDj9DhArft"
      },
      "outputs": [],
      "source": [
        "# loading data\n",
        "import pandas as pd\n",
        "train_data_df = pd.read_json(path_or_buf=\"./sample_data/train.jsonl\", lines=True,orient='records')\n",
        "dev_data_df = pd.read_json(path_or_buf=\"./sample_data/dev.jsonl\", lines=True,orient='records')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UGcLQm9yagAT",
        "outputId": "a534249d-4be6-4229-e514-b340770c3efa"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "                                            question  \\\n",
            "0    do iran and afghanistan speak the same language   \n",
            "1  do good samaritan laws protect those who help ...   \n",
            "2  is windows movie maker part of windows essentials   \n",
            "3  is confectionary sugar the same as powdered sugar   \n",
            "4         is elder scrolls online the same as skyrim   \n",
            "\n",
            "                      title  answer  \\\n",
            "0          Persian language    True   \n",
            "1        Good Samaritan law    True   \n",
            "2       Windows Movie Maker    True   \n",
            "3            Powdered sugar    True   \n",
            "4  The Elder Scrolls Online   False   \n",
            "\n",
            "                                             passage  \n",
            "0  Persian (/ˈpɜːrʒən, -ʃən/), also known by its ...  \n",
            "1  Good Samaritan laws offer legal protection to ...  \n",
            "2  Windows Movie Maker (formerly known as Windows...  \n",
            "3  Powdered sugar, also called confectioners' sug...  \n",
            "4  As with other games in The Elder Scrolls serie...  \n"
          ]
        }
      ],
      "source": [
        "print(train_data_df.head(5))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iOjuRYiHBAJF"
      },
      "outputs": [],
      "source": [
        "import random\n",
        "import torch\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from tqdm import tqdm\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "from transformers import AutoTokenizer, AutoModel, AdamW,AutoModelForSequenceClassification"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "q8BD2Rvnjrb6",
        "outputId": "2e8d536e-5651-47ed-f75a-716c249f6bec"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://download.pytorch.org/whl/cpu\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (2.1.0+cu118)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.10/dist-packages (0.16.0+cu118)\n",
            "Requirement already satisfied: torchaudio in /usr/local/lib/python3.10/dist-packages (2.1.0+cu118)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch) (3.12.4)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch) (4.5.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch) (3.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch) (3.1.2)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch) (2023.6.0)\n",
            "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch) (2.1.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from torchvision) (1.23.5)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from torchvision) (2.31.0)\n",
            "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.10/dist-packages (from torchvision) (9.4.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch) (2.1.3)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->torchvision) (3.3.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->torchvision) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->torchvision) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->torchvision) (2023.7.22)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch) (1.3.0)\n"
          ]
        }
      ],
      "source": [
        "!pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hqM8NJi6BCkJ"
      },
      "outputs": [],
      "source": [
        "import gc\n",
        "gc.collect()\n",
        "torch.cuda.empty_cache()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 156,
          "referenced_widgets": [
            "e43d09106ad54b259a8b135c7c033204",
            "0bf84df9a59d4342be2ad7998c9d2a36",
            "020fb28df2fa4488b83879539db2faad",
            "17a565f3da664ac3ba31dff990580a1e",
            "8ee0f7cd15544a2ba5c76e460404a3e6",
            "b439c52fe720408cb27b6189ae5076f6",
            "79ba8ee5dcce48f5a778f1db5cce06a0",
            "816805fd957346c9834f645bf5f70936",
            "a4a96fc58eeb42e5bcabd97e24dffa36",
            "b4d18a931ef543cf97fb06738cf03c96",
            "984b867633aa44a3872747b7e8930e31"
          ]
        },
        "id": "jlW9--EbBHz2",
        "outputId": "f7d5888a-053b-4fa9-9c13-2868c84b666e"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/478M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e43d09106ad54b259a8b135c7c033204"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.weight', 'lm_head.layer_norm.weight', 'lm_head.dense.weight', 'lm_head.dense.bias', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.bias', 'lm_head.decoder.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ],
      "source": [
        "# use GPU if available\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "#setting seeds\n",
        "random.seed(26)\n",
        "np.random.seed(26)\n",
        "torch.manual_seed(26)\n",
        "model_name = 'roberta-base'\n",
        "#model_name = 'bert-base-uncased'\n",
        "#model_name = 'bert-base-cased'\n",
        "#model_name = 'bert-large-uncased'\n",
        "#model_name = 'roberta-large'\n",
        "\n",
        "# xlm-roberta-base\n",
        "'''from transformers import AutoTokenizer, XLMRobertaForSequenceClassification\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"cardiffnlp/twitter-roberta-base-emotion\")\n",
        "model = XLMRobertaForSequenceClassification.from_pretrained(\"cardiffnlp/twitter-roberta-base-emotion\")'''\n",
        "\n",
        "# xlm-roberta-large\n",
        "'''from transformers import AutoTokenizer, XLMRobertaXLForSequenceClassification\n",
        "#tokenizer = AutoTokenizer.from_pretrained(\"xlm-roberta-large\")\n",
        "#model = XLMRobertaXLForSequenceClassification.from_pretrained(\"xlm-roberta-large\", num_labels=2)'''\n",
        "\n",
        "model = AutoModelForSequenceClassification.from_pretrained(model_name)\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_name)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eYPt4eBDBLg5",
        "outputId": "f16afac7-6549-4c7e-8005-1ab92afe5b26"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "model.to(device)\n",
        "learning_rate = 1e-5\n",
        "optimizer = AdamW(model.parameters(),lr=learning_rate,eps=1e-8)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RhTuitFFBO4z"
      },
      "outputs": [],
      "source": [
        "def encode_data(tokenizer, questions, passages, max_length):\n",
        "    \"\"\"Encode the question/passage pairs into features than can be fed to the model\"\"\"\n",
        "    input_ids = []\n",
        "    attention_masks = []\n",
        "\n",
        "    for question,passage in zip(questions,passages):\n",
        "        encoded_data = tokenizer.encode_plus(question, passage, max_length=max_length, pad_to_max_length=True,truncation_strategy=\"longest_first\")\n",
        "        encoded_pair = encoded_data[\"input_ids\"]\n",
        "        attention_mask = encoded_data[\"attention_mask\"]\n",
        "\n",
        "        input_ids.append(encoded_pair)\n",
        "        attention_masks.append(attention_mask)\n",
        "\n",
        "    return np.array(input_ids),np.array(attention_masks)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9gJXI9ovBQ_l"
      },
      "outputs": [],
      "source": [
        "passages_train = train_data_df.passage.values\n",
        "questions_train = train_data_df.question.values\n",
        "answers_train = train_data_df.answer.values.astype(int)\n",
        "\n",
        "passages_dev = dev_data_df.passage.values\n",
        "questions_dev = dev_data_df.question.values\n",
        "answers_dev = dev_data_df.answer.values.astype(int)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-ocFhJS2BTpO",
        "outputId": "ee946ce9-aa82-440f-99f9-1a9004db7208"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:2301: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "max_seq_length = 256\n",
        "#max_seq_length = 512\n",
        "input_ids_train,attention_masks_train = encode_data(tokenizer, questions_train, passages_train, max_seq_length)\n",
        "input_ids_dev,attention_masks_dev = encode_data(tokenizer, questions_dev, passages_dev, max_seq_length)\n",
        "\n",
        "train_features = (input_ids_train,attention_masks_train,answers_train)\n",
        "dev_features = (input_ids_dev,attention_masks_dev,answers_dev)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PTI1dW7lBW3A"
      },
      "outputs": [],
      "source": [
        "# building dataloaders\n",
        "batch_size = 16\n",
        "train_features_tensors = [torch.tensor(feature, dtype=torch.long) for feature in train_features]\n",
        "dev_features_tensors = [torch.tensor(feature, dtype=torch.long) for feature in dev_features]\n",
        "\n",
        "train_dataset = TensorDataset(*train_features_tensors)\n",
        "dev_dataset = TensorDataset(*dev_features_tensors)\n",
        "\n",
        "train_sampler = RandomSampler(train_dataset)\n",
        "dev_sampler = SequentialSampler(dev_dataset)\n",
        "\n",
        "train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=batch_size)\n",
        "dev_dataloader = DataLoader(dev_dataset, sampler=dev_sampler, batch_size=batch_size)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JVRodJUzBcy-"
      },
      "outputs": [],
      "source": [
        "torch.cuda.empty_cache()\n",
        "def cache_clear():\n",
        "    gc.collect()\n",
        "    torch.cuda.empty_cache()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "n7Ol-5jk7937"
      },
      "outputs": [],
      "source": [
        "from sklearn.metrics import confusion_matrix\n",
        "from sklearn.metrics import f1_score"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mcHaqWzyBmJr",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "92eb34bc-58a1-416b-874a-9196d41b6bb4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch: 100%|██████████| 3/3 [22:03<00:00, 441.03s/it]\n"
          ]
        }
      ],
      "source": [
        "epochs = 3\n",
        "grad_acc_steps = 4\n",
        "train_loss_values = []\n",
        "dev_acc_values = []\n",
        "\n",
        "for _ in tqdm(range(epochs), desc=\"Epoch\"):\n",
        "\n",
        "    #training\n",
        "    epoch_train_loss = 0\n",
        "    model.train()\n",
        "    model.zero_grad()\n",
        "\n",
        "    for step,batch in enumerate(train_dataloader):\n",
        "        inputs_ids = batch[0].to(device)\n",
        "        attention_masks = batch[1].to(device)\n",
        "        labels = batch[2].to(device)\n",
        "\n",
        "        # cache_clear()\n",
        "        outputs = model(inputs_ids, token_type_ids=None, attention_mask=attention_masks, labels=labels)\n",
        "        #outputs = model(inputs_ids, attention_mask=attention_masks)\n",
        "\n",
        "        loss = outputs[0]\n",
        "        loss = loss / grad_acc_steps\n",
        "        epoch_train_loss += loss.item()\n",
        "\n",
        "        loss.backward()\n",
        "\n",
        "        if(step+1) % grad_acc_steps == 0:\n",
        "            torch.nn.utils.clip_grad_norm_(model.parameters(),1.0)\n",
        "            optimizer.step()\n",
        "            model.zero_grad()\n",
        "\n",
        "    epoch_train_loss = epoch_train_loss / len(train_dataloader)\n",
        "    train_loss_values.append(epoch_train_loss)\n",
        "\n",
        "    #Evaluation\n",
        "    epoch_dev_accuracy = 0\n",
        "    model.eval()\n",
        "\n",
        "    y_pred = []\n",
        "    y_true = []\n",
        "    for batch in dev_dataloader:\n",
        "\n",
        "        input_ids = batch[0].to(device)\n",
        "        attention_masks = batch[1].to(device)\n",
        "        labels = batch[2]\n",
        "\n",
        "        with torch.no_grad():\n",
        "            outputs = model(input_ids, token_type_ids=None, attention_mask = attention_masks)\n",
        "            # outputs = model(inputs_ids, attention_mask=attention_masks, labels=labels)\n",
        "\n",
        "        logits = outputs[0]\n",
        "        logits = logits.detach().cpu().numpy()\n",
        "\n",
        "        predictions = np.argmax(logits,axis=1).flatten()\n",
        "        labels = labels.numpy().flatten()\n",
        "\n",
        "        y_pred.extend(predictions)\n",
        "        y_true.extend(labels)\n",
        "        epoch_dev_accuracy += np.sum(predictions==labels)/len(labels)\n",
        "\n",
        "\n",
        "    cf_matrix = confusion_matrix(y_true, y_pred)\n",
        "    score_f1 = f1_score(y_true, y_pred)\n",
        "    epoch_dev_accuracy = epoch_dev_accuracy / len(dev_dataloader)\n",
        "    dev_acc_values.append(epoch_dev_accuracy)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5rK3vT76igmW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b7a53b49-ac1f-41ce-e381-dae3455aba10"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[ 752  485]\n",
            " [ 265 1768]]\n"
          ]
        }
      ],
      "source": [
        "print(cf_matrix)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XbBr76DPMiu-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "72f0582c-a0b1-4f78-82cd-69db2f16e06e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "          No       0.74      0.61      0.67      1237\n",
            "         Yes       0.78      0.87      0.83      2033\n",
            "\n",
            "    accuracy                           0.77      3270\n",
            "   macro avg       0.76      0.74      0.75      3270\n",
            "weighted avg       0.77      0.77      0.77      3270\n",
            "\n"
          ]
        }
      ],
      "source": [
        "from sklearn.metrics import classification_report\n",
        "target_names = ['No', 'Yes']\n",
        "print(classification_report(y_true, y_pred,target_names=target_names))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2vPxvFJ9Ojt1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bfbfdde8-f48e-46d4-f739-d406f100f0df"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Losses : [0.16526830287302954, 0.14491456675074868, 0.11725915095184819]\n",
            "Accuracy : [0.6275406504065041, 0.7116869918699188, 0.7708333333333334]\n"
          ]
        }
      ],
      "source": [
        "print(\"Training Losses :\",train_loss_values)\n",
        "print(\"Accuracy :\",dev_acc_values)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QlnX90NNRPq_"
      },
      "outputs": [],
      "source": [
        "def predict(question, passage):\n",
        "  sequence = tokenizer.encode_plus(question, passage, return_tensors=\"pt\")['input_ids'].to(device)\n",
        "\n",
        "  logits = model(sequence)[0]\n",
        "  probabilities = torch.softmax(logits, dim=1).detach().cpu().tolist()[0]\n",
        "  proba_yes = round(probabilities[1], 2)\n",
        "  proba_no = round(probabilities[0], 2)\n",
        "\n",
        "  print(f\"Question: {question}, Yes: {proba_yes}, No: {proba_no}\")\n",
        "\n",
        "passage_planets = '''A planet is a large, rounded astronomical body that is neither a star nor its remnant. The best available theory of planet formation is the nebular hypothesis, which posits that an interstellar cloud collapses out of a nebula to create a young protostar orbited by a protoplanetary disk. Planets grow in this disk by the gradual accumulation of material driven by gravity, a process called accretion. The Solar System has at least eight planets: the terrestrial planets Mercury, Venus, Earth and Mars, and the giant planets Jupiter, Saturn, Uranus and Neptune. These planets each rotate around an axis tilted with respect to its orbital pole. All planets of the Solar System other than Mercury possess a considerable atmosphere, and some share such features as ice caps, seasons, volcanism, hurricanes, tectonics, and even hydrology. Apart from Venus and Mars, the Solar System planets generate magnetic fields, and all except Venus and Mercury have natural satellites. The giant planets bear planetary rings, the most prominent being those of Saturn.'''\n",
        "planets_questions = [\n",
        "    \"Mercury is a planet\",\n",
        "    \"Planet is a star\",\n",
        "    \"Mercury has a no atmosphere\",\n",
        "    \"There are eight planets in solar system\",\n",
        "    \"There are hundred planets in solar system\",\n",
        "    \"Venus is closer to sun compared to mercury\"\n",
        "\n",
        "]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eiv39OhdNXrF",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7278cf92-b18e-4413-e520-4526252b3c91"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Question: Mercury is a planet, Yes: 0.82, No: 0.18\n",
            "Question: Planet is a star, Yes: 0.07, No: 0.93\n",
            "Question: Mercury has a no atmosphere, Yes: 0.33, No: 0.67\n",
            "Question: There are eight planets in solar system, Yes: 0.92, No: 0.08\n",
            "Question: There are hundred planets in solar system, Yes: 0.74, No: 0.26\n",
            "Question: Venus is closer to sun compared to mercury, Yes: 0.35, No: 0.65\n"
          ]
        }
      ],
      "source": [
        "for s_question in planets_questions:\n",
        "  predict(s_question, passage_planets)"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyOwe1UZzlfwBeKns3MUJI7a",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "e43d09106ad54b259a8b135c7c033204": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0bf84df9a59d4342be2ad7998c9d2a36",
              "IPY_MODEL_020fb28df2fa4488b83879539db2faad",
              "IPY_MODEL_17a565f3da664ac3ba31dff990580a1e"
            ],
            "layout": "IPY_MODEL_8ee0f7cd15544a2ba5c76e460404a3e6"
          }
        },
        "0bf84df9a59d4342be2ad7998c9d2a36": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b439c52fe720408cb27b6189ae5076f6",
            "placeholder": "​",
            "style": "IPY_MODEL_79ba8ee5dcce48f5a778f1db5cce06a0",
            "value": "Downloading: 100%"
          }
        },
        "020fb28df2fa4488b83879539db2faad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_816805fd957346c9834f645bf5f70936",
            "max": 501200538,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a4a96fc58eeb42e5bcabd97e24dffa36",
            "value": 501200538
          }
        },
        "17a565f3da664ac3ba31dff990580a1e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b4d18a931ef543cf97fb06738cf03c96",
            "placeholder": "​",
            "style": "IPY_MODEL_984b867633aa44a3872747b7e8930e31",
            "value": " 478M/478M [00:13&lt;00:00, 27.2MB/s]"
          }
        },
        "8ee0f7cd15544a2ba5c76e460404a3e6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b439c52fe720408cb27b6189ae5076f6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "79ba8ee5dcce48f5a778f1db5cce06a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "816805fd957346c9834f645bf5f70936": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a4a96fc58eeb42e5bcabd97e24dffa36": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b4d18a931ef543cf97fb06738cf03c96": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "984b867633aa44a3872747b7e8930e31": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}