semantic_similarity_with_bert.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "semantic_similarity_with_bert",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.0"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NfLAa-N6-hS2",
        "outputId": "bb71930e-4282-47b6-da46-158643601831"
      },
      "source": [
        "!pip install transformers\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import tensorflow as tf\n",
        "import transformers"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/3a/83/e74092e7f24a08d751aa59b37a9fc572b2e4af3918cb66f7766c3affb1b4/transformers-3.5.1-py3-none-any.whl (1.3MB)\n",
            "\u001b[K     |████████████████████████████████| 1.3MB 5.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.8)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.4)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
            "Collecting tokenizers==0.9.3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/4c/34/b39eb9994bc3c999270b69c9eea40ecc6f0e97991dba28282b9fd32d44ee/tokenizers-0.9.3-cp36-cp36m-manylinux1_x86_64.whl (2.9MB)\n",
            "\u001b[K     |████████████████████████████████| 2.9MB 26.1MB/s \n",
            "\u001b[?25hCollecting sentencepiece==0.1.91\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/a4/d0a884c4300004a78cca907a6ff9a5e9fe4f090f5d95ab341c53d28cbc58/sentencepiece-0.1.91-cp36-cp36m-manylinux1_x86_64.whl (1.1MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1MB 37.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.6/dist-packages (from transformers) (3.12.4)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 36.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (1.15.0)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf->transformers) (50.3.2)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.17.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.11.8)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893257 sha256=e15b60249dd4020ff853f049c3bc2894bea1ec6b279260011231ae79e17a9f00\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: tokenizers, sentencepiece, sacremoses, transformers\n",
            "Successfully installed sacremoses-0.0.43 sentencepiece-0.1.91 tokenizers-0.9.3 transformers-3.5.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z6MvmHlA-hS3"
      },
      "source": [
        "## Configuration"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2CcAo4zI-hS3"
      },
      "source": [
        "max_length = 512  # Maximum length of input sentence to the model.\n",
        "batch_size = 8\n",
        "epochs = 2\n",
        "\n",
        "# Labels in our dataset.\n",
        "labels = [\"contradiction\", \"entailment\", \"neutral\"]"
      ],
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-w5BCdA6-hS3"
      },
      "source": [
        "## Load the Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FE5WkNHK-hS3",
        "outputId": "e6cada8d-1c0b-4f24-fe63-282ce6899e5d"
      },
      "source": [
        "# !curl -LO https://github.com/raw/MohamadMerchant/SNLI/master/data.tar.gz\n",
        "# !tar -xvzf data.tar.gz"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "100 11.1M  100 11.1M    0     0   9.9M      0  0:00:01  0:00:01 --:--:--  9.9M\n",
            "SNLI_Corpus/\n",
            "SNLI_Corpus/snli_1.0_dev.csv\n",
            "SNLI_Corpus/snli_1.0_train.csv\n",
            "SNLI_Corpus/snli_1.0_test.csv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RhJiWuLj-hS3",
        "outputId": "d0710c7c-6c6d-4607-a815-e961e9d18d7e"
      },
      "source": [
        "# There are more than 550k samples in total; we will use 100k for this example.\n",
        "train_df = pd.read_csv(\"SNLI_Corpus/snli_1.0_train.csv\", nrows=100000)\n",
        "valid_df = pd.read_csv(\"SNLI_Corpus/snli_1.0_dev.csv\")\n",
        "test_df = pd.read_csv(\"SNLI_Corpus/snli_1.0_test.csv\")\n",
        "\n",
        "# Shape of the data\n",
        "print(f\"Total train samples : {train_df.shape[0]}\")\n",
        "print(f\"Total validation samples: {valid_df.shape[0]}\")\n",
        "print(f\"Total test samples: {valid_df.shape[0]}\")"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Total train samples : 100000\n",
            "Total validation samples: 10000\n",
            "Total test samples: 10000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "id": "6Jy2E4ncPy4T",
        "outputId": "3c2582e6-d137-4629-f79b-1d8a7ac8d97f"
      },
      "source": [
        "test=test_df.drop(test_df[test_df['label'] == 2].index) \n",
        "test"
      ],
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>similarity</th>\n",
              "      <th>sentence1</th>\n",
              "      <th>sentence2</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>entailment</td>\n",
              "      <td>This church choir sings to the masses as they ...</td>\n",
              "      <td>The church is filled with song.</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>contradiction</td>\n",
              "      <td>This church choir sings to the masses as they ...</td>\n",
              "      <td>A choir singing at a baseball game.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>entailment</td>\n",
              "      <td>A woman with a green headscarf, blue shirt and...</td>\n",
              "      <td>The woman is very happy.</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>contradiction</td>\n",
              "      <td>A woman with a green headscarf, blue shirt and...</td>\n",
              "      <td>The woman has been shot.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>entailment</td>\n",
              "      <td>An old man with a package poses in front of an...</td>\n",
              "      <td>A man poses in front of an ad.</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9992</th>\n",
              "      <td>contradiction</td>\n",
              "      <td>A female dancer wearing all white dancing whil...</td>\n",
              "      <td>Some players are playing cricket in ground</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9995</th>\n",
              "      <td>contradiction</td>\n",
              "      <td>Two women are observing something together.</td>\n",
              "      <td>Two women are standing with their eyes closed.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9996</th>\n",
              "      <td>entailment</td>\n",
              "      <td>Two women are observing something together.</td>\n",
              "      <td>Two girls are looking at something.</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9997</th>\n",
              "      <td>contradiction</td>\n",
              "      <td>A man in a black leather jacket and a book in ...</td>\n",
              "      <td>A man is flying a kite.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9998</th>\n",
              "      <td>entailment</td>\n",
              "      <td>A man in a black leather jacket and a book in ...</td>\n",
              "      <td>A man is speaking in a classroom.</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>6605 rows × 4 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "         similarity  ... label\n",
              "1        entailment  ...     1\n",
              "2     contradiction  ...     0\n",
              "4        entailment  ...     1\n",
              "5     contradiction  ...     0\n",
              "6        entailment  ...     1\n",
              "...             ...  ...   ...\n",
              "9992  contradiction  ...     0\n",
              "9995  contradiction  ...     0\n",
              "9996     entailment  ...     1\n",
              "9997  contradiction  ...     0\n",
              "9998     entailment  ...     1\n",
              "\n",
              "[6605 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3RiYXjk8-hS3"
      },
      "source": [
        "Let's look at one sample from the dataset:"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cRgOy_MQ-hS3"
      },
      "source": [
        "## Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "g71QhUKj-hS3",
        "outputId": "3f82ed5e-18d2-415d-904f-085e92190c3d"
      },
      "source": [
        "# We have some NaN entries in our train data, we will simply drop them.\n",
        "print(\"Number of missing values\")\n",
        "print(train_df.isnull().sum())\n",
        "train_df.dropna(axis=0, inplace=True)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of missing values\n",
            "similarity    0\n",
            "sentence1     0\n",
            "sentence2     3\n",
            "dtype: int64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P0tmY2E8-hS4"
      },
      "source": [
        "train_df[\"label\"] = train_df[\"similarity\"].apply(\n",
        "    lambda x: 0 if x == \"contradiction\" else 1 if x == \"entailment\" else 2\n",
        ")\n",
        "y_train = tf.keras.utils.to_categorical(train_df.label, num_classes=3)\n",
        "\n",
        "valid_df[\"label\"] = valid_df[\"similarity\"].apply(\n",
        "    lambda x: 0 if x == \"contradiction\" else 1 if x == \"entailment\" else 2\n",
        ")\n",
        "y_val = tf.keras.utils.to_categorical(valid_df.label, num_classes=3)\n",
        "\n",
        "test[\"label\"] = test[\"similarity\"].apply(\n",
        "    lambda x: 0 if x == \"contradiction\" else 1\n",
        ")\n",
        "y_test = tf.keras.utils.to_categorical(test.label, num_classes=2)"
      ],
      "execution_count": 48,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vYxm3jFFGNEz"
      },
      "source": [
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split\n",
        "df=pd.read_csv(\"/content/sample_data/combined_data_paraphrase.csv\", delimiter=',', header=None, names=['sentence1', 'sentence2', 'label', 'is_sts'])\n",
        "df=df.iloc[1:]"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xiHmnxeHL4h2"
      },
      "source": [
        "df.label=df.label.astype(int)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pTVG1pF_GZer"
      },
      "source": [
        "y=df.label\n",
        "x=df.drop(['label','is_sts'],axis=1)\n",
        "x_train,x_test,y_train1,y_test1=train_test_split(x,y,test_size=0.2)"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PxpjTUgkT0RQ",
        "outputId": "54a4465c-ca72-4f20-c7bf-fcf8488ec4a5"
      },
      "source": [
        "length=0\n",
        "for index,row in x.iterrows():\n",
        "    if len(row['sentence2'])>length:\n",
        "      length=len(row['sentence2'])\n",
        "print(length)"
      ],
      "execution_count": 96,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1184\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BfV3iLHSYExX"
      },
      "source": [
        "%reset_selective -f len"
      ],
      "execution_count": 103,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2SspTPnSMujs",
        "outputId": "7c454ab7-43ee-49a8-9dec-4c205aff49ef"
      },
      "source": [
        "y_train2=tf.keras.utils.to_categorical(y_train1, num_classes=2)\n",
        "y_train2"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[1., 0.],\n",
              "       [1., 0.],\n",
              "       [0., 1.],\n",
              "       ...,\n",
              "       [1., 0.],\n",
              "       [0., 1.],\n",
              "       [1., 0.]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vsp90YgLNbGr"
      },
      "source": [
        "y_test2=tf.keras.utils.to_categorical(y_test1, num_classes=2)"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C1Zlc7u2-hS4"
      },
      "source": [
        "## Bert Custom Data Generator"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y7-VnNlE-hS4"
      },
      "source": [
        "\n",
        "class BertSemanticDataGenerator(tf.keras.utils.Sequence):\n",
        "    \"\"\"Generates batches of data.\n",
        "\n",
        "    Args:\n",
        "        sentence_pairs: Array of premise and hypothesis input sentences.\n",
        "        labels: Array of labels.\n",
        "        batch_size: Integer batch size.\n",
        "        shuffle: boolean, whether to shuffle the data.\n",
        "        include_targets: boolean, whether to incude the labels.\n",
        "\n",
        "    Returns:\n",
        "        Tuples `([input_ids, attention_mask, `token_type_ids], labels)`\n",
        "        (or just `[input_ids, attention_mask, `token_type_ids]`\n",
        "         if `include_targets=False`)\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(\n",
        "        self,\n",
        "        sentence_pairs,\n",
        "        labels,\n",
        "        batch_size=batch_size,\n",
        "        shuffle=True,\n",
        "        include_targets=True,\n",
        "    ):\n",
        "        self.sentence_pairs = sentence_pairs\n",
        "        self.labels = labels\n",
        "        self.shuffle = shuffle\n",
        "        self.batch_size = batch_size\n",
        "        self.include_targets = include_targets\n",
        "        # Load our BERT Tokenizer to encode the text.\n",
        "        # We will use base-base-uncased pretrained model.\n",
        "        self.tokenizer = transformers.BertTokenizer.from_pretrained(\n",
        "            \"bert-base-uncased\", do_lower_case=True\n",
        "        )\n",
        "        self.indexes = np.arange(len(self.sentence_pairs))\n",
        "        self.on_epoch_end()\n",
        "\n",
        "    def __len__(self):\n",
        "        # Denotes the number of batches per epoch.\n",
        "        return len(self.sentence_pairs) // self.batch_size\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        # Retrieves the batch of index.\n",
        "        indexes = self.indexes[idx * self.batch_size : (idx + 1) * self.batch_size]\n",
        "        sentence_pairs = self.sentence_pairs[indexes]\n",
        "\n",
        "        # With BERT tokenizer's batch_encode_plus batch of both the sentences are\n",
        "        # encoded together and separated by [SEP] token.\n",
        "        encoded = self.tokenizer.batch_encode_plus(\n",
        "            sentence_pairs.tolist(),\n",
        "            add_special_tokens=True,\n",
        "            max_length=max_length,\n",
        "            truncation=True,\n",
        "            return_attention_mask=True,\n",
        "            return_token_type_ids=True,\n",
        "            pad_to_max_length=True,\n",
        "            return_tensors=\"tf\",\n",
        "        )\n",
        "\n",
        "        # Convert batch of encoded features to numpy array.\n",
        "        input_ids = np.array(encoded[\"input_ids\"], dtype=\"int32\")\n",
        "        attention_masks = np.array(encoded[\"attention_mask\"], dtype=\"int32\")\n",
        "        token_type_ids = np.array(encoded[\"token_type_ids\"], dtype=\"int32\")\n",
        "\n",
        "        # Set to true if data generator is used for training/validation.\n",
        "        if self.include_targets:\n",
        "            labels = np.array(self.labels[indexes], dtype=\"int32\")\n",
        "            return [input_ids, attention_masks, token_type_ids], labels\n",
        "        else:\n",
        "            return [input_ids, attention_masks, token_type_ids]\n",
        "\n",
        "    def on_epoch_end(self):\n",
        "        # Shuffle indexes after each epoch if shuffle is set to True.\n",
        "        if self.shuffle:\n",
        "            np.random.RandomState(42).shuffle(self.indexes)\n"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9eWhG6uF-hS4"
      },
      "source": [
        "## Build the model."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KM3RZRzz-hS4",
        "outputId": "9965096f-b223-4feb-ab27-4abc06155a80"
      },
      "source": [
        "# Create the model under a distribution strategy scope.\n",
        "strategy = tf.distribute.MirroredStrategy()\n",
        "\n",
        "with strategy.scope():\n",
        "    # Encoded token ids from BERT tokenizer.\n",
        "    input_ids = tf.keras.layers.Input(\n",
        "        shape=(max_length,), dtype=tf.int32, name=\"input_ids\"\n",
        "    )\n",
        "    # Attention masks indicates to the model which tokens should be attended to.\n",
        "    attention_masks = tf.keras.layers.Input(\n",
        "        shape=(max_length,), dtype=tf.int32, name=\"attention_masks\"\n",
        "    )\n",
        "    # Token type ids are binary masks identifying different sequences in the model.\n",
        "    token_type_ids = tf.keras.layers.Input(\n",
        "        shape=(max_length,), dtype=tf.int32, name=\"token_type_ids\"\n",
        "    )\n",
        "    # Loading pretrained BERT model.\n",
        "    bert_model = transformers.TFBertModel.from_pretrained(\"bert-base-uncased\")\n",
        "    # Freeze the BERT model to reuse the pretrained features without modifying them.\n",
        "    bert_model.trainable = False\n",
        "\n",
        "    sequence_output, pooled_output = bert_model(\n",
        "        input_ids, attention_mask=attention_masks, token_type_ids=token_type_ids\n",
        "    )\n",
        "    # Add trainable layers on top of frozen layers to adapt the pretrained features on the new data.\n",
        "    bi_lstm = tf.keras.layers.Bidirectional(\n",
        "        tf.keras.layers.LSTM(64, return_sequences=True)\n",
        "    )(sequence_output)\n",
        "    # Applying hybrid pooling approach to bi_lstm sequence output.\n",
        "    avg_pool = tf.keras.layers.GlobalAveragePooling1D()(bi_lstm)\n",
        "    max_pool = tf.keras.layers.GlobalMaxPooling1D()(bi_lstm)\n",
        "    concat = tf.keras.layers.concatenate([avg_pool, max_pool])\n",
        "    dropout = tf.keras.layers.Dropout(0.3)(concat)\n",
        "    output = tf.keras.layers.Dense(2, activation=\"softmax\")(dropout)\n",
        "    model = tf.keras.models.Model(\n",
        "        inputs=[input_ids, attention_masks, token_type_ids], outputs=output\n",
        "    )\n",
        "\n",
        "    model.compile(\n",
        "        optimizer=tf.keras.optimizers.Adam(),\n",
        "        loss=\"categorical_crossentropy\",\n",
        "        metrics=[\"acc\"],\n",
        "    )\n",
        "\n",
        "\n",
        "print(f\"Strategy: {strategy}\")\n",
        "model.summary()"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Using MirroredStrategy with devices ('/job:localhost/replica:0/task:0/device:GPU:0',)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some layers from the model checkpoint at bert-base-uncased were not used when initializing TFBertModel: ['nsp___cls', 'mlm___cls']\n",
            "- This IS expected if you are initializing TFBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "All the layers of TFBertModel were initialized from the model checkpoint at bert-base-uncased.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFBertModel for predictions without further training.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Strategy: <tensorflow.python.distribute.mirrored_strategy.MirroredStrategy object at 0x7f29e30b2048>\n",
            "Model: \"functional_7\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_ids (InputLayer)          [(None, 512)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "attention_masks (InputLayer)    [(None, 512)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "token_type_ids (InputLayer)     [(None, 512)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "tf_bert_model_3 (TFBertModel)   ((None, 512, 768), ( 109482240   input_ids[0][0]                  \n",
            "                                                                 attention_masks[0][0]            \n",
            "                                                                 token_type_ids[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "bidirectional_3 (Bidirectional) (None, 512, 128)     426496      tf_bert_model_3[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "global_average_pooling1d_3 (Glo (None, 128)          0           bidirectional_3[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "global_max_pooling1d_3 (GlobalM (None, 128)          0           bidirectional_3[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_3 (Concatenate)     (None, 256)          0           global_average_pooling1d_3[0][0] \n",
            "                                                                 global_max_pooling1d_3[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "dropout_151 (Dropout)           (None, 256)          0           concatenate_3[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "dense_3 (Dense)                 (None, 2)            514         dropout_151[0][0]                \n",
            "==================================================================================================\n",
            "Total params: 109,909,250\n",
            "Trainable params: 427,010\n",
            "Non-trainable params: 109,482,240\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G_Pvf3pY-hS4"
      },
      "source": [
        "Create train and validation data generators"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xEEgr2j9-hS5"
      },
      "source": [
        "train_data = BertSemanticDataGenerator(\n",
        "    x_train[[\"sentence1\", \"sentence2\"]].values.astype(\"str\"),\n",
        "    y_train2,\n",
        "    batch_size=batch_size,\n",
        "    shuffle=True,\n",
        ")\n",
        "valid_data = BertSemanticDataGenerator(\n",
        "    x_test[[\"sentence1\", \"sentence2\"]].values.astype(\"str\"),\n",
        "    y_test2,\n",
        "    batch_size=batch_size,\n",
        "    shuffle=False,\n",
        ")"
      ],
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hgUrllr9-hS5"
      },
      "source": [
        "## Train the Model\n",
        "\n",
        "Training is done only for the top layers to perform \"feature extraction\",\n",
        "which will allow the model to use the representations of the pretrained model."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_CNjIBxq-hS5",
        "outputId": "be8f62ef-0ac6-4d3b-cecc-38cc437bf2b8"
      },
      "source": [
        "history = model.fit(\n",
        "    train_data,\n",
        "    validation_data=valid_data,\n",
        "    epochs=epochs,\n",
        "    use_multiprocessing=True,\n",
        "    workers=-1,\n",
        ")"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/transformers/tokenization_utils_base.py:2022: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/2\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/data/ops/multi_device_iterator_ops.py:601: get_next_as_optional (from tensorflow.python.data.ops.iterator_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.data.Iterator.get_next_as_optional()` instead.\n",
            "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
            "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
            "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
            "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
            "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
            "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
            "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
            "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
            "697/697 [==============================] - ETA: 0s - loss: 0.5871 - acc: 0.6680INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
            "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
            "697/697 [==============================] - 1294s 2s/step - loss: 0.5871 - acc: 0.6680 - val_loss: 0.4897 - val_acc: 0.7500\n",
            "Epoch 2/2\n",
            "697/697 [==============================] - 1296s 2s/step - loss: 0.4642 - acc: 0.7670 - val_loss: 0.4111 - val_acc: 0.8015\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BHuBbjgM-hS5"
      },
      "source": [
        "## Fine-tuning\n",
        "\n",
        "This step must only be performed after the feature extraction model has\n",
        "been trained to convergence on the new data.\n",
        "\n",
        "This is an optional last step where `bert_model` is unfreezed and retrained\n",
        "with a very low learning rate. This can deliver meaningful improvement by\n",
        "incrementally adapting the pretrained features to the new data."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "osnTay7N-hS5",
        "outputId": "39463160-694e-46e9-9ae3-55976f1b8589"
      },
      "source": [
        "# Unfreeze the bert_model.\n",
        "bert_model.trainable = True\n",
        "# Recompile the model to make the change effective.\n",
        "model.compile(\n",
        "    optimizer=tf.keras.optimizers.Adam(1e-5),\n",
        "    loss=\"categorical_crossentropy\",\n",
        "    metrics=[\"accuracy\"],\n",
        ")\n",
        "model.summary()"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"functional_7\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_ids (InputLayer)          [(None, 512)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "attention_masks (InputLayer)    [(None, 512)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "token_type_ids (InputLayer)     [(None, 512)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "tf_bert_model_3 (TFBertModel)   ((None, 512, 768), ( 109482240   input_ids[0][0]                  \n",
            "                                                                 attention_masks[0][0]            \n",
            "                                                                 token_type_ids[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "bidirectional_3 (Bidirectional) (None, 512, 128)     426496      tf_bert_model_3[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "global_average_pooling1d_3 (Glo (None, 128)          0           bidirectional_3[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "global_max_pooling1d_3 (GlobalM (None, 128)          0           bidirectional_3[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_3 (Concatenate)     (None, 256)          0           global_average_pooling1d_3[0][0] \n",
            "                                                                 global_max_pooling1d_3[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "dropout_151 (Dropout)           (None, 256)          0           concatenate_3[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "dense_3 (Dense)                 (None, 2)            514         dropout_151[0][0]                \n",
            "==================================================================================================\n",
            "Total params: 109,909,250\n",
            "Trainable params: 109,909,250\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Tl_iA4yM-hS5"
      },
      "source": [
        "# Train the entire model end-to-end."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vDff1z2w-hS5",
        "outputId": "2c2b358d-bd3c-47a7-8a88-c6b5a56a0b02"
      },
      "source": [
        "history = model.fit(\n",
        "    train_data,\n",
        "    validation_data=valid_data,\n",
        "    epochs=epochs,\n",
        "    use_multiprocessing=True,\n",
        "    workers=-1,\n",
        ")"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/transformers/tokenization_utils_base.py:2022: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/2\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model_3/bert/pooler/dense/kernel:0', 'tf_bert_model_3/bert/pooler/dense/bias:0'] when minimizing the loss.\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model_3/bert/pooler/dense/kernel:0', 'tf_bert_model_3/bert/pooler/dense/bias:0'] when minimizing the loss.\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model_3/bert/pooler/dense/kernel:0', 'tf_bert_model_3/bert/pooler/dense/bias:0'] when minimizing the loss.\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model_3/bert/pooler/dense/kernel:0', 'tf_bert_model_3/bert/pooler/dense/bias:0'] when minimizing the loss.\n",
            "2790/2790 [==============================] - 3122s 1s/step - loss: 0.3722 - accuracy: 0.8134 - val_loss: 0.2488 - val_accuracy: 0.8885\n",
            "Epoch 2/2\n",
            "2790/2790 [==============================] - 3118s 1s/step - loss: 0.1634 - accuracy: 0.9308 - val_loss: 0.2001 - val_accuracy: 0.9154\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tYLyaL8v-hS5"
      },
      "source": [
        "## Evaluate model on the test set"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JEMkxh6k-hS5",
        "outputId": "df77ecb8-ac95-4fc4-e679-f84fa2c91abb"
      },
      "source": [
        "test_data = BertSemanticDataGenerator(\n",
        "    test[[\"sentence1\", \"sentence2\"]].values.astype(\"str\"),\n",
        "    y_test,\n",
        "    batch_size=batch_size,\n",
        "    shuffle=False,\n",
        ")\n",
        "model.evaluate(test_data, verbose=1)"
      ],
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/transformers/tokenization_utils_base.py:2022: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "825/825 [==============================] - 283s 343ms/step - loss: 1.0604 - accuracy: 0.5738\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[1.060362458229065, 0.5737878680229187]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 49
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s3wxLdHfdhg3",
        "outputId": "cfecd1cb-a656-417f-872f-de5f06d6247c"
      },
      "source": [
        "test_data1 = BertSemanticDataGenerator(\n",
        "    x_test[[\"sentence1\", \"sentence2\"]].values.astype(\"str\"),\n",
        "    y_test2,\n",
        "    batch_size=batch_size,\n",
        "    shuffle=False,\n",
        ")\n",
        "model.evaluate(test_data1, verbose=1)"
      ],
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/transformers/tokenization_utils_base.py:2022: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "697/697 [==============================] - 239s 343ms/step - loss: 0.2001 - accuracy: 0.9154\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.2000652551651001, 0.9153515100479126]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BOeNMLHYrm2B",
        "outputId": "92af70c4-b57a-46d7-a328-4c2e4bc2dbd1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 306
        }
      },
      "source": [
        "import csv\n",
        "data_msr=pd.read_csv(\"/content/sample_data/msr_paraphrase_data.txt\", sep = '\\t', quoting=csv.QUOTE_NONE,error_bad_lines=False);\n",
        "data_msr.head()"
      ],
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Sentence ID</th>\n",
              "      <th>String</th>\n",
              "      <th>Author</th>\n",
              "      <th>URL</th>\n",
              "      <th>Agency</th>\n",
              "      <th>Date</th>\n",
              "      <th>Web Date</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>702876</td>\n",
              "      <td>Amrozi accused his brother, whom he called \"th...</td>\n",
              "      <td>Darren Goodsir</td>\n",
              "      <td>www.theage.com.au</td>\n",
              "      <td>*</td>\n",
              "      <td>June 5 2003</td>\n",
              "      <td>2003/06/04</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>702977</td>\n",
              "      <td>Referring to him as only \"the witness\", Amrozi...</td>\n",
              "      <td>Darren Goodsir</td>\n",
              "      <td>www.smh.com.au</td>\n",
              "      <td>Sydney Morning Herald</td>\n",
              "      <td>June 5 2003</td>\n",
              "      <td>2003/06/04</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2108705</td>\n",
              "      <td>Yucaipa owned Dominick's before selling the ch...</td>\n",
              "      <td>MICHAEL GIBBS</td>\n",
              "      <td>www.nwherald.com</td>\n",
              "      <td>*</td>\n",
              "      <td>*</td>\n",
              "      <td>2003/08/23</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2108831</td>\n",
              "      <td>Yucaipa bought Dominick's in 1995 for $693 mil...</td>\n",
              "      <td>ALEX VEIGA</td>\n",
              "      <td>www.miami.com</td>\n",
              "      <td>*</td>\n",
              "      <td>*</td>\n",
              "      <td>2003/08/23</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1330381</td>\n",
              "      <td>They had published an advertisement on the Int...</td>\n",
              "      <td>Philip Pangalos</td>\n",
              "      <td>www.alertnet.org</td>\n",
              "      <td>*</td>\n",
              "      <td>*</td>\n",
              "      <td>2003/06/25</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Sentence ID  ...    Web Date\n",
              "0       702876  ...  2003/06/04\n",
              "1       702977  ...  2003/06/04\n",
              "2      2108705  ...  2003/08/23\n",
              "3      2108831  ...  2003/08/23\n",
              "4      1330381  ...  2003/06/25\n",
              "\n",
              "[5 rows x 7 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 57
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XtN0LuJxrwd4",
        "outputId": "be0cf196-d187-451d-a8ce-dcb25456d35f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 32
        }
      },
      "source": [
        "\n",
        "df_msr=pd.DataFrame()\n",
        "df_msr\n"
      ],
      "execution_count": 78,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "Empty DataFrame\n",
              "Columns: []\n",
              "Index: []"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 78
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VAJQ6uDAtjHB"
      },
      "source": [
        "syn, ant, raw,ind = [],[],[],[]\n",
        "index=0\n",
        "for i,row in data_msr.iterrows():\n",
        "  if i%2==0:\n",
        "    syn.append(row['String'])\n",
        "  else:\n",
        "    ant.append(row['String'])\n",
        "    raw.append(1)\n",
        "    index+=1\n",
        "    ind.append(index)\n",
        "# print(len(syn))\n",
        "# print(len(ant))\n",
        "# print(len(raw))\n",
        "# print(index)\n",
        "df_msr['sentence1']=syn\n",
        "df_msr['sentence2']=ant\n",
        "df_msr['label']=raw\n",
        "df_msr['index']=ind"
      ],
      "execution_count": 79,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AYY81dIuwGOb"
      },
      "source": [
        "testlabel_msr=df_msr.label\n",
        "trainfeatures_msr=df_msr.drop(['label','index'],axis=1)\n",
        "test_msr = tf.keras.utils.to_categorical(testlabel_msr, num_classes=2)"
      ],
      "execution_count": 94,
      "outputs": []
    }
  ]
}