BERT_Practice.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "BERT Practice",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyO3ScgMOSIIQlwNEeNNenep",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/debjyoti003/Important-Colab-Notebooks/blob/main/BERT_Practice.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sz-cvIzXg9we",
        "outputId": "090114fc-5f5a-4085-a95e-f1b18ab034b7"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cPacOPMxhp1y"
      },
      "source": [
        "import os\n",
        "import pandas as pd\n",
        "import seaborn as sns\n",
        "import numpy as np\n",
        "import re\n",
        "from zipfile import ZipFile\n",
        "import matplotlib.pyplot as plt\n",
        "import tensorflow as tf\n",
        "import tensorflow_hub as hub\n",
        "import logging\n",
        "logging.basicConfig(level=logging.INFO)"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4FGWc48phzGE"
      },
      "source": [
        "os.chdir(r'/content/drive/MyDrive/')"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rvnP_SzikkcJ"
      },
      "source": [
        "with ZipFile(r'/content/drive/MyDrive/Disaster tweets prediction.zip', 'r') as zipObj:\n",
        "   zipObj.extractall(r'/content/drive/MyDrive/Disaster tweets prediction')"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_3rUbflklMu1",
        "outputId": "4157aff0-4ee2-4571-b30b-6ad82cea33e9"
      },
      "source": [
        "os.chdir(r'/content/drive/MyDrive/Disaster tweets prediction')\n",
        "data_files = os.listdir()\n",
        "data_files"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['sample_submission.csv',\n",
              " 'test.csv',\n",
              " 'train.csv',\n",
              " 'tokenization.py',\n",
              " '__pycache__']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 203
        },
        "id": "rVsCQM4_mohG",
        "outputId": "40b2e54f-1a70-4d9d-fa60-555899664764"
      },
      "source": [
        "train_df = pd.read_csv(data_files[2])\n",
        "train_df.head()"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>keyword</th>\n",
              "      <th>location</th>\n",
              "      <th>text</th>\n",
              "      <th>target</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Our Deeds are the Reason of this #earthquake M...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>4</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Forest fire near La Ronge Sask. Canada</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>5</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>All residents asked to 'shelter in place' are ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>6</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>13,000 people receive #wildfires evacuation or...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>7</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Just got sent this photo from Ruby #Alaska as ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   id keyword  ...                                               text target\n",
              "0   1     NaN  ...  Our Deeds are the Reason of this #earthquake M...      1\n",
              "1   4     NaN  ...             Forest fire near La Ronge Sask. Canada      1\n",
              "2   5     NaN  ...  All residents asked to 'shelter in place' are ...      1\n",
              "3   6     NaN  ...  13,000 people receive #wildfires evacuation or...      1\n",
              "4   7     NaN  ...  Just got sent this photo from Ruby #Alaska as ...      1\n",
              "\n",
              "[5 rows x 5 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aWwTpkk0nDBo",
        "outputId": "1bc26ae0-9db2-4254-cf27-2bf84963c1a4"
      },
      "source": [
        "train_df.info()"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 7613 entries, 0 to 7612\n",
            "Data columns (total 5 columns):\n",
            " #   Column    Non-Null Count  Dtype \n",
            "---  ------    --------------  ----- \n",
            " 0   id        7613 non-null   int64 \n",
            " 1   keyword   7552 non-null   object\n",
            " 2   location  5080 non-null   object\n",
            " 3   text      7613 non-null   object\n",
            " 4   target    7613 non-null   int64 \n",
            "dtypes: int64(2), object(3)\n",
            "memory usage: 297.5+ KB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 349
        },
        "id": "ZcKtCatpnV60",
        "outputId": "eca8bb79-0453-416a-e989-e269a10703ea"
      },
      "source": [
        "sns.set_style('darkgrid')\n",
        "sns.catplot(x = \"target\",data = train_df, kind = \"count\",height=4,aspect=2)\n",
        "plt.title('Bar Plot showing the count of target column', size = 20)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'Bar Plot showing the count of target column')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkYAAAE7CAYAAAA4iU7uAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3QU9d7H8c+SAFISmiFRetuAhN4SQZCEJNI0tAsi0m0oCiIK+MhV8UERRK7kEeGiCHK9XikhXqK00BIhdEQQEBAQhCw1RVraPH8w2cOSQoJsloT36xzOgd+07+zODJ+d+c2MxTAMQwAAAFAxVxcAAABwtyAYAQAAmAhGAAAAJoIRAACAiWAEAABgIhgBAACYCEa4Y7Zs2SJfX1/NnDnT1aXcUmBgoAIDA11dRq5mzpwpX19fbdmyxdWl5Flh2gYgnT17Vm+88Ybat2+vBg0ayNfXV0lJSa4uC5JOnjwpX19fjRs3ztWl3HPcXV3AvczX1zdLW/HixVW5cmW1atVKzz77rOrUqeOCyq47efKkgoKCHNrc3NxUvnx5+fn56amnnlKHDh3u+HJnzpyp8PBwLViwQG3atLnj88fty9wmevTooQ8++MDV5RRJvr6+at26tb766iunL2vcuHH68ccf1bVrV9WoUUMWi0UlS5a8a+orKOPGjVNERISio6NVtWpVV5cDFyMY3QVeeukl+9+Tk5O1Z88eLVu2TKtWrdLXX3+tBg0auLA6ycPDQ4MGDZIkXbt2TQcOHNCGDRu0YcMGvfnmmxo4cKBL6yuqnnrqKXXp0kUPPvigq0tBEZSSkqJNmzbp4Ycf1kcffeTqcoC7BsHoLjBy5MgsbZMmTdLChQs1f/58l/8y9/T0zFLjkiVLNGHCBH388cfq06ePSpUq5aLqiq6KFSuqYsWKri4DRdS5c+eUkZGhypUru7oU4K5CH6O7VNu2bSVJFy5ccGhPTk7W3LlzNXDgQLVv315+fn7y9/fX888/r127dmU7L19fXz399NM6e/as3nzzTT3yyCNq0KCBli5detv19ezZU6VLl9bly5d16NChW45/7Ngxvf7663rkkUfk5+endu3a6fXXX9exY8ccxgsMDFR4eLgkaeDAgfL19bX/uRXDMBQREaF+/frJ399fjRo1UocOHTRs2DB9//332U5z+fJlTZkyRY8++qj8/PwUHBysOXPmKKc35Xz//fd66qmn1KJFCzVu3Fjdu3fX7NmzlZKS4jBe37595efnp8uXLzu0DxgwQL6+vpowYYJD+5EjR+Tr66vXX3/d3pZTH6PM7/PChQt666231K5dO/n5+alr165asmRJtnWnpKRo5syZCgoKkp+fnwIDA/Xxxx8rJSXFPr9byZxekiIiIhy+m+y2pf379+vZZ59Vy5Yt1aRJEw0YMEA7d+7Mdt5paWn617/+pb/97W9q3ry5mjRporCwMC1cuFAZGRm3rO1GCQkJ+vjjj9WtWzc1adJELVq00OOPP65p06Zl+T7yul1K1y+3+Pr66uTJk1mG5dS36umnn5avr6/S0tL02WefKSQkRH5+furQoYOmTp3qsN0sXbrUvp1v3brV4fPNa5+t/OxnHTt2lOT4XebWnyWv9S1dulQjR45UUFCQGjdurObNm6tfv36KjIzMdr6Zn1FKSorCw8MVGhoqPz8/h1r27NmjoUOHqlmzZmrevLkGDx6sXbt25doP78iRIxo3bpw6dOggPz8/PfzwwxozZox+++03h/F8fX0VEREhSQoKCrKvU376IH7//fcaNGiQWrdurUaNGikwMFCvvvqqfv75Z4fxUlJSNGfOHHXv3l1NmjRR8+bN1b9//xyPT7l9XtnJ/I5u3h8z+1ReunRJkydPVocOHdS4cWM98cQTWrNmjaTr++CsWbMUEhKiRo0aqVOnTlq4cGGWZdy4rednHy9MOGN0l9q0aZMkyc/Pz6H9yJEjmjFjhlq2bKlHH31Unp6eOn36tNauXauYmBjNmjVL7du3zzK/hIQE9e3bV6VLl1ZISIgsFosqVar0l2rMDA8WiyXX8fbs2aMhQ4bo0qVLCgwMVN26dfXbb7/pu+++U3R0tObNm6fGjRtLuh6GoqOjtXXrVvXo0UNVqlTJcz0ff/yxZs+erapVq6pz587y8PDQ2bNn9fPPP2vFihXq0qWLw/ipqakaNmyYzpw5o/bt28vNzU1r1qzRRx99pJSUFIdLnJI0ffp0zZ49WxUqVFC3bt1UunRpxcTEaPr06YqNjdXnn3+uEiVKSJL8/f21e/dubd++3f59XLlyRbt375YkxcXFOcx78+bNkqSAgIA8rWtSUpKefPJJlShRQqGhoUpJSdGKFSs0YcIEFStWTD169LCPaxiGRo4cqfXr16tmzZoaMGCA0tLSFBERocOHD+f5823durUGDhyoBQsWqH79+urUqZN92M2Xe/fu3au5c+eqadOm6tOnj06dOqVVq1Zp8ODBWrZsmWrXrm0fNzU1Vc8//7xiY2NVq1YtdevWTSVLltSWLVs0adIk/fTTT5o6dWqeajxx4oQGDRqkP/74Qw0bNtSTTz6pjIwMHTt2TF9++aX69eun0qVLS8rfdvlXjRkzRjt27NAjjzyiDh06aOPGjZo7d64uXLig999/3/4ZvvTSSwoPD1eVKlUcvsPWrVvfchn53c/++OOPLN9lbpft81rf22+/rbp166pVq1by8vJSQkKCNmzYoNdff11Hjx7VqFGjsp3/yy+/rJ9//lnt27dXp06d7Menbdu2aejQocrIyFBwcLCqV6+uX3/9VQMHDpS/v3+289q4caNGjhyptLQ0dezYUdWrV5fNZtOqVau0fv16LViwQA0bNpR0vSvDmjVrdODAAQ0cOFCenp6SrnchuBXDMDR+/HhFRESoQoUKCg4OVsWKFRUfH68tW7aoVq1aatSokaTroWjYsGHaunWrateurf79++vq1atauXKlRo8erQMHDujVV1+95TJvV2pqqoYOHaqEhAQFBQUpNTVVy5cv18iRI/XFF1/o66+/1k8//aT27durRIkSWrFihSZNmqSKFStmOXZK+dvHCx0DLmO1Wg2r1Wp88skn9j+TJ082nnzyScPX19d47rnnjOTkZIdpkpKSjPPnz2eZ1+nTp422bdsajz32WI7LGTt2rJGamprn+k6cOGFYrVajY8eOWYYtWrTIsFqtRtOmTY0rV64YhmEYcXFx9vXJlJGRYTz22GOG1Wo1IiMjHeYRFRVlWK1WIzQ01EhPT7e3f/LJJ4bVajXi4uLyXKthGEbr1q2NRx55xLh8+XKWYTd/Zh07djSsVqsxfPhwe/2GYRjnzp0zWrRoYbRo0cJISUmxt+/cudOwWq1Ghw4djDNnztjbU1NTjeeee86wWq3GrFmz7O2bNm0yrFar8cEHH9jbNm7caFitVmPIkCGG1Wo1jh8/bh82YsQIw2q1GqdOnbrl55D5fU6YMMFIS0uztx86dMho0KCB0blzZ4fxIyIiDKvVavTv39+4du2avT0xMdEIDQ01rFarMWDAgGw+0awyt4k33ngj2+GZ24DVajWWLFniMOzf//63YbVajb///e8O7Znr+e677zqsT1pamjF+/HjDarUaq1evzlN9ffv2NaxWq/HZZ59lGXb+/Hnj6tWrhmHc3nb5xhtvGFar1Thx4kSO633jtm8YhjFgwADDarUaPXr0MC5evGhvv3TpktGpUyejfv36DtuTYRj5+j4y3c763Oq7zMmt6rtxu8507do1Y+DAgcZDDz1kxMfHOwzL/Iy6deuWZT9NT083goODDavVaqxfv95h2Ndff23f1m7cRxISEoyWLVsarVu3Ng4dOuQwzcGDB42mTZsaYWFhDu25fbe5+eabbwyr1Wr06tXLSEpKchiWlpZm2Gw2+78/++wz+zHnxuPwuXPn7MejHTt22Ntz+n4yP6/sLFmyJNt9L3P+zz33nMMxYNu2bYbVajVatWpl9OzZ00hMTLQP+/33342GDRsaTzzxhMO8bmcfL2y4lHYXCA8Pt//58ssvtWPHDtWpU0ddu3ZV2bJlHcb18PDItt+Jj4+PHnvsMf322286depUluHFixfXG2+8IXf3/J8kTEpK0syZMzVz5kxNmzZNzzzzjN58801J0ujRo3XfffflOO3OnTv122+/qVmzZnr88ccdhnXp0kUtWrTQ0aNHtWPHjnzXlR13d3e5ubllac+pr87//M//ONRfqVIlBQUFKTk5WUePHrW3Z16ieuGFF+Tl5eWwvDfeeEPFihXTokWL7O3NmzdXyZIlHc4Mbd68We7u7vb+WplniTIyMrR161bVrFlTDzzwQJ7Ws1SpUho/frzDutatW1fNmzfXkSNHdOnSJXv7smXLJEmjRo2yn9GSrvcdGzFiRJ6Wl1/NmzdXz549Hdp69eold3d37dmzx96WkZGhhQsXysvLK8v6uLm5ady4cbJYLPrvf/97y2Xu3btXu3btUoMGDfTMM89kGV6xYkX7HVcFvV2+9tprKl++vP3fpUuXVvfu3ZWRkaG9e/f+5fkX9Prkpnr16lnaSpQooaeeekppaWn27f5mr7zySpb9dOfOnTp+/LjatGmT5Q7Yvn37qmbNmlnms2zZMiUlJenll19W3bp1HYZZrVb16dNHv/zyS77OluYk81LTu+++m+UMk5ubm0P/rSVLlshisWjcuHEOx+FKlSrphRdekCSHY4gzTJgwweEY0LJlS1WtWlWJiYl67bXX7GfLJKlatWpq1qyZDh06pPT09Czzyus+XhhxKe0ucPDgQfvfL1++rMOHD2vatGl67bXXdPjwYY0ePdph/B07dmjBggXavXu3zp8/r9TUVIfhNpsty51MVapUue1LZ8nJyfZ+P25ubipXrpzat2+vAQMG3PJ2/V9++UWScrzt3t/fXzt27NAvv/yiVq1a3VZ9mbp3766vvvpKXbp0UefOndWqVSs1a9Ysx1PiHh4eqlGjRpZ2Hx8fSXJ4nkvmemR36r5WrVry8fHRyZMnlZycLA8PD5UsWVLNmjXTli1bdPHiRVWoUEFxcXFq1KiRmjVrpvvvv1+bN29W3759tW/fPiUlJalz5855XtcaNWpkCc03116mTBlJ1/v6FCtWTM2aNcsyfosWLfK8zPy4+RKwdD2cV6pUyeFzPXr0qBISElSzZk3NmjUr23ndd999WfqFZOenn36SJLVr107FiuX+m68gt0sp+88jMwQnJib+5fkX9Prk5tSpU/rnP/+pzZs36/Tp07p69arDcJvNlu102V223L9/v6Tst9NixYqpefPmWfpPZV6uPnDgQLZ9szLHP3LkSJbglB+XL1/Wr7/+qvvvv18PPfRQruP++eefOn78uLy9vbN9BEvmcSVzfZ3B09Mz29BauXJlnTx5Mttt1NvbW2lpaTp37py8vb0dhuV1Hy+MCEZ3mdKlS6tx48YKDw9Xhw4dNHfuXPXr189+EF29erVefvlllSxZUg8//LCqV6+uUqVKqVixYtq6dau2bt2apSOwJIezHPlVpUoVrV279ramTU5OlqQc73zJrCtzvL9i/Pjxqlq1qpYuXao5c+Zozpw5cnd3V/v27TVu3LgsIejGX0c3yvw1d+OvpMz6cvocvby8dOrUKSUlJdmDWEBAgOLi4rRlyxYFBARo//79eu655yRdPxBu2rRJhmHku3/R7dRerly5bM8W3n///XleZn7kVt+NnakTEhIkXf/PKjN8Z+fGM2A5yTwY33wAz05BbpdS9p9H5tmx/HYuz05Br09OTpw4od69eyspKUktW7ZUu3btVLZsWbm5uemPP/5QREREtsenG2u8UWa9OW2n2f3Yy9ymvv3221xrvbkjfn5l1paX7e3PP/+UlPPxI/N7c2agyOkHYuZxIbvhmcNu/vEt5X0fL4wIRncpT09P1apVS/v27dO+ffvswegf//iHihcvriVLlmT55TFx4kRt3bo12/ndqoO0s2TubGfPns12eGZ7dmc/8svNzU2DBw/W4MGDdf78ee3YsUNRUVFasWKFDh8+rKioKIfTyPmRuR7nzp3L9ldX5nrceHDJ/BW4adMmFStWTBkZGfbw4+/vr+XLl+vAgQOKi4uTxWJx2sMsy5Ytq8TERKWlpWUJR+fOnXPKMvMq8/MKDg7ONRjlReaBOqczEtktNz/bZeY+lN1lBWcHjlspyP0sN/PmzVNCQoLef//9LJdZli9fbr/7KzvZHaMy681pOz1//nyWtszPIjIyUvXr189z7fmVuZy8bG+3Wo8zZ844zDM3mZ9TdvtzYT9Tc7egj9FdLPMUu3HDrePHjx9X3bp1s4SijIyMAuk/kF+Zd7nkFNgyb7PNvENEkv0yyF/51VGpUiWFhIToH//4h/z9/fX777/r119/ve35Za5HdrcFHz9+XPHx8apatarDr6hGjRqpbNmyiouLU1xcnO677z775azM0LRhwwbt3LlTvr6+TntmUYMGDZSRkZHt4xzyu81knuXILhzcjtq1a8vT01O7d+/O9ldpfjRp0kSSFBsbe8tt53a2y3LlykmSTp8+nWX8m2/L/iuKFSuW78/3dtbnduVW3/HjxyVJISEhWYblVFtuMi9RZbedZmRkZHtreOZ2kJ9t+3aOOaVLl5bVatW5c+fslzJzUrZsWfudcdk9CiLz+7nVJTkp9+3wTvRXA8HorrVmzRqdPHlSxYsXd+gbUqVKFR07dszhV4phGJo5c+Yd6Ux4p7Vo0UK1atXSjh07tGLFCodhK1as0Pbt21WzZk2HPgSZnVSz60Sek5SUlGwPhKmpqfaA+VceQtmrVy9J0qxZsxyeLZWenq4pU6YoIyNDvXv3dpjGzc1NrVq10vHjx7VixQq1aNHCfsaqWrVqqlKlihYsWKArV67keNvxnRAWFiZJmjFjhsNljOTkZH366af5mpenp6csFku2B+Xb4e7urgEDBujs2bN67733svRHka7/ms7Ltu3n56dmzZpp//79+uc//5ll+MWLF3Xt2jVJt7ddZvaBubmD7MGDB7VgwYJbr2welS9fXvHx8fma5nbWxxn1ZT5e4+YQFBMTo8WLF+d7Wc2bN1f16tW1ZcsWbdiwwWHYf/7zn2xDRs+ePeXp6anw8PBsOwFnZGRk+YFzO8ccSfbnf02cODHLWcOMjAz7mSDp+jHEMAx9+OGHDsHywoUL9v0w8ziTm8zb/2/eDjdv3qyoqKh81Y/scSntLnBjB8HLly/ryJEj2rhxo6Trd33deH198ODB+vvf/64ePXooJCRE7u7u2rlzp44cOaKOHTtq3bp1BV5/biwWi6ZMmaIhQ4Zo9OjRWr58uWrXrq2jR49qzZo1KlOmjD788EOHzrL+/v4qVqyYpk+frkOHDtnPwuR2B9XVq1fVv39/1ahRQw0bNtSDDz6oa9euadOmTTpy5IgCAwP/0nvnmjdvruHDh2vu3Lnq1q2bQkNDVapUKcXExOjXX39VixYtNGzYsCzT+fv7a926dTp//nyW8BMQEGD/zyI//YvyKywsTFFRUYqJiVH37t0VGBio1NRUrVq1So0aNdLRo0fzfKm1TJkyatKkibZv364xY8aoVq1aKlasmAIDA2/7ssWIESN04MABffPNN1q3bp38/f3l7e2t8+fP6/jx49q5c6dGjx6dp46yU6dO1cCBAzV9+nStXLlSbdq0kWEYOnbsmH788Uf98MMPqlq16m1tl0FBQapZs6aWL1+u+Ph4NW7cWKdPn1Z0dLSCgoL0ww8/3Nb63ywgIEBRUVF6/vnn9dBDD8nd3V2tWrXKtdP07ayPM+rr37+/li5dqldeeUWhoaGqXLmyDh06pJiYGHXu3DlfDzKUrp/Jee+99zR8+HCNGDFCISEhql69ug4ePKgff/xR7du318aNGx3Wq0KFCvrkk0/04osv6m9/+5sCAgJUt25dWSwWxcfHa9euXUpISHA4yxcQEKDPP/9cb731lkJCQlSmTBl5enpqwIABudbXp08fbd++XZGRkQoJCVFQUJAqVqyoM2fOKC4uTr169bLfhTp06FBt3LhR0dHReuKJJ9S+fXtdvXpVK1as0Pnz5zV8+HC1bNnylp9Jr1699Pnnn2v27Nk6cOCA6tSpo2PHjikmJkbBwcFauXJlvj5jZEUwugvc2LfCzc1NFStWVMeOHTVgwAD7E7Az9evXTyVKlND8+fO1bNkylSxZUi1bttT777+vVatW3XXBSLp+anvx4sWaNWuWNm/erHXr1qlChQrq2rWrRowYkeVBYHXq1NEHH3xgf+hY5q/83IJRqVKl9Nprr2nLli3atWuX/T+D6tWr6+23387TL7FbGTt2rB566CEtXLhQy5YtU1pamqpXr65Ro0Zp6NCh2fZfujHw5BSM3N3d83RAvF0Wi0X/93//p88++0yRkZH66quvVLlyZfXo0UP9+/fXmjVr8tX35MMPP9T777+v2NhYRUVFyTAM+fj43HYwKl68uD799FNFRkYqIiJC69ev1+XLl1WhQgVVrVpVr7zyirp3756neVWrVk1Lly7V3LlztWbNGi1cuFAlS5ZUlSpVNHToUIfOuvndLkuWLKkvv/xSU6ZM0aZNm/Tzzz+rXr16+uijj1SuXLk7FozefPNNWSwWbd68WRs2bFBGRoZeeumlW95Nlt/1cUZ99evX14IFCzRjxgxt2LBBaWlpql+/vsLDw+Xh4ZHvYCRdv9Nu4cKFmjFjhtavX29f1wULFtgf43Dz9hsQEKDvvvtOX3zxhWJjY7V9+3b7C7r9/f0VGhrqMP4jjzyicePG6dtvv9X8+fOVmpqqKlWq3DIYWSwWffjhh2rXrp2+/fZb/fDDD0pJSZGXl5datGjh8PTsEiVKaN68eZo3b56WL1+uhQsXys3NTfXr19eECRPUrVu3PH0elSpV0sKFC/Xhhx9q27Zt2rZtm/z8/PTFF1/o5MmTBKM7wGIYObz7AECR9+OPP2ro0KF69tlnNWbMGFeXA+RLv379tGfPHm3fvt3+RHPgr6KPEXAPyO7OmYsXL9rfqh4cHFzQJQF5cuXKlWzvtlq6dKl27dqltm3bEopwR3EpDbgHfPDBBzpw4ICaNWtmf5dTTEyM/R16d+qdYMCddurUKfXo0UMPP/ywatSoofT0dP3yyy/asWOHPD09c33xLXA7CEbAPSA4OFjnzp3TunXrlJycrBIlSqhevXrq3bt3lrvpgLvJ/fffr+7du2vbtm3asmWLUlNTdf/996tnz5564YUXsn2uGPBX0McIAADAVCTPGKWkpCkx8YqrywAAAHcpL6/snzReJDtfu+r1FwAAoHArksEIAADgdhCMAAAATAQjAAAAE8EIAADARDACAAAwEYwAAABMBCMAAAATwQgAAMBEMAIAADARjAAAAExF8l1pBaWs530qVbK4q8sACq0r11L1Z9JVV5cBAHYEo7+gVMniajF2gavLAAqtHVMH6k8RjADcPbiUBgAAYCIYAQAAmAhGAAAAJoIRAACAiWAEAABgIhgBAACYCEYAAAAmghEAAICJYAQAAGAiGAEAAJgIRgAAACaCEQAAgIlgBAAAYCIYAQAAmJwejNLT0xUWFqbnnntOknTixAn16dNHwcHBGjVqlFJSUiRJKSkpGjVqlIKDg9WnTx+dPHnSPo/Zs2crODhYoaGhiomJcXbJAADgHuX0YLRgwQLVqVPH/u9p06Zp8ODBWr16tTw9PbV48WJJ0qJFi+Tp6anVq1dr8ODBmjZtmiTp8OHDioqKUlRUlObOnat33nlH6enpzi4bAADcg5wajOLj47V+/Xr17t1bkmQYhuLi4hQaGipJ6tGjh6KjoyVJa9euVY8ePSRJoaGh2rx5swzDUHR0tLp27aoSJUqoWrVqqlGjhvbs2ePMsgEAwD3K3Zkznzx5ssaOHatLly5Jki5evChPT0+5u19frI+Pj2w2myTJZrPpgQceuF6Uu7s8PDx08eJF2Ww2NWnSxD5Pb29v+zQ5cXOzqHz50s5YJQB3GPsqgLuJ04LRunXrVLFiRfn5+WnLli3OWky20tMNJSRcdvpyvLw8nL4MoKgriH0VAG6W0//hTgtGO3fu1Nq1a7Vx40Zdu3ZNf/75p/73f/9XSUlJSktLk7u7u+Lj4+Xt7S3p+pmg06dPy8fHR2lpaUpOTlaFChXk7e2t+Ph4+3xtNpt9GgAAgDvJaX2MxowZo40bN2rt2rWaPn26/P399dFHH6lNmzZauXKlJCkiIkKBgYGSpMDAQEVEREiSVq5cKX9/f1ksFgUGBioqKkopKSk6ceKEjh07psaNGzurbAAAcA8r8OcYjR07VvPmzVNwcLASEhLUp08fSVLv3r2VkJCg4OBgzZs3T6+99pokqV69eurcubO6dOmi4cOHa+LEiXJzcyvosgEAwD3AYhiG4eoi7rTU1PQC62PUYuwCpy8HKKp2TB2os2eTXV0GgHtQTn2MePI1AACAiWAEAABgIhgBAACYCEYAAAAmghEAAICJYAQAAGAiGAEAAJgIRgAAACaCEQAAgIlgBAAAYCIYAQAAmAhGAAAAJoIRAACAiWAEAABgIhgBAACYCEYAAAAmghEAAICJYAQAAGAiGAEAAJgIRgAAACaCEQAAgIlgBAAAYCIYAQAAmAhGAAAAJoIRAACAiWAEAABgIhgBAACYCEYAAAAmghEAAICJYAQAAGAiGAEAAJgIRgAAACaCEQAAgIlgBAAAYHJ3dQEAUJRULFdcbiXuc3UZQKGUnnJVFxJTXVoDwQgA7iC3Evfp93cbuboMoFCqPvFnSa4NRlxKAwAAMBGMAAAATAQjAAAAE8EIAADARDACAAAwEYwAAABMBCMAAAATwQgAAMBEMAIAADA5LRhdu3ZNvXv31uOPP66uXbvqk08+kSSdOHFCffr0UXBwsEaNGqWUlBRJUkpKikaNGqXg4GD16dNHJ0+etM9r9uzZCg4OVmhoqGJiYpxVMgAAuMc5LRiVKFFC8+fP13fffadly5YpJiZGu3fv1rRp0zR48GCtXr1anp6eWrx4sSRp0aJF8vT01OrVqzV48GBNmzZNknT48GFFRUUpKipKc+fO1TvvvKP09HRnlQ0AAO5hTgtGFiZ5vhAAABDtSURBVItFZcqUkSSlpaUpLS1NFotFcXFxCg0NlST16NFD0dHRkqS1a9eqR48ekqTQ0FBt3rxZhmEoOjpaXbt2VYkSJVStWjXVqFFDe/bscVbZAADgHubUPkbp6el64okn9PDDD+vhhx9WtWrV5OnpKXf36++u9fHxkc1mkyTZbDY98MADkiR3d3d5eHjo4sWLstls8vHxsc/T29vbPg0AAMCd5O7Mmbu5uSkyMlJJSUl68cUX9dtvvzlzcTcs16Ly5UsXyLIA/DXsqwBu5OpjglODUSZPT0+1adNGu3fvVlJSktLS0uTu7q74+Hh5e3tLun4m6PTp0/Lx8VFaWpqSk5NVoUIFeXt7Kz4+3j4vm81mnyYn6emGEhIuO3WdJMnLy8PpywCKuoLYVwsSxwXgrymoY0JO+6rTLqVduHBBSUlJkqSrV69q06ZNqlOnjtq0aaOVK1dKkiIiIhQYGChJCgwMVEREhCRp5cqV8vf3l8ViUWBgoKKiopSSkqITJ07o2LFjaty4sbPKBgAA9zCnnTE6c+aMxo0bp/T0dBmGoccee0wdO3ZU3bp1NXr0aM2YMUMNGjRQnz59JEm9e/fW2LFjFRwcrHLlyunjjz+WJNWrV0+dO3dWly5d5ObmpokTJ8rNzc1ZZQMAgHuYxTAMw9VF3GmpqekFdimtxdgFTl8OUFTtmDpQZ88mu7qMO8rLy0O/v9vI1WUAhVL1iT8X2DGhwC+lAQAAFDYEIwAAABPBCAAAwEQwAgAAMBGMAAAATAQjAAAAE8EIAADARDACAAAwEYwAAABMBCMAAAATwQgAAMBEMAIAADARjAAAAEwEIwAAABPBCAAAwEQwAgAAMBGMAAAATAQjAAAAE8EIAADARDACAAAwEYwAAABMBCMAAAATwQgAAMCUp2A0aNCgPLUBAAAUZu65Dbx27ZquXLmiixcvKjExUYZhSJL+/PNP2Wy2AikQAACgoOQajL755hvNnz9fZ86cUc+ePe3BqGzZshowYECBFAgAAFBQcg1GgwYN0qBBg/TVV1/p6aefLqiaAAAAXCLXYJTp6aef1s6dO/XHH38oPT3d3h4WFua0wgAAAApanoLR2LFjdeLECdWvX19ubm6SJIvFQjACAABFSp6C0d69e/X999/LYrE4ux4AAACXydPt+vXq1dPZs2edXQsAAIBL5emM0cWLF9W1a1c1btxYxYsXt7d/9tlnTisMAACgoOUpGI0cOdLZdQAAALhcnoJR69atnV0HAACAy+UpGDVr1sze8To1NVVpaWkqVaqUdu7c6dTiAAAAClKegtGuXbvsfzcMQ9HR0dq9e7fTigIAAHCFPN2VdiOLxaJOnTopNjbWGfUAAAC4TJ7OGK1atcr+94yMDO3du1clS5Z0WlEAAACukKdgtG7dOvvf3dzcVKVKFX366adOKwoAAMAV8hSM3n//fWfXAQAA4HJ56mMUHx+vF198UQEBAQoICNDIkSMVHx/v7NoAAAAKVJ6C0fjx4xUYGKiYmBjFxMSoY8eOGj9+vLNrAwAAKFB5CkYXLlxQr1695O7uLnd3d/Xs2VMXLlxwdm0AAAAFKk/BqHz58oqMjFR6errS09MVGRmp8uXLO7s2AACAApWnYDR58mT98MMPatu2rdq1a6eVK1fqgw8+cHZtAAAABSpPd6V98sknmjJlisqVKydJSkhI0JQpU7hbDQAAFCl5OmN08OBBeyiSrl9a279/f67TnD59Wk8//bS6dOmirl27av78+ZKuh6ohQ4YoJCREQ4YMUWJioqTrrxp57733FBwcrO7du2vfvn32eUVERCgkJEQhISGKiIjI90oCAADkRZ6CUUZGhj3ASNfDTXp6eq7TuLm5ady4cfr+++/1n//8R19//bUOHz6sOXPmKCAgQKtWrVJAQIDmzJkjSdq4caOOHTumVatWadKkSXr77bftywoPD9e3336rRYsWKTw83KEWAACAOyVPwWjo0KHq27evZsyYoRkzZqhfv34aNmxYrtNUrlxZDRs2lCSVLVtWtWvXls1mU3R0tMLCwiRJYWFhWrNmjSTZ2y0Wi5o2baqkpCSdOXNGsbGxatu2rcqXL69y5cqpbdu2iomJ+SvrDAAAkK089TEKCwuTn5+f4uLiJEnh4eGqW7dunhdy8uRJ7d+/X02aNNH58+dVuXJlSZKXl5fOnz8vSbLZbPLx8bFP4+PjI5vNlqXd29tbNpst1+W5uVlUvnzpPNcHwHXYVwHcyNXHhDwFI0mqW7duvsJQpkuXLunll1/WhAkTVLZsWYdhFotFFosl3/O8lfR0QwkJl+/4fG/m5eXh9GUARV1B7KsFieMC8NcU1DEhp301T5fSbldqaqpefvllde/eXSEhIZKkSpUq6cyZM5KkM2fOqGLFipKunwm68TUj8fHx8vb2ztJus9nk7e3tzLIBAMA9ymnByDAMvfnmm6pdu7aGDBlibw8MDNSyZcskScuWLVNQUJBDu2EY2r17tzw8PFS5cmW1a9dOsbGxSkxMVGJiomJjY9WuXTtnlQ0AAO5heb6Ull87duxQZGSkrFarnnjiCUnSq6++qmeffVajRo3S4sWL9eCDD2rGjBmSpA4dOmjDhg0KDg5WqVKlNHnyZEnXHw0wYsQI9e7dW5L04osv8tRtAADgFBbDMAxXF3GnpaamF1gfoxZjFzh9OUBRtWPqQJ09m+zqMu4oLy8P/f5uI1eXARRK1Sf+XGDHBJf0MQIAAChMCEYAAAAmghEAAICJYAQAAGAiGAEAAJgIRgAAACaCEQAAgIlgBAAAYCIYAQAAmAhGAAAAJoIRAACAiWAEAABgIhgBAACYCEYAAAAmghEAAICJYAQAAGAiGAEAAJgIRgAAACaCEQAAgIlgBAAAYCIYAQAAmAhGAAAAJoIRAACAiWAEAABgIhgBAACYCEYAAAAmghEAAICJYAQAAGAiGAEAAJgIRgAAACaCEQAAgIlgBAAAYCIYAQAAmAhGAAAAJoIRAACAiWAEAABgIhgBAACYCEYAAAAmghEAAICJYAQAAGAiGAEAAJgIRgAAACaCEQAAgIlgBAAAYCIYAQAAmAhGAAAAJqcFo/HjxysgIEDdunWztyUkJGjIkCEKCQnRkCFDlJiYKEkyDEPvvfeegoOD1b17d+3bt88+TUREhEJCQhQSEqKIiAhnlQsAAOC8YNSzZ0/NnTvXoW3OnDkKCAjQqlWrFBAQoDlz5kiSNm7cqGPHjmnVqlWaNGmS3n77bUnXg1R4eLi+/fZbLVq0SOHh4fYwBQAAcKc5LRi1atVK5cqVc2iLjo5WWFiYJCksLExr1qxxaLdYLGratKmSkpJ05swZxcbGqm3btipfvrzKlSuntm3bKiYmxlklAwCAe5x7QS7s/Pnzqly5siTJy8tL58+flyTZbDb5+PjYx/Px8ZHNZsvS7u3tLZvNdsvluLlZVL586TtcPQBnYF8FcCNXHxMKNBjdyGKxyGKxOGXe6emGEhIuO2XeN/Ly8nD6MoCiriD21YLEcQH4awrqmJDTvlqgd6VVqlRJZ86ckSSdOXNGFStWlHT9TFB8fLx9vPj4eHl7e2dpt9ls8vb2LsiSAQDAPaRAg1FgYKCWLVsmSVq2bJmCgoIc2g3D0O7du+Xh4aHKlSurXbt2io2NVWJiohITExUbG6t27doVZMkAAOAe4rRLaa+++qq2bt2qixcvqn379ho5cqSeffZZjRo1SosXL9aDDz6oGTNmSJI6dOigDRs2KDg4WKVKldLkyZMlSeXLl9eIESPUu3dvSdKLL76o8uXLO6tkAABwj7MYhmG4uog7LTU1vcD6GLUYu8DpywGKqh1TB+rs2WRXl3FHeXl56Pd3G7m6DKBQqj7x5wI7JtwVfYwAAADuZgQjAAAAE8EIAADARDACAAAwEYwAAABMBCMAAAATwQgAAMBEMAIAADARjAAAAEwEIwAAABPBCAAAwEQwAgAAMBGMAAAATAQjAAAAE8EIAADARDACAAAwEYwAAABMBCMAAAATwQgAAMBEMAIAADARjAAAAEwEIwAAABPBCAAAwEQwAgAAMBGMAAAATAQjAAAAE8EIAADARDACAAAwEYwAAABMBCMAAAATwQgAAMBEMAIAADARjAAAAEwEIwAAABPBCAAAwEQwAgAAMBGMAAAATAQjAAAAE8EIAADARDACAAAwEYwAAABMBCMAAAATwQgAAMBEMAIAADARjAAAAEwEIwAAAFOhCUYbN25UaGiogoODNWfOHFeXAwAAiqBCEYzS09P17rvvau7cuYqKitLy5ct1+PBhV5cFAACKmEIRjPbs2aMaNWqoWrVqKlGihLp27aro6GhXlwUAAIoYd1cXkBc2m00+Pj72f3t7e2vPnj05jl+8uJu8vDwKojTtmDqwQJYDFFUFta8WpOoTf3Z1CUCh5epjQqE4YwQAAFAQCkUw8vb2Vnx8vP3fNptN3t7eLqwIAAAURYUiGDVq1EjHjh3TiRMnlJKSoqioKAUGBrq6LAAAUMQUij5G7u7umjhxooYPH6709HT16tVL9erVc3VZAACgiLEYhmG4uggAAIC7QaG4lAYAAFAQCEYAAAAmghGKJF4hA+BG48ePV0BAgLp16+bqUnCXIxihyOEVMgBu1rNnT82dO9fVZaAQIBihyOEVMgBu1qpVK5UrV87VZaAQIBihyMnuFTI2m82FFQEACguCEQAAgIlghCKHV8gAAG4XwQhFDq+QAQDcLp58jSJpw4YNmjx5sv0VMi+88IKrSwLgQq+++qq2bt2qixcvqlKlSho5cqT69Onj6rJwFyIYAQAAmLiUBgAAYCIYAQAAmAhGAAAAJoIRAACAiWAEAABgIhgBKBSSkpL0r3/9y+nLWbNmDS8dBu5hBCMAhUJSUpL+/e9/53l8wzCUkZGR7+UQjIB7G88xAlAojB49WtHR0apVq5batGmjgwcPKikpSWlpaXrllVfUqVMnnTx5UsOGDVOTJk20b98+zZkzR8uWLdN3332nihUr6oEHHlDDhg01bNgw/f7773rnnXd08eJF3XfffZo0aZISExP1/PPPq2zZsvLw8NDMmTNVvXp1V686gALk7uoCACAvxowZo0OHDikyMlJpaWm6evWqypYtqwsXLqhv374KCgqSJB0/flxTpkxR06ZNtWfPHq1atUrfffedUlNT1bNnTzVs2FCS9NZbb+mdd95RzZo19dNPP+mdd97RggULFBgYqEcffVSPPfaYK1cXgIsQjAAUOoZhaPr06dq2bZuKFSsmm82mc+fOSZIefPBBNW3aVJK0c+dOBQUFqWTJkipZsqQ6duwoSbp06ZJ27dqlV155xT7PlJSUgl8RAHcdghGAQue///2vLly4oKVLl6p48eIKDAzUtWvXJEmlS5e+5fSGYcjT01ORkZHOLhVAIUPnawCFQpkyZXTp0iVJUnJysipVqqTixYsrLi5Of/zxR7bTNG/eXOvWrdO1a9d06dIlrV+/XpJUtmxZVa1aVT/88IOk60HpwIEDWZYD4N5DMAJQKFSoUEHNmzdXt27ddODAAe3du1fdu3dXZGSkateune00jRs3VmBgoB5//HE988wzslqt8vDwkCRNnTpVixcv1uOPP66uXbtqzZo1kqQuXbro888/V1hYmH7//fcCWz8AdwfuSgNQpF26dEllypTRlStX9NRTT2nSpEn2DtgAcDP6GAEo0iZOnKjDhw/r2rVr6tGjB6EIQK44YwQAAGCijxEAAICJYAQAAGAiGAEAAJgIRgAAACaCEQAAgOn/AUSAgcA9Eu1xAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 576x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5n__9t0c-VNN"
      },
      "source": [
        "!wget --quiet https://github.com/raw/tensorflow/models/master/official/nlp/bert/tokenization.py"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QXiM8Iek-m7K",
        "outputId": "34aafe9a-7edf-4dc3-bf39-785c18dd01ed"
      },
      "source": [
        "!pip install sentencepiece"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ac/aa/1437691b0c7c83086ebb79ce2da16e00bef024f24fec2a5161c35476f499/sentencepiece-0.1.96-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2MB)\n",
            "\u001b[K     |████████████████████████████████| 1.2MB 6.9MB/s \n",
            "\u001b[?25hInstalling collected packages: sentencepiece\n",
            "Successfully installed sentencepiece-0.1.96\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P5Jpreuo-b-U",
        "outputId": "e5c31b92-6ecf-4d99-ea1b-67492d45404f"
      },
      "source": [
        "import tensorflow_hub as hub\n",
        "import sentencepiece\n",
        "import tokenization\n",
        "module_url = 'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/2'\n",
        "bert_layer = hub.KerasLayer(module_url, trainable=True)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:absl:Using /tmp/tfhub_modules to cache modules.\n",
            "INFO:absl:Downloading TF-Hub Module 'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/2'.\n",
            "INFO:absl:Downloaded https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/2, Total size: 421.50MB\n",
            "INFO:absl:Downloaded TF-Hub Module 'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/2'.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 203
        },
        "id": "gDi2UZD2Ag5U",
        "outputId": "a9dc6018-d1d9-46db-ba6e-d6808559204b"
      },
      "source": [
        "def cleaned_text(text):\n",
        "    clean=re.sub(\"http\\S+\",\"\",text)\n",
        "    clean=re.sub(\"pic.twitter\\S+\",\"\",clean)\n",
        "    clean=re.sub(\"@\\S+\",\"\",clean)\n",
        "    clean = re.sub('#', '', clean)\n",
        "    clean = clean.lower()\n",
        "    return clean\n",
        "train_df['cleaned_text'] = train_df['text'].apply(cleaned_text)\n",
        "train_df.head()"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>keyword</th>\n",
              "      <th>location</th>\n",
              "      <th>text</th>\n",
              "      <th>target</th>\n",
              "      <th>cleaned_text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Our Deeds are the Reason of this #earthquake M...</td>\n",
              "      <td>1</td>\n",
              "      <td>our deeds are the reason of this earthquake ma...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>4</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Forest fire near La Ronge Sask. Canada</td>\n",
              "      <td>1</td>\n",
              "      <td>forest fire near la ronge sask. canada</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>5</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>All residents asked to 'shelter in place' are ...</td>\n",
              "      <td>1</td>\n",
              "      <td>all residents asked to 'shelter in place' are ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>6</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>13,000 people receive #wildfires evacuation or...</td>\n",
              "      <td>1</td>\n",
              "      <td>13,000 people receive wildfires evacuation ord...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>7</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Just got sent this photo from Ruby #Alaska as ...</td>\n",
              "      <td>1</td>\n",
              "      <td>just got sent this photo from ruby alaska as s...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   id keyword  ... target                                       cleaned_text\n",
              "0   1     NaN  ...      1  our deeds are the reason of this earthquake ma...\n",
              "1   4     NaN  ...      1             forest fire near la ronge sask. canada\n",
              "2   5     NaN  ...      1  all residents asked to 'shelter in place' are ...\n",
              "3   6     NaN  ...      1  13,000 people receive wildfires evacuation ord...\n",
              "4   7     NaN  ...      1  just got sent this photo from ruby alaska as s...\n",
              "\n",
              "[5 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 203
        },
        "id": "X1gtxCJaA5N4",
        "outputId": "42c6c074-45b8-4bdf-b732-b54432c7f921"
      },
      "source": [
        "def word_count(text):\n",
        "  text = text.split()\n",
        "  length = len(text)\n",
        "  return length\n",
        "\n",
        "train_df['Word Count'] = train_df['cleaned_text'].apply(word_count)\n",
        "train_df.head()"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>keyword</th>\n",
              "      <th>location</th>\n",
              "      <th>text</th>\n",
              "      <th>target</th>\n",
              "      <th>cleaned_text</th>\n",
              "      <th>Word Count</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Our Deeds are the Reason of this #earthquake M...</td>\n",
              "      <td>1</td>\n",
              "      <td>our deeds are the reason of this earthquake ma...</td>\n",
              "      <td>13</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>4</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Forest fire near La Ronge Sask. Canada</td>\n",
              "      <td>1</td>\n",
              "      <td>forest fire near la ronge sask. canada</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>5</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>All residents asked to 'shelter in place' are ...</td>\n",
              "      <td>1</td>\n",
              "      <td>all residents asked to 'shelter in place' are ...</td>\n",
              "      <td>22</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>6</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>13,000 people receive #wildfires evacuation or...</td>\n",
              "      <td>1</td>\n",
              "      <td>13,000 people receive wildfires evacuation ord...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>7</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Just got sent this photo from Ruby #Alaska as ...</td>\n",
              "      <td>1</td>\n",
              "      <td>just got sent this photo from ruby alaska as s...</td>\n",
              "      <td>16</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   id keyword  ...                                       cleaned_text Word Count\n",
              "0   1     NaN  ...  our deeds are the reason of this earthquake ma...         13\n",
              "1   4     NaN  ...             forest fire near la ronge sask. canada          7\n",
              "2   5     NaN  ...  all residents asked to 'shelter in place' are ...         22\n",
              "3   6     NaN  ...  13,000 people receive wildfires evacuation ord...          8\n",
              "4   7     NaN  ...  just got sent this photo from ruby alaska as s...         16\n",
              "\n",
              "[5 rows x 7 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 482
        },
        "id": "lCEyX-5XB-s8",
        "outputId": "123f9b94-81e3-4984-f966-90332942cb13"
      },
      "source": [
        "plt.style.use('ggplot')\n",
        "plt.figure(figsize=(14,6))\n",
        "sns.distplot(train_df['Word Count'],kde = False,color=\"red\")\n",
        "plt.title(\"Frequency distribution of number of words for each tweet\", size=20)"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/seaborn/distributions.py:2557: FutureWarning: `distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `histplot` (an axes-level function for histograms).\n",
            "  warnings.warn(msg, FutureWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'Frequency distribution of number of words for each tweet')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzUAAAGLCAYAAAD+s7zAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeXgUVcL24ac7CxA6hHSCILgAMh2WBEjYgmwStmFTRMURwQ30VRQE/VTAAd9RUQdFlM3l1UGRmRFcQEQGxmEVIewRhUFAEFlkSychAUK2+v5wuocmHbJVSCr53dfFpamuPnXqnKrqfrpOVdkMwzAEAAAAABZlL+8KAAAAAEBpEGoAAAAAWBqhBgAAAIClEWoAAAAAWBqhBgAAAIClEWoAAAAAWBqhBqhihg8frqioKJ9pmzZtUlRUlGbOnFkudTpy5IiioqI0fvx4n+njx49XVFSUjhw5Ui71ksq/bcw0b9489evXTy1btlRUVJQ++OCD8q6S6WbOnKmoqCht2rSpvKtimoyMDL344otKSEhQ8+bNFRUVpX//+9/lXS1TRUVFafjw4WVS9vfff6/7779fHTp0UFRUlG655ZYyWY6VJCQkKCEhobyrAZgqsLwrgIrt0i+/l3r55Zc1ePDgK1QbVFSff/65JkyYYMnt4ciRI+rRo4duvfVWvfLKK+VdnTLz1VdfacqUKWrevLnuvfdeBQcHq3Xr1uVdLRTB1KlTtWDBAnXv3l0333yzAgICFBkZWd7VsoSMjAz9z//8jy5cuKBbbrlF4eHhtJ3Jhg8frs2bN+vHH38s76qUWGVYBxBqUESPPfaY3+nNmjW7wjVBWWjZsqWWLVum8PDwcll+3bp1tWzZMoWGhpbL8i+nvNvGLKtXr5Ykvf3226pbt2451wbFsWbNGjVs2FBvv/12eVfFcnbu3Knk5GSNGzdODz/8cHlXB0AZItSgSEaPHl3eVUAZqlGjhm644YZyW35QUFC5Lv9yyrttzHLy5ElJItBY0MmTJ9WuXbvyroYlebb7q666qpxrAqCscU0NSu3i6yEOHjyosWPHqmPHjmratKnPuPZvvvlGDz74oDp06KDo6Gj17NlTf/7zn3XmzBm/5W7YsEFDhw5V69at1b59e40aNUo//fST3+ssCrvu4XLjh5cuXarhw4erbdu2iomJUd++fTVnzhxlZWXlm9cz7tvtdmvSpEnq3LmzoqOj1b9/f3322WcFttH69ev18MMPq2PHjoqOjla3bt30yCOPaMOGDd62iYqK0oQJE/y+PysrSx06dFCHDh381sufr776SoMHD1bLli3VsWNHPfXUUzpx4oTfeQtqv8OHD2vSpEnq1auXWrZsqfbt22vgwIGaPHmyUlJSJP122t5T7wkTJigqKsr7z9NHF1/n8OWXX+qOO+5QbGyst08KuqbGwzAMzZ07V7///e8VExOjrl276qWXXlJGRka+eS83Nv/SbWfmzJnq0aOHJGnRokU+df/8888v2zaS9PPPP+vpp59Wly5dFB0drc6dO+vpp5/Wzz//nG/ei9tg+fLluv3229WqVSu1b99e48aNK7BvCpKVlaV3331XAwcOVKtWrRQXF6ehQ4dq2bJlBS7X0z6ef4W5uL0+/vhjDRw4UDExMbrxxhs1adIkpaen53tPcdpf8u37X375RWPGjFGHDh0UGxurBx54QHv37pUkn30uJiZGt912mxITEy9b/0WLFmnQoEHefWDChAk6deqU33lTU1M1bdo09e3bVy1btlSbNm107733av369fnm/fzzz73byLp16zR8+HC1adOmSG0q/fZF+09/+pMSEhIUHR2t+Ph4PfbYY/rhhx985vNc/2YYhjZv3uztt8KuPbnzzjsVHR2tc+fO+UwfNmyYoqKiNHHiRJ/pP/30k6KiovT000/7TE9PT9e0adPUp08fxcTEqF27dhoxYoT3uHWxi/eTnTt36qGHHlL79u19+jsrK0uzZ89Wz549FR0drYSEBE2fPr3AY1pGRoZmz56tAQMGKC4uTrGxserZs6fGjh2br60u5dmunnnmGUm+xybPvi0VvS+k0vf7+fPn9c477+iWW25R69atFRsbqzvvvFNLly7NN29WVpbmz5+vBx98UN27d1d0dLTat2+v++67T2vXri1wGcePH9eLL76o3r17e4/Zt99+u2bPnu13/nPnzunPf/6zbrrpJkVHR6tXr1569913ZRhGoevjaePNmzdL8j22eLZRs7bFnJwc/fWvf9WQIUMUFxenVq1aadCgQZo/f77y8vL81u+7777TmDFj1KlTJ+/n7uTJk32OtUVZB1gHZ2pgml9++UVDhgxRw4YNNXDgQGVmZsrhcEiSZs2apZkzZ6p27dq66aab5HQ6tXfvXv3lL3/RunXrtGDBAu+8krR8+XKNGzdOQUFB6tevn+rUqaNt27bpD3/4Q5E/QIpiwoQJ+vzzz1WvXj317t1btWrVUlJSkt58801t3LhRc+fOVWCg725y5swZ3XXXXQoODlafPn2UlZWl5cuXa+LEibLb7br11lt95p8xY4Zmz56tkJAQ9ezZU1dffbVOnjypHTt2aMmSJbrxxhvVuXNnXXfddfrHP/6hiRMn5huGtWLFCqWmpuqBBx5QcHBwoev1wQcf6OWXX1atWrU0aNAghYaGav369brrrrt82vlyTp48qdtvv10ZGRnq2rWrevfurQsXLujIkSNasmSJhg0bpvDwcN16660KDQ3VypUr1aNHD58hibVq1fIpc+7cufr222/VvXt3dejQwe+XYn9eeuklbd26VX379vWuy4cffqitW7fq73//u6pVq1akci7Vvn173XPPPZo3b56aNm2qnj17el8rbGjlzp07df/99+vs2bNKSEhQkyZNdODAAS1ZskQrV67U3Llz1bJly3zv+9vf/qZVq1YpISFB7dq1086dO7Vs2TLt2bNHX3zxRZH6NysrSyNGjNDmzZvVuHFjDR06VJmZmVqxYoXGjRunPXv26IknnvCu42OPPaZFixbp6NGjBQ4lvZxXX31V69evV/fu3dWpUydt2rRJCxcu1KFDhzRv3rxil+fP0aNHdccdd+iGG27QrbfeqqNHj+rrr7/W8OHDtWDBAo0cOVIOh0N9+/ZVWlqali1bpgcffFArVqxQ/fr185X3wQcf6Ntvv1W/fv3UpUsXbdu2TZ9//rk2b96sTz75RE6n02fZw4cP19GjR9W2bVt16dJF58+f1+rVqzVy5Eg9//zzGjJkSL5lrFixQt988426du2qP/zhDzp27Fih63n48GENHTpUJ0+eVHx8vPr3769ff/1Vy5cv15o1azRz5kx1795dknTrrbeqffv2mjVrlho0aOA9tjRo0OCyy4iPj1dSUpK2bt2qrl27SvrtC3VSUpIk5QuDGzdulCR17NjRO81znNu/f79iYmJ07733KiUlRf/4xz/0wAMP6H//93/1hz/8Id+yk5KS9M4776hNmza67bbblJKSoqCgIBmGobFjx2rlypW67rrrNGzYMGVnZ+uzzz7zBteLGYahkSNHaseOHYqNjdUdd9yhgIAAnThxQps2bVLbtm0VHR1dYBvUqlVLjz32mP7973/nOzZ5/lucvrhYSfr9zJkzuvfee7V79261aNFCt912m/Ly8rR+/Xo9+eST2rdvn8aNG+edPy0tTVOmTFFsbKxuvPFGOZ1OnTp1SqtXr9ZDDz2kF198UXfccYfPMr7//nuNHDlSqampateunXr16qXMzEzt379fs2bN0qOPPuozf3Z2tkaMGKGTJ0+qa9euCggI0L/+9S9NmzZNWVlZhR4rPG3s79ji2UbN2Bazs7P18MMPa/369WrUqJEGDBigatWqadOmTXrhhRf03Xff6dVXX/Up59NPP9XkyZMVHByshIQE1atXT4cOHdInn3yiVatWaeHChapfv36R1gEWYgCX4XK5DJfLZcyYMSPfv88++8wwDMM4fPiwd75p06blK2Pjxo2Gy+Uy7rzzTiMtLc3ntc8++8xwuVzGlClTvNMyMjKM9u3bG82bNzd27tzpM/+UKVO8yzp8+LB3emJioree/nTv3t3o3r2732U/+uijxvnz531emzFjhuFyuYwPPvjAb3tMnDjRyMnJ8U7ft2+f0axZM6Nv374+83/zzTeGy+UyEhISjOPHj+er16+//ur9//fee89wuVzGRx99lG++YcOGGS6Xyzhw4IDf9bvY4cOHjRYtWhjt2rXzaaPc3Fzjscce867Dxfy137x58/y2gWEYxtmzZ33azNOWnm3iUp72bNWqlbFr1y6/dXa5XMYzzzzjM/2ZZ54xXC6X0b59e+PIkSN+12XWrFk+73G5XMawYcP81sNT3sXtUtCyPfy1TV5envH73//ecLlcxhdffOEz/1dffWW4XC6jT58+Rm5ubr42iI2NNfbs2ePznieeeMJwuVzGV1995bcOl3r77bcNl8tljBw50sjOzvZOP336tNG9e3fD5XIZ27Zt83mPZxsqDk97devWzTh69Kh3enZ2tjF06FDD5XIZ3333nc97Str+LpfLmDNnjs/8s2bNMlwul9GuXTtj0qRJPu25aNGifMcOw/hvO7do0SLftuY5fkyYMMFn+rBhw4yoqChj6dKlPtPT0tKMm2++2YiJiTFOnTrlne7Z3qOiooy1a9f6XdeCPPDAA37Xddu2bUazZs2M9u3bGxkZGT6vXa5N/dmwYYPhcrmMV155xTtt3bp1hsvlMu6//37D5XIZhw4d8r42atQow+VyGceOHfNOmzRpkuFyuYxJkyYZeXl53ukHDx404uLijBYtWvg9BrtcLuPvf/97vjotWbLEcLlcxpAhQ4zMzEzv9JSUFKNHjx751nHPnj2Gy+UyRo0ala+s3NxcIzU1tUhtcbljU3H7ojT97tn23333XZ/pmZmZxgMPPGBERUUZu3fv9k6/cOGCz+eDx5kzZ4z+/fsb7dq18zkGX7hwwbvvL1myJN/7Li3LM+/IkSN9yjl9+rTRpk0bo02bNkZWVlaR1u1yxxYztkXPPv3888/7fO7m5OQYEyZMMFwul/H11197px84cMBo0aKF0bNnz3yfuxs2bDCaNm2ab7sqyfERFQ/Dz1Aks2bNyvdv0aJFPvNERkb6/WXno48+kiS98MIL+X65Hzx4sJo1a6Yvv/zSO23lypVKTU3VgAEDFBMT4zP/6NGjTbuYfN68eQoMDNRLL72k6tWr+7w2atQo1a5d26deHjVq1NCECRMUEBDgndakSRPFxcXpp59+0tmzZ73T58+fL+m3YTf+rmWoV6+e9/8HDx6satWqacGCBT7zHDhwQJs3b1aHDh3UqFGjQtfryy+/VHZ2toYNG6ZrrrnGO91ut+vpp5+W3V683f7StpGkkJAQv9MLM2TIEDVv3rzY77vnnnt8fjW7eF0uN+yvrGzfvl0HDhxQbGysbr75Zp/X+vXrpzZt2ujgwYPatm1bvvf6u6W25xfX77//vkjL/+yzz2Sz2TR+/HifM4kRERF65JFHJEmffPJJsdbpch599FGfsyGBgYHeu9zt3LnTlGU0aNBADz30kM80z5mJrKysfNvuwIEDFRgYWOCtjW+++eZ825rn+LF06VLvkKc9e/Zo8+bN6t27t/r37+8zf61atTR69GhduHBBK1asyLeMHj16eH99Lorjx49r/fr1ql+/vkaOHOnzWlxcnPr376/U1FR9/fXXRS7Tn7i4OFWrVs3nV/CNGzcqMDDQe32k5xfxvLw8bd68WQ0bNtTVV18t6bf2XrJkiUJCQvTEE0/IZrN5y2nYsKGGDx+u7OxsLV68ON+ymzVr5vcMjmfI17hx43zOrNauXVujRo0qcF38HWfsdrvCwsIu2waFKU1fFLffU1JStGTJEkVHR+vBBx/0ea1atWp66qmnZBiGz+dNcHCwz+eDR2hoqG677TalpaX5HC9Wr16to0ePKiEhQQMHDsz3Pn9lSdIf//hHnzaOiIhQjx49lJ6eroMHDxZ5HQtS2m0xLy9P8+fPV506dfJ97gYEBGj8+PGy2Ww+bff3v/9d2dnZevbZZ/N97nbs2FEJCQlavXq13+HLsDaGn6FIinKbw6ZNm/odOpOUlKSgoCAtX75cy5cvz/d6dna23G63UlJSFB4ert27d0uS3wtjQ0ND1axZM+/415I6f/689uzZo/DwcH344Yd+5wkODtZPP/2Ub/r111/vdwiX50PjzJkzqlmzpqTf1t1ms6lLly6F1ik8PFx9+/bV4sWLtX37dsXFxUmSFi5cKEl+vyj4c7n2u/baa3X11Vfr6NGjhZaTkJCg119/Xc8//7zWr1+vzp07Ky4uTk2aNPH5klMc/oZjFUX79u3zTbt4Xc6cOZMvMJclTxt36NDB7+vx8fHatm2bdu/ena8fLg3qkrwf4GlpaYUuOyMjQ4cOHVLdunX93sAgPj5ekkx9jom/YT7FqXNRNGvWzOcLi/Tfi7sbNmyYb58LCAhQREREgdci+dtmLj5+/PTTT2rWrJl27Ngh6bd29XfdlNvtlvTbjwuXKu727Nlu2rRpo6CgoHyvx8fHa8mSJdq9e7cGDRpUrLIvVq1aNcXGxmrTpk3e42piYqJiYmIUGxuryMhIbdy4UXfeead27dqlM2fOqG/fvt73Hzx4UOfPn1dcXJxq167tt55vvfWW322soDbZvXu37Ha72rRpk+81f33VpEkTNWvWTEuXLtXRo0fVo0cPtWnTRtHR0UUaolmY0vRFcfv9+++/V25urmw2m99tLCcnR1L+bWzfvn16//33tWXLFp06dUoXLlzwef3ibd8znKs4YSs0NFTXX399vukXf5aVlhnbYmpqqho2bKi33nrL7zKqV6/u03aetti8ebPfH4qSk5OVm5urn3/++bJDGGE9hBqYpqB7/6empionJ0ezZs267PvPnTun8PBw73UWBZVnxjMGzpw5I8Mw5Ha7C63XpQr68uz5xTw3N9c7LT09XWFhYUU+qzF06FAtXrxYCxYsUFxcnLKysrRo0SJFRESoV69eRSqjKO1XlFDToEEDffrpp5o5c6a++eYb/fOf/5T025fZBx54QPfcc0+R6nPpsksiIiKiwPKOHj2q9PT0KxpqPG1c0B2V6tSp4zPfxfydafR8mS/ogteLeX5d9CzjUp46mfGFxKO0dS7pMjz7VEFnZwMDA71fCC91uW1G+m/fpKamSpK+/fZbffvttwXW79ILnS8uq6g8yyyo7y633RRXx44dlZiYqE2bNqljx47697//rf/5n/+R9NsX9g0bNsgwDL/XMBS1nv62sYLaxHMs9Bcg/C0nICBAH374oWbPnq0VK1botddekyTVrFlTt956q5544gnvj0clUZq+KG6/e7ax77///rJnYy8+y5+UlKR7771Xubm5io+PV0JCghwOh+x2u/c6oYtvsOCpZ3Hublicz7LSKM226Gm7n3/++bKf1Re3nec977///mXr5W+fhrURamCagn69dzgc3rv3FIXnC8zp06f9vu5vumdYSkFfcC79Jd/zq2/z5s3zDaMzU2hoqFJTU5WZmVmkYNOqVSs1b97ce8OAdevWKTU1VQ8++KDfLwMFLVP6rZ1+97vf5Xu9oHb154YbbtAbb7yhnJwc7dmzRxs2bND8+fM1ZcoU1ahRI9+FqoUp6Rme5ORkNW7cON90z7pc/KXXZrNddjswg2d5Bd1JyzO9qDdlKA5PmQX1o+cWtuX1zJ8r0f5FkZyc7Hf6pduM57/PPvtssYN6cbfnwo5tZm43njN2GzZskN1uV15envfLYnx8vJYuXao9e/YoMTFRNpvN56xjUevpbxsrqE1CQ0OVlpam7OzsfMeygvajsLAwTZw4URMnTtShQ4e0efNmLViwQPPnz9eZM2fyXRxeHKXpi5L2+3333VfgHS4v9dZbbykzM1Pz5s3Ld0b4nXfe0cqVK/0uo7h3UbwSzNgWe/XqVeQfID19tm3btjI5BqPi4poalLnWrVsrLS1N+/btK9L8nnHwW7Zsyfdaenq63yEPnsBy/PjxfK8dOnQo369tNWvW1O9+9zvt27fP+6tOWWjdurUMw9A333xT5PcMHTpUFy5c0OLFi7Vw4ULZbDbdeeedRX7/5drv8OHD+vXXX4tclkdgYKCio6P10EMP6fXXX5cknw9VT6g065e9S/kLxJ51adCggU9gDQsL87sd5Obmas+ePfmme844FKfunrsnFRTUPbdPbtGiRZHLLCqHw6HrrrtOJ06c8HvraM+yS3LtkhmK2/5lxV/feI4f1apV8w7da9WqlSRp69atZV4nT59s27bNb/Azc7uJiYmRw+FQYmKiEhMTVb16dcXGxkr675fMtWvXavv27YqKivK5G1yjRo1Uo0YN7dmzx28QLck21rx5c+Xl5fm9zqwoP3hdf/31uuOOOzR//nyFhITk+1JfXFeyL1q2bCm73V6sbezQoUOqXbu23yGu/tqrdevWkqR169aVvKIlVNjxvzTbYuPGjb13Jc3Ozi5SfTxtUZz2LuvPMFwZhBqUufvuu0+SNGnSJL+/Ip07d847Blb67SLMsLAwLV26NN+p+pkzZ/odDtC4cWM5HA6tXLnS5xfazMxMvfjiiwXWKzs7WxMnTvT7wZ2WlqZdu3YVaR0LMmzYMEnSK6+84nfd/U0bMGCAQkND9d5772nz5s3q1KmTrr322iIvc+DAgQoKCtL8+fN9ngeSl5enqVOnFnm40A8//OC3rT2/bF585ik8PFySShSYimLevHk+Q+YuXhfPBeseMTExOnbsWL7ni7z11lt+h93VqlVLNputWHVv06aNGjVqpG3btuW7Tmz58uXaunWrGjZs6Pf6ATPcdtttMgxDU6dO9fkQdrvdmjNnjnee8lDc9i8rnushLuY5fvTv3997XUZMTIzatm2rr7/+Wp9++qnfsn788ccCz/wUR7169dSpUycdPXo037V83333nZYuXaqwsDCfW4uXVEBAgNq1a6dDhw5p+fLlatOmjXedr732WjVo0EDz5s3T+fPnvV8sPYKDgzVw4ECdPXtWb775ps9rv/zyiz766CMFBQXplltuKXJ9PPvpG2+84XNtSGpqqt9rJQ4fPqzDhw/nm+4521OSG5Vc7Er2RUREhAYOHKgffvhBs2fP9vvF+ZdffvFZ3wYNGig1NTXfDwGffPKJ32cnde/eXQ0aNNCqVav8PvfG3w8NZvFcd1XQra1Lsy0GBgZq2LBhOnXqlF588UVlZmbmK//kyZPav3+/9++7775bQUFBevnll/3e7CArKytf4ClsHWANDD9DmevYsaOefPJJvf766+rTp4+6du2qa665RufOndOxY8e0ZcsWxcXFece/1qxZU88//7zGjRunu+++2+c5Nfv27VO7du3ynYUICgrSPffcozlz5mjQoEHq1auXcnJytGHDBl111VV+r324/fbbtWvXLv3tb39Tr1691LlzZ1199dVKS0vTkSNHtGXLFg0ePFjPP/98ide9c+fOeuSRR/TWW2+pb9++3ufUnD59Wtu2bVPr1q31yiuv+LynRo0aGjRokPeuccU5SyNJ11xzjZ588km98soruvXWW32e7ZKenq6oqKgi3fjhiy++0IIFC9SmTRtde+21CgsL0y+//KLVq1crODhY9957r3fe1q1bq0aNGvrwww+VmprqHXM+fPhwU4ZBxcXFadCgQT7rsmfPHrVo0SLf3YRGjBih9evXa9SoUerXr5/CwsK0Y8cOHTlyRO3bt8/3K2fNmjXVqlUrbd26VU8++aQaNWoku92uhIQENW3a1G99bDab/vznP+v+++/XuHHjtHTpUjVu3FgHDx7Uv/71L9WsWVNTp04t9p3miuqBBx7QunXrtHLlSt1yyy3q2rWrMjMztXz5ciUnJ2vkyJFq27ZtmSy7MMVt/7LSpUsX3XXXXerbt6/3+LFt2zY1aNBA/+///T+feadNm6Z7771Xzz77rD766CO1atVKoaGhOn78uPbu3au9e/dqwYIFBV6nUxx/+tOfdNddd2nq1Kn69ttvFR0d7X02it1u10svvWTakJn4+HitXr1aycnJ+b4sduzY0RviLr6GwePJJ5/U1q1bNX/+fH3//ffq0KGD9zk1Z8+e1aRJk4r1Y8uAAQO0bNkyrVq1SgMGDFCPHj2Uk5Oj5cuXKyYmRr/88ovP/D/++KMee+wxxcTE6IYbbtBVV10lt9utlStXKjs7O99+XxJXsi8mT56sQ4cOacaMGVqyZIni4uIUGRmpkydP6qefftL333+v119/3dumnge/Dh061Hvc++GHH7Rt2zb16dMn3934goOD9eabb2rEiBF68skntWDBArVq1UoXLlzQgQMHtHHjxnwh3ywdO3bU8uXLNXr0aHXr1k3VqlVT/fr1fW6wUJptcdSoUdqzZ48+/vhjrV69WvHx8apbt66Sk5N16NAhbd++XePGjVOTJk0k/TZsesqUKXr22Wc1YMAAdenSRQ0bNlROTo6OHTumbdu2KTw83OcHqaKsAyo+Qg2uiIceekhxcXH66KOPtG3bNq1atUoOh0N169bVkCFDNGDAAJ/5f//73ys0NFSzZs3SP/7xDwUHB6tt27b6+OOP9X//939+h1aNGTNGNWrU0MKFC7Vw4UJFRkaqX79+Gj16dL5btXo899xz6tq1qz7++GNt2LDBezHr1VdfrREjRuS7XW9JjB07VrGxsZo3b57WrFmjc+fOKSIiQtHR0QX+0nn77bfro48+Up06dZSQkFDsZd5///2qU6eO3n//fS1atEg1a9ZU586d9dRTT+X7QleQAQMGKCsrSzt27NCuXbuUmZmpunXrqn///rr//vvlcrm884aFhXkfMrpo0SLvBZg333yzKaFm4sSJ+vrrr7Vw4UIdPXpUtWvX1j333KPHH38834M3O3bsqNmzZ2v27Nn66quvFBISohtvvFHTp0/3e+chSZo6dapefvllrV+/Xl999ZUMw1C9evUKDDXSb8OWPv30U7311lvauHGjVq9erfDwcPXv31+jRo3yew2QWYKDgzV37lzNnTtXS5cu1fz58xUQEKCmTZtq4sSJ+fanK6kk7SPQU/UAACAASURBVF8W7rvvPvXq1Usffvihli1bppCQEA0ePFjjxo3LF07q1aunzz77TPPnz9c///lPffnll8rNzVVkZKSaNGmiYcOG+WzvpXHttdfqs88+05w5c7Ru3Tpt3rxZNWvWVJcuXfTwww+X+A6B/lz8BbGgL5KBgYF+A3Dt2rW1YMECvfPOO/r66681d+5cVa9eXS1bttSIESPUuXPnYtXFZrPpzTff1LvvvqtFixZp/vz5uuqqq3Tbbbfp0UcfzXdXQM9w182bN+ubb75RWlqanE6nWrRooeHDh6tbt27FWr4/V7IvHA6HPvroIy1cuFBLly7VP//5T124cEGRkZG6/vrrNWHCBN14443e+bt27aq3335bb731lpYtW6aAgAC1bNlS8+bN0+HDh/3eYjwmJkaLFy/Wu+++q3Xr1mnHjh2qWbOmrrvuOo0ZM8a0dbnUHXfcoWPHjumrr77Se++9p5ycHLVv394nEJRmWwwKCtKcOXP0xRdfaNGiRd7P0fDwcF1zzTV6/PHH893G+pZbblHTpk01d+5cbdq0SevXr1dISIiuuuoq9enTx+cOa0VdB1R8NsMwjPKuBFAc48eP16JFi7Ry5Uqf57BUJp9//rkmTJigRx55RGPHji3v6gAAAFRoXFMDVDA5OTmaO3euAgMDi/xsGgAAgKqM4WdABbF161Zt2bJFmzdv1t69ezVs2LACnwINAACA/yLUABXExo0bNWvWLNWuXVtDhgzRU089Vd5VAgAAsASuqQEAAABgaVxTAwAAAMDSCDUAAAAALK3CXFNTnKe4RkZGep9qjsqDfq286NvKiX6tnOjXyou+rZyqUr/Wr1+/wNc4UwMAAADA0gg1AAAAACyNUAMAAADA0gg1AAAAACyNUAMAAADA0gg1AAAAACyNUAMAAADA0gg1AAAAACyNUAMAAADA0gg1AAAAACyNUAMAAADA0gg1AAAAACyNUAMAAADA0gLLuwIAKq8At1t2t1t5J04oKD29VGXlOZ3KdTpNqhkAAKhMCDUAyozd7VZQYqICHQ4FZWSUqqzs+HhCDQAA8IvhZwAAAAAsjVADAAAAwNIINQAAAAAsrUjX1Dz66KOqXr267Ha7AgIC9MorrygjI0PTp0/XqVOnVKdOHY0bN04Oh0OGYWju3LnasWOHqlWrplGjRqlx48ZlvR4AAAAAqqgi3yjgueeeU61atbx/L168WDExMRo0aJAWL16sxYsXa9iwYdqxY4eOHz+uGTNmaN++fXrvvff00ksvlUnlAQAAAKDEw8+2bNmibt26SZK6deumLVu2SJK2bt2qrl27ymazyeVy6ezZs0pJSTGntgAAAABwiSKfqZkyZYokqVevXurZs6fS0tIUHh4uSapdu7bS0tIkSW63W5GRkd73RUREyO12e+ctyMXvKbTSgYHFmh/WQL9WPnknTijQ4ZDdbpfD4ShVWTmhobKzfVQo7LOVE/1aedG3lRP9+psihZoXXnhBTqdTaWlpevHFF1W/fn2f1202m2w2W6kqcvr06SLPGxkZWaz5YQ30a+UTlJ6uoIwMORwOZZT2OTXp6cpm+6hQ2GcrJ/q18qJvK6eq1K+XZpCLFSnUOP/zwLuwsDC1a9dO+/fvV1hYmFJSUhQeHq6UlBTv9TZOp9OnYZOTk73vBwAgwO2W3e02paw8p5OHsgIACg81mZmZMgxDNWrUUGZmpnbu3Knbb79dbdu21dq1azVo0CCtXbtW7dq1kyS1bdtWy5cvV6dOnbRv3z6FhIQUOvQMAFB12N1uBSUmmlJWdnw8oQYAUHioSUtL02uvvSZJys3NVefOndW6dWvdcMMNmj59ulatWuW9pbMkxcbGavv27RozZoyCg4M1atSosl0DAAAAAFVaoaGmbt26evXVV/NNDw0N1eTJk/NNt9lsGjlypDm1AwAAAIBClPiWzgAAAABQERBqAAAAAFgaoQYAAACApRFqAAAAAFgaoQYAAACApRFqAAAAAFgaoQYAAACApRFqAAAAAFgaoQYAAACApRFqAAAAAFhaYHlXAACsLMDtlt3tNqWsPKdTuU6nKWUBAFCVEGoAoBTsbreCEhNNKSs7Pp5QAwBACTD8DAAAAIClEWoAAAAAWBqhBgAAAIClEWoAAAAAWBqhBgAAAIClEWoAAAAAWBqhBgAAAICl8ZwaAIBl2TMzFbR/vyll8fBTALAuQg0AwLLsaWkKSEoypSwefgoA1sXwMwAAAACWRqgBAAAAYGmEGgAAAACWRqgBAAAAYGmEGgAAAACWRqgBAAAAYGmEGgAAAACWRqgBAAAAYGmEGgAAAACWRqgBAAAAYGmEGgAAAACWRqgBAAAAYGmEGgAAAACWRqgBAAAAYGmEGgAAAACWFljeFQAAVHwBbrfsbrfPtLwTJxSUnl7ssuwZGWZVCwAASYQaAEAR2N1uBSUm+kwLdDgUVIKAYjRqZFa1AACQRKgBUAX5O+tQUpx1AACg/BFqAFQ5/s46lBRnHQAAKH/cKAAAAACApXGmBgAqKYbZAQCqCkINAFRSDLMDAFQVDD8DAAAAYGmEGgAAAACWRqgBAAAAYGmEGgAAAACWRqgBAAAAYGmEGgAAAACWxi2dAViCPTNTQfv3m1MWz1wBAKBSIdQAsAR7WpoCkpJMKYtnrgAAULkUOdTk5eVp/PjxcjqdGj9+vE6ePKk33nhD6enpaty4sUaPHq3AwEBlZ2dr1qxZOnDggEJDQzV27FhdddVVZbkOAAAAAKqwIoeaZcuWqUGDBjp//rwkaf78+erfv786deqkd999V6tWrVLv3r21atUq1axZUzNnztS3336rv/71rxo3blyZrQBQ1QW43bK73aaVl+d0KtfpNK08AACAslakUJOcnKzt27dr8ODBWrp0qQzD0K5du/T4449Lkm666SZ98skn6t27t7Zu3ao77rhDkhQfH6+//OUvMgxDNput7NYCqMLsbreCEhNNKy87Pp5QAwAALKVIdz/74IMPNGzYMG8wSU9PV0hIiAICAiRJTqdT7v/8Uux2uxURESFJCggIUEhIiNLT08ui7gAAAABQ+Jmabdu2KSwsTI0bN9auXbvKrCKRkZFFnjcwMLBY88Ma6NeSyTtxQoEOh2nl5YSGym5SP3jqZrfb5ShlHbOrV1eQSetZUcsys+0lc7cNf+tZ0n6tKu1vVRyLKy/6tnKiX39TaKj58ccftXXrVu3YsUNZWVk6f/68PvjgA507d065ubkKCAiQ2+2W8z/DVZxOp5KTkxUREaHc3FydO3dOoaGhhVbk9OnTRa50ZGRkseaHNdCvJROUnq4gE29RnJ2ermyT+sFTN4fDoYxS1tHIzNQFk9azopZlZttL5m4b/tazpP1aVdrfqjgWV170beVUlfq1fv36Bb5W6PCzoUOH6u2339bs2bM1duxYRUdHa8yYMWrRooUS/zOOf82aNWrbtq0kqU2bNlqzZo0kKTExUS1atOB6GgAAAABlpkjX1Phz9913a+nSpRo9erQyMjKUkJAgSUpISFBGRoZGjx6tpUuX6u677zatsgAAAABwqWI9fLNFixZq0aKFJKlu3bp6+eWX880THBysJ554wpzaAQAAAEAhSnymBgAAAAAqAkINAAAAAEsj1AAAAACwNEINAAAAAEsj1AAAAACwNEINAAAAAEsj1AAAAACwNEINAAAAAEsj1AAAAACwNEINAAAAAEsLLO8KAABQEdgzMxW0f79p5eU5ncp1Ok0rDwBQMEINAACS7GlpCkhKMq287Ph4Qg0AXCEMPwMAAABgaYQaAAAAAJZGqAEAAABgaYQaAAAAAJbGjQKAchDgdsvudptSlj0jw5RyAAAArIpQA5QDu9utoMREU8oyGjUypRwAAACrYvgZAAAAAEsj1AAAAACwNEINAAAAAEvjmhoAACo4M28ukud0KtfpNKUsAKgoCDUAAFRwZt5cJDs+nlADoNJh+BkAAAAASyPUAAAAALA0Qg0AAAAASyPUAAAAALA0Qg0AAAAASyPUAAAAALA0Qg0AAAAASyPUAAAAALA0Qg0AAAAASyPUAAAAALA0Qg0AAAAASyPUAAAAALA0Qg0AAAAASyPUAAAAALA0Qg0AAAAASyPUAAAAALA0Qg0AAAAASyPUAAAAALA0Qg0AAAAASyPUAAAAALA0Qg0AAAAASyPUAAAAALA0Qg0AAAAASyPUAAAAALA0Qg0AAAAASyPUAAAAALA0Qg0AAAAASyPUAAAAALA0Qg0AAAAASwssbIasrCw999xzysnJUW5uruLj4zVkyBCdPHlSb7zxhtLT09W4cWONHj1agYGBys7O1qxZs3TgwAGFhoZq7Nixuuqqq67EugAAAACoggo9UxMUFKTnnntOr776qqZOnaqkpCTt3btX8+fPV//+/TVz5kzVrFlTq1atkiStWrVKNWvW1MyZM9W/f3/99a9/LfOVAAAAAFB1FRpqbDabqlevLknKzc1Vbm6ubDabdu3apfj4eEnSTTfdpC1btkiStm7dqptuukmSFB8frx9++EGGYZRR9QEAAABUdYUOP5OkvLw8PfPMMzp+/Lj69OmjunXrKiQkRAEBAZIkp9Mpt9stSXK73YqIiJAkBQQEKCQkROnp6apVq9ZllxEZGVn0SgcGFmt+WENV6te8EycU6HCYUlZ29eoKMqksScoJDZXdpH7wrKfdbpejlHU0cz0rallmtr1U9ttZSfu1ora/FfYlMxRWr6p0LK5q6NvKiX79TZFCjd1u16uvvqqzZ8/qtdde07Fjx0yvyOnTp4s8b2RkZLHmhzVUpX4NSk9XUEaGKWUZmZm6YFJZkpSdnq5sk/rBs54Oh0MZpayjmetZUcsys+2lst/OStqvFbX9rbAvmaGwelWlY3FVQ99WTlWpX+vXr1/ga8W6+1nNmjXVokUL7d27V+fOnVNubq6k387OOJ1OSb+dtUlOTpb023C1c+fOKTQ0tKR1BwAAAIDLKvRMzZkzZxQQEKCaNWsqKytLO3fu1C233KIWLVooMTFRnTp10po1a9S2bVtJUps2bbRmzRq5XC4lJiaqRYsWstlsZb4iQFkLcLtl/88wy9Kym/hrMAAAQFVXaKhJSUnR7NmzlZeXJ8Mw1LFjR7Vp00bXXHON3njjDX388cdq1KiREhISJEkJCQmaNWuWRo8eLYfDobFjx5b5SgBXgt3tVlBioillGY0amVIOAAAAihBqrr/+ek2dOjXf9Lp16+rll1/ONz04OFhPPPGEObUDAAAAgEIU65oaAAAAAKhoCDUAAAAALI1QAwAAAMDSCDUAAAAALI1QAwAAAMDSCDUAAAAALI1QAwAAAMDSCDUAAAAALI1QAwAAAMDSCDUAAAAALI1QAwAAAMDSCDUAAAAALI1QAwAAAMDSCDUAAAAALI1QAwAAAMDSCDUAAAAALC2wvCsAAACuHHtmpoL27y/w9bwTJxSUnl6ksvKcTuU6nWZVDQBKjFADAEAVYk9LU0BSUoGvBzocCsrIKFJZ2fHxhBoAFQLDzwAAAABYGqEGAAAAgKURagAAAABYGtfUAACAEinspgPFwU0HAJQGoQYAAJRIYTcdKA5uOgCgNBh+BgAAAMDSOFODSi3A7Zbd7TalLHsRb3EKAACAK4tQg0rN7nYrKDHRlLKMRo1MKQcAAADmYvgZAAAAAEsj1AAAAACwNEINAAAAAEsj1AAAAACwNEINAAAAAEsj1AAAAACwNEINAAAAAEsj1AAAAACwNEINAAAAAEsj1AAAAACwNEINAAAAAEsj1AAAAACwNEINAAAAAEsj1AAAAACwNEINAAAAAEsj1AAAAACwNEINAAAAAEsj1AAAAACwNEINAAAAAEsj1AAAAACwNEINAAAAAEsj1AAAAACwtMDyrgAAAIA9M1NB+/ebVl6e06lcp9O08gBUbIQaAABQ7uxpaQpISjKtvOz4eEINUIUw/AwAAACApRV6pub06dOaPXu2UlNTZbPZ1LNnT/Xr108ZGRmaPn26Tp06pTp16mjcuHFyOBwyDENz587Vjh07VK1aNY0aNUqNGze+EusCAAAAoAoq9ExNQECAhg8frunTp2vKlClasWKFjhw5osWLFysmJkYzZsxQTEyMFi9eLEnasWOHjh8/rhkzZuihhx7Se++9V+YrAQAAAKDqKjTUhIeHe8+01KhRQw0aNJDb7daWLVvUrVs3SVK3bt20ZcsWSdLWrVvVtWtX2Ww2uVwunT17VikpKWW4CgAAAACqsmJdU3Py5EkdPHhQTZo0UVpamsLDwyVJtWvXVlpamiTJ7XYrMjLS+56IiAi53W4TqwwAAAAA/1Xku59lZmZq2rRpuu+++xQSEuLzms1mk81mK1VFLg5ChQkMDCzW/ChbucePy5acXOpybKdPy5mbKyMiQgH16plQMynvxAkFOhymlJVdvbqCKnlZkpQTGiq7SfuXp/3tdrscpaxjRW0zM8sys+2lst/+S9qvFbX9rbAvmaGw9SxOv1aV9jfrc06SqZ9xxcX3p8qJfv1NkUJNTk6Opk2bpi5duqhDhw6SpLCwMKWkpCg8PFwpKSmqVauWJMnpdOr06dPe9yYnJ8tZhFsqXvyewkRGRhZrfpStoJ9/VlBiYqnLcTgcyszIUHZ8vLIDzbnbeFB6uoIyMkwpy8jM1IVKXpYkZaenK9uk/cvT/g6HQxmlrGNFbTMzyzKz7aWy3/5L2q8Vtf2tsC+ZobD1LE6/Vpn2N+lzTpKpn3HFxfenyqkq9Wv9+vULfK3Q4WeGYejtt99WgwYNNGDAAO/0tm3bau3atZKktWvXql27dt7p69atk2EY2rt3r0JCQrzD1AAAAADAbIX+VPDjjz9q3bp1uu666/TUU09Jku666y4NGjRI06dP16pVq7y3dJak2NhYbd++XWPGjFFwcLBGjRpVtmsAAAAAoEorNNQ0bdpUCxcu9Pva5MmT802z2WwaOXJk6WsGAAAAAEVQrLufAQAAAEBFQ6gBAAAAYGmEGgAAAACWRqgBAAAAYGmEGgAAAACWRqgBAAAAYGmEGgAAAACWRqgBAAAAYGmEGgAAAACWRqgBAAAAYGmEGgAAAACWRqgBAAAAYGmEGgAAAACWRqgBAAAAYGmEGgAAAACWRqgBAAAAYGmEGgAAAACWRqgBAAAAYGmEGgAAAACWRqgBAAAAYGmB5V0BAACAqsKemamg/ftNKSvP6VSu02lKWYDVEWoAAACuEHtamgKSkkwpKzs+nlAD/AfDzwAAAABYGqEGAAAAgKURagAAAABYGqEGAAAAgKURagAAAABYGqEGAAAAgKURagAAAABYGs+pqaIC3G7Z3W5TyrJnZJhSDgAAAFAShJoqyu52Kygx0ZSyjEaNTCkHAAAAKAmGnwEAAACwNEINAAAAAEsj1AAAAACwNEINAAAAAEsj1AAAAACwNEINAAAAAEsj1AAAAACwNEINAAAAAEsj1AAAAACwNEINAAAAAEsj1AAAAACwNEINAAAAAEsj1AAAAACwNEINAAAAAEsj1AAAAACwNEINAAAAAEsLLO8KoOgC3G7Z3W5TyrJnZJhSDgAAAFDeCDUWYne7FZSYaEpZRqNGppQDAAAAlDeGnwEAAACwNM7UAAAAWJA9M1NB+/cXef68EycUlJ7u/zWnU7lOp1lVA644Qg0AAIAF2dPSFJCUVOT5Ax0OBRVwTW12fDyhBpZWaKiZM2eOtm/frrCwME2bNk2SlJGRoenTp+vUqVOqU6eOxo0bJ4fDIcMwNHfuXO3YsUPVqlXTqFGj1Lhx4zJfCQAAAABVV6HX1Nx0002aOHGiz7TFixcrJiZGM2bMUExMjBYvXixJ2rFjh44fP64ZM2booYce0nvvvVc2tQYAAACA/yg01DRv3lwOh8Nn2pYtW9StWzdJUrdu3bRlyxZJ0tatW9W1a1fZbDa5XC6dPXtWKSkpZVBtAAAAAPhNia6pSUtLU3h4uCSpdu3aSktLkyS53W5FRkZ654uIiJDb7fbOezkXv68wgYGBxZq/ssg7cUKBlwTMksquXl1BFawsu90uh8OhnNBQ2U3q38reZmaXJalM2t/Tt6VRUdvMzLLMbHup7Lf/kvZrRW1/K+xLZihsPYvTr7R/8ZVnm12ub80+/uDKqarfiy9V6hsF2Gw22Wy2Ulfk9OnTRZ43MjKyWPNXFkHp6QVe4FdcRmamLlSwshwOhzIyMpSdnq5sk/q3sreZ2WVJKpP29/RtaVTUNjOzLDPbXir77b+k/VpR298K+5IZClvP4vQr7V985dlml+tbs48/uHKq0vfi+vXrF/haiZ5TExYW5h1WlpKSolq1akmSnE6nT6MmJyfLyZ00AAAAAJShEoWatm3bau3atZKktWvXql27dt7p69atk2EY2rt3r0JCQoo09AwAAAAASqrQ4WdvvPGGdu/erfT0dD388MMaMmSIBg0apOnTp2vVqlXeWzpLUmxsrLZv364xY8YoODhYo0aNKvMVAAAAAFC1FRpqxo4d63f65MmT802z2WwaOXJk6WsFAAAAAEVUouFnAAAAAFBREGoAAAAAWBqhBgAAAIClEWoAAAAAWBqhBgAAAIClEWoAAAAAWBqhBgAAAIClEWoAAAAAWFqhD98EAABA5WbPzFTQ/v2mlZfndCrX6TStPKAwhBoAAIAqzp6WpoCkJNPKy46PJ9TgimL4GQAAAABLI9QAAAAAsDRCDQAAAABLI9QAAAAAsDRuFFDGAtxu2d1uU8qyZ2SYUg4AAABQmRBqypjd7VZQYqIpZRmNGplSDgAAAFCZMPwMAAAAgKURagAAAABYGqEGAAAAgKURagAAAABYGqEGAAAAgKVx9zMAAABUWGY+HiPP6VSu02lKWahYCDUAAACosMx8PEZ2fDyhppJi+BkAAAAASyPUAAAAALA0Qg0AAAAASyPUAAAAALA0Qg0AAAAASyPUAAAAALA0Qg0AAAAASyPUAAAAALA0Qg0AAAAASyPUAAAAALA0Qg0AAAAASyPUAAAAALC0wPKuAAAAAHAl2DMzFbR/vyll5TmdynU6TSkLpUeoAQAAQJVgT0tTQFKSKWVlx8cTaioQQo0fAW637G63KWXZMzJMKQcAAACAf4QaP+xut4ISE00py2jUyJRyAAAAAPjHjQIAAAAAWBqhBgAAAIClEWoAAAAAWBqhBgAAAIClEWoAAAAAWBqhBgAAAIClEWoAAAAAWBqhBgAAAIClEWoAAAAAWFpgeVcAAAAAsBp7ZqaC9u83paw8p1O5TqcpZVVVhBoAAACgmOxpaQpISjKlrOz4eEJNKTH8DAAAAICllcmZmqSkJM2dO1d5eXnq0aOHBg0aVBaLAQAAAADzQ01eXp7ef/99/fGPf1RERIQmTJigtm3b6pprrjF7UQAAAIDlleb6nLwTJxSUnu47rQpeo2N6qNm/f7/q1aununXrSpJuvPFGbdmyhVADAAAA+FGa63MCHQ4FZWT4TMtt3Vp2t9uMqlkmIJkeatxutyIiIrx/R0REaN++fWYvBgAAAIAfVfEmBjbDMAwzC0xMTFRSUpIefvhhSdK6deu0b98+jRgxwszFAAAAAICkMrj7mdPpVHJysvfv5ORkOS2Q7gAAAABYk+mh5oYbbtCvv/6qkydPKicnRxs2bFDbtm3NXgwAAAAASCqD4WeStH37dn344YfKy8tT9+7dNXjwYLMXAQAAAACSyijUAAAAAMCVYvrwMwAAAAC4kgg1AAAAACzN9OfUlKWkpCTNnTtXeXl56tGjhwYNGlTeVYJJHn30UVWvXl12u10BAQF65ZVXyrtKKIE5c+Zo+/btCgsL07Rp0yRJGRkZmj59uk6dOqU6depo3Lhxcjgc5VxTFJe/vl24cKFWrlypWrVqSZLuuusuxcXFlWc1UUynT5/W7NmzlZqaKpvNpp49e6pfv37stxZXUL+yz1pfVlaWnnvuOeXk5Cg3N1fx8fEaMmSITp48qTfeeEPp6elq3LixRo8ercBAS33NLzXLXFOTl5enxx9/XH/84x8VERGhCRMm6PHHH9c111xT3lWDCR599FG9/PLL3gMtrGn37t2qXr26Zs+e7f3iO3/+fDkcDg0aNEiLFy9WRkaGhg0bVs41RXH569uFCxeqevXquvnmm8u5diiplJQUpaSkqHHjxjp//rzGjx+vp556SmvWrGG/tbCC+nXDhg3ssxZnGIYuXLig6tWrKycnR5MnT9Z9992npUuXqkOHDurUqZPeffddNWzYUL179y7v6l5Rlhl+tn//ftWrV09169ZVYGCgbrzxRm3ZsqW8qwXgIs2bN8/3a+6WLVvUrVs3SVK3bt3Yby3KX9/C+sLDw9W4cWNJUo0aNdSgQQO53W72W4srqF9hfTabTdWrV5ck5ebmKjc3VzabTbt27VJ8fLwk6aabbqqS+6xlzku53W5FRER4/46IiNC+ffvKsUYw25QpUyRJvXr1Us+ePcu5NjBLWlqawsPDJUm1a9dWWlpaOdcIZlqxYoXWrVunxo0b65577iH4WNjJkyd18OBBNWnShP22Erm4X/fs2cM+Wwnk5eXpmWee0fHjx9WnTx/VrVtXISEhCggIkCQ5nc4qGWItE2pQub3wwgtyOp1KS0vTiy++qPr166t58+blXS2YzGazyWazlXc1YJLevXvr9ttvlyQtWLBA8+bN06hRo8q5ViiJzMxMTZs2Tffdd59CQkJ8XmO/ta5L+5V9tnKw2+169dVXdfbsWb322ms6duxYeVepQrDM8DOn06nk5GTv38nJyXI6neVYI5jJ05dhYWFq166d9u/fX841glnCwsKUkpIi6bdx3lw3VXnUrl1bdrtddrtdPf5/O/cb0lTbxwH829y01NDNrUjLTNvcHzWw2R9yQti7gqTMqDDoj0WQ9qIXvQgjMIpASQ2hUMkICnyREQlBVg6yQlJQU4clzsrSGFtsU2ecq6HobwAABvtJREFUbc8LufewO8ezO+xZp/v7eXW8zr/f2eHC8+Vc5yosxOjoaKRLop8gCAJqampgMpmwefNmAOy3f4KF7iv77J8lLi4OBoMBIyMjmJmZgdfrBTA/uunf+IwsmlCTkZGBL1++4OvXrxAEAS9fvoTRaIx0WbQIPB4PZmdnA8v9/f1ITU2NcFW0WIxGI8xmMwDAbDYjLy8vwhXRYvnroRcAuru7sWbNmghWQz/D7/fjxo0bSElJwa5duwLt7LfiFuq+ss+Kn9PpxPT0NID5mdD6+/uRkpICg8GA169fAwA6Ozv/lc/Iopn9DAB6e3tx+/Zt+Hw+bN++HXv27Il0SbQIpqamUF1dDWD+o7f8/HzeW5Gqra3F0NAQXC4XEhISUFJSgry8PFy7dg02m41Tw4rYQvd2cHAQVqsVS5YsgUqlwokTJwLfYZA4WCwWXLhwAampqYEhZgcOHIBarWa/FbFQ97Wrq4t9VuTGx8fR0NAAn88Hv9+PrVu3ori4GFNTU6itrYXb7ca6detQXl4OmUwW6XL/r0QVaoiIiIiIiP5ONMPPiIiIiIiIFsJQQ0REREREosZQQ0REREREosZQQ0REREREosZQQ0REREREosZQQ0REEdPa2or6+vpIl0FERCLHUENERAFtbW24fPlyUFtFRcWCbV1dXb+8npmZGbS0tODUqVMoLS1FeXk5Wlpa4HQ6f+l5Ozs7UVlZ+UvPQUREi0ca6QKIiOj3odPp8ODBA/h8PkgkEjgcDni9XoyNjQW1TU5OQqfT/aNje71eREVFhb29IAioqqpCbGwszp8/j+TkZLhcLjx58gTv379Hbm7uP708IiL6QzHUEBFRwPr16+H1emG1WpGeno7h4WEYDAZMTU0Fta1cuRIKhQJ2ux2NjY2wWCyIj4/H7t27sWPHDgDzQ8s+fvwImUyGnp4eHD58GNnZ2WhoaMDY2BjUajWSk5ND1mI2m2Gz2XD9+nUsXboUAJCQkIDi4uLANp8+fUJTUxOsVisUCgUOHjwIo9EIALh48SJMJhMKCwsBzL99efr0KaqqqgAAJSUlOH78OB49egSn04n8/HwcO3YMExMTaGxshCAIKC0tRVRUFFpaWn7Fz01ERIuEw8+IiChAKpVCrVZjaGgIADA8PAytVgutVhvU9tdbmrq6OiQlJeHmzZs4e/Ys7t27h7dv3waO9+bNG2zZsgW3bt2CyWRCXV0d0tPT0dzcjL1798JsNoesZWBgABs2bAgEmr8TBAFXr15FTk4OmpqacPToUdTX1+Pz589hX29vby+uXLmC6upqvHr1Cn19fVi9ejXKysqg0Whw584dBhoiIhFgqCEioiA6nQ7Dw8MAAIvFAp1O90ObXq+HzWaDxWLBoUOHEB0djbS0NBQWFgYFFY1Gg02bNkEikcDpdGJ0dBT79++HTCaDXq/Hxo0bQ9bhcrkgl8tDrn/37h08Hg+KiooglUqRlZWF3NxcvHjxIuxrLSoqQlxcHJRKJQwGA6xWa9j7EhHR74OhhoiIguj1elgsFrjdbjidTqxatQoajQYjIyNwu9348OED9Ho9HA4H4uPjsWzZssC+SqUSdrs98HdSUlJg2W63Iy4uLujNi0qlClnH8uXL4XA4Qq53OBxQKpWQSP77r0ylUgWd/39JTEwMLMfExMDj8YS9LxER/T4YaoiIKIhGo8HMzAw6OjqQmZkJAIiNjYVcLkdHRwcUCgVWrFgBuVwOt9uN2dnZwL42mw0KhWLB48rlckxPTwcFB5vNFrKO7Oxs9PX1hQwacrkcNpsNPp9vwfPHxMRgbm4usO7bt29hXD0REYkRQw0REQWJjo5GRkYG2tvbodVqA+1arRbt7e2B72mUSiUyMzNx9+5dfP/+HePj43j+/DlMJtOCx1WpVMjIyEBraysEQYDFYkFPT0/IOgoKCqBUKlFTU4OJiQn4fD64XC7cv38fvb29UKvViImJwcOHDyEIAgYHB9HT04Nt27YBANLS0tDd3Y25uTlMTk7i2bNnYf8GiYmJsNvtEAQh7H2IiChyOPsZERH9QK/XY2Rk5IdQ8/jx46CpnM+cOYPGxkacPHkS8fHx2LdvH3JyckIet6KiAg0NDThy5Ag0Gg0KCgowPT294LYymQyVlZVobW3FpUuX4Ha7kZiYCKPRCLVaDalUinPnzqGpqQltbW1QKBQ4ffo0UlJSAAA7d+7E6OgoysrKsHbtWuTn52NgYCCs68/KygpMGCCRSNDc3BzWfkREFBlL/H6/P9JFEBERERER/SwOPyMiIiIiIlFjqCEiIiIiIlFjqCEiIiIiIlFjqCEiIiIiIlFjqCEiIiIiIlFjqCEiIiIiIlFjqCEiIiIiIlFjqCEiIiIiIlFjqCEiIiIiIlH7DyvC1KToe4SVAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 1008x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HOLNBo8Q-3Ol"
      },
      "source": [
        "vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()\n",
        "do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()\n",
        "tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LKUCj07FCaXI"
      },
      "source": [
        "def bert_encode(texts, tokenizer, max_len=100):\n",
        "    all_tokens = []\n",
        "    all_masks = []\n",
        "    all_segments = []\n",
        "    \n",
        "    for text in texts:\n",
        "        text = tokenizer.tokenize(text)\n",
        "            \n",
        "        text = text[:max_len-2]\n",
        "        input_sequence = [\"[CLS]\"] + text + [\"[SEP]\"]\n",
        "        pad_len = max_len - len(input_sequence)\n",
        "        \n",
        "        tokens = tokenizer.convert_tokens_to_ids(input_sequence) + [0] * pad_len\n",
        "        pad_masks = [1] * len(input_sequence) + [0] * pad_len\n",
        "        segment_ids = [0] * max_len\n",
        "        \n",
        "        all_tokens.append(tokens)\n",
        "        all_masks.append(pad_masks)\n",
        "        all_segments.append(segment_ids)\n",
        "    \n",
        "    return np.array(all_tokens), np.array(all_masks), np.array(all_segments)"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ciGKMfuq66OA",
        "outputId": "e57f1a44-df01-49f8-c2c0-322c03b6b7ef"
      },
      "source": [
        "print(bert_encode(train_df['cleaned_text'][0], tokenizer)[0])"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[ 101 1051  102 ...    0    0    0]\n",
            " [ 101 1057  102 ...    0    0    0]\n",
            " [ 101 1054  102 ...    0    0    0]\n",
            " ...\n",
            " [ 101 1037  102 ...    0    0    0]\n",
            " [ 101 1048  102 ...    0    0    0]\n",
            " [ 101 1048  102 ...    0    0    0]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nQfJvDqKCspT"
      },
      "source": [
        "def build_model(bert_layer, max_len=100):\n",
        "    input_word_ids = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n",
        "    input_mask = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name=\"input_mask\")\n",
        "    segment_ids = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name=\"segment_ids\")\n",
        "\n",
        "    pooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])\n",
        "    clf_output = sequence_output[:, 0, :]\n",
        "    net = tf.keras.layers.Dense(64, activation='relu')(clf_output)\n",
        "    net = tf.keras.layers.Dropout(0.2)(net)\n",
        "    net = tf.keras.layers.Dense(32, activation='relu')(net)\n",
        "    net = tf.keras.layers.Dropout(0.2)(net)\n",
        "    out = tf.keras.layers.Dense(1, activation='sigmoid')(net)\n",
        "    \n",
        "    model = tf.keras.models.Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)\n",
        "    model.compile(tf.keras.optimizers.Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])\n",
        "    \n",
        "    return model"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "haNpmPl-Fyyh"
      },
      "source": [
        "from tensorflow import keras"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o92LrDlVEYaY"
      },
      "source": [
        "max_len = 100\n",
        "train_input = bert_encode(train_df.cleaned_text.values, tokenizer, max_len=max_len)\n",
        "# test_input = bert_encode(test.Review.values, tokenizer, max_len=max_len)\n",
        "train_labels = train_df['target']"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xVlXZLmhFq8q",
        "outputId": "67d524dc-12da-4309-c89f-b23354d91ff2"
      },
      "source": [
        "model = build_model(bert_layer, max_len=max_len)\n",
        "model.summary()"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_word_ids (InputLayer)     [(None, 100)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "input_mask (InputLayer)         [(None, 100)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "segment_ids (InputLayer)        [(None, 100)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "keras_layer (KerasLayer)        [(None, 768), (None, 109482241   input_word_ids[0][0]             \n",
            "                                                                 input_mask[0][0]                 \n",
            "                                                                 segment_ids[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "tf.__operators__.getitem (Slici (None, 768)          0           keras_layer[0][1]                \n",
            "__________________________________________________________________________________________________\n",
            "dense (Dense)                   (None, 64)           49216       tf.__operators__.getitem[0][0]   \n",
            "__________________________________________________________________________________________________\n",
            "dropout (Dropout)               (None, 64)           0           dense[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "dense_1 (Dense)                 (None, 32)           2080        dropout[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dropout_1 (Dropout)             (None, 32)           0           dense_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dense_2 (Dense)                 (None, 1)            33          dropout_1[0][0]                  \n",
            "==================================================================================================\n",
            "Total params: 109,533,570\n",
            "Trainable params: 109,533,569\n",
            "Non-trainable params: 1\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/optimizer_v2/optimizer_v2.py:375: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  \"The `lr` argument is deprecated, use `learning_rate` instead.\")\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xDrzVdyzXbsE",
        "outputId": "1bd973f4-5e85-4e5e-e309-c61b7e0c2215"
      },
      "source": [
        "train_history = model.fit(\n",
        "    train_input, train_labels, \n",
        "    validation_split=0.2,\n",
        "    epochs=5,\n",
        "    batch_size=32,\n",
        "    verbose=1)"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/5\n",
            "191/191 [==============================] - 91s 409ms/step - loss: 0.5046 - accuracy: 0.7640 - val_loss: 0.4218 - val_accuracy: 0.8260\n",
            "Epoch 2/5\n",
            "191/191 [==============================] - 78s 406ms/step - loss: 0.3761 - accuracy: 0.8521 - val_loss: 0.3813 - val_accuracy: 0.8365\n",
            "Epoch 3/5\n",
            "191/191 [==============================] - 78s 406ms/step - loss: 0.3218 - accuracy: 0.8834 - val_loss: 0.4224 - val_accuracy: 0.8359\n",
            "Epoch 4/5\n",
            "191/191 [==============================] - 78s 407ms/step - loss: 0.2711 - accuracy: 0.9044 - val_loss: 0.4368 - val_accuracy: 0.8332\n",
            "Epoch 5/5\n",
            "191/191 [==============================] - 78s 407ms/step - loss: 0.2229 - accuracy: 0.9227 - val_loss: 0.4859 - val_accuracy: 0.8293\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wksPecP1Zl4A"
      },
      "source": [
        "model.save('/content/drive/MyDrive/Disaster tweets prediction/disaster_model.h5')"
      ],
      "execution_count": 28,
      "outputs": []
    }
  ]
}