bert.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f8ed0fef",
   "metadata": {},
   "source": [
    "## Bert Tokanizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c71add18",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-29T19:02:35.883503Z",
     "start_time": "2021-12-29T19:02:35.870434Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.model_selection import train_test_split\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eb1e5f2f",
   "metadata": {},
   "source": [
    "### Preprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "fa2c8205",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-29T19:02:46.565977Z",
     "start_time": "2021-12-29T19:02:40.970980Z"
    }
   },
   "outputs": [],
   "source": [
    "from transformers import BertTokenizer, TFBertForSequenceClassification\n",
    "from transformers import InputExample, InputFeatures\n",
    "# model = TFBertForSequenceClassification.from_pretrained(\"bert-base-uncased\")\n",
    "tokenizer = BertTokenizer.from_pretrained(\"bert-base-uncased\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "1edf1fae",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-29T19:05:16.532254Z",
     "start_time": "2021-12-29T19:05:15.111184Z"
    }
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"Data/malicious_phish.csv\").sample(100000)\n",
    "df[\"type\"] = pd.Categorical(df[\"type\"]).codes\n",
    "X = df[\"url\"]\n",
    "y = df[\"type\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "38971e1c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-29T19:05:18.499959Z",
     "start_time": "2021-12-29T19:05:18.471697Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((80000,), (20000,), (80000,), (20000,))"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train,X_test,y_train,y_test = train_test_split(X.values,y.values,test_size=0.2,random_state=42)\n",
    "X_train.shape,X_test.shape,y_train.shape,y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "7c633b4e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-29T19:33:50.364567Z",
     "start_time": "2021-12-29T19:33:50.304065Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "405d4855b49f45768b649fa5ee66e3a7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Pandas Apply:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[101,\n",
       " 2599,\n",
       " 23632,\n",
       " 2487,\n",
       " 1012,\n",
       " 4012,\n",
       " 1013,\n",
       " 2982,\n",
       " 1013,\n",
       " 2249,\n",
       " 1013,\n",
       " 2047,\n",
       " 1011,\n",
       " 2259,\n",
       " 1011,\n",
       " 2103,\n",
       " 1012,\n",
       " 16129,\n",
       " 102]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import swifter\n",
    "df[\"url\"][:1].swifter.apply(lambda text: tokenizer.encode_plus(\n",
    "            text,\n",
    "            add_special_tokens=True,max_length=128,\n",
    "            return_token_type_ids=False,return_attention_mask=False,\n",
    "            padding=True,truncation=True\n",
    "        )[\"input_ids\"])[305990]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "ea6037f5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-29T19:25:36.640985Z",
     "start_time": "2021-12-29T19:25:19.707107Z"
    }
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Attempt to convert a value ({'input_ids': array([  101,  2998,  2850,  2696,  1012,  1047,  9189,  2226,  1012,\n        4012,  1013,  2374,  1013,  5088,  1011,  2780,  1012,  2004,\n        2361,  2595,  1029,  3931,  1027,  1013,  2951,  1013,  5088,\n        1013,  2780,  1013,  2136, 23833,  1012, 16129,   102]), 'attention_mask': array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]), 'token_type_ids': array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])}) with an unsupported type (<class 'dict'>) to a Tensor.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-29-e5acba22cedd>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m# validation_data = convert_examples_to_tf_dataset(list(validation_InputExamples), tokenizer)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[0mdataset_to_numpy\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalidation_data\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_numpy_iterator\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0mshape\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdataset_to_numpy\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;31m# validation_data = validation_data.batch(32)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\util\\dispatch.py\u001b[0m in \u001b[0;36mwrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    204\u001b[0m     \u001b[1;34m\"\"\"Call target, and fall back on dispatchers if there is a TypeError.\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    205\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 206\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mtarget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    207\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mTypeError\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    208\u001b[0m       \u001b[1;31m# Note: convert_to_eager_tensor currently raises a ValueError, not a\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\ops\\array_ops.py\u001b[0m in \u001b[0;36mshape_v2\u001b[1;34m(input, out_type, name)\u001b[0m\n\u001b[0;32m    622\u001b[0m     \u001b[0mA\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;31m`\u001b[0m \u001b[0mof\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0mout_type\u001b[0m\u001b[0;31m`\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    623\u001b[0m   \"\"\"\n\u001b[1;32m--> 624\u001b[1;33m   \u001b[1;32mreturn\u001b[0m \u001b[0mshape\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mout_type\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    625\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    626\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\util\\dispatch.py\u001b[0m in \u001b[0;36mwrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    204\u001b[0m     \u001b[1;34m\"\"\"Call target, and fall back on dispatchers if there is a TypeError.\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    205\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 206\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mtarget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    207\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mTypeError\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    208\u001b[0m       \u001b[1;31m# Note: convert_to_eager_tensor currently raises a ValueError, not a\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\ops\\array_ops.py\u001b[0m in \u001b[0;36mshape\u001b[1;34m(input, name, out_type)\u001b[0m\n\u001b[0;32m    649\u001b[0m     \u001b[0mA\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;31m`\u001b[0m \u001b[0mof\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0mout_type\u001b[0m\u001b[0;31m`\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    650\u001b[0m   \"\"\"\n\u001b[1;32m--> 651\u001b[1;33m   \u001b[1;32mreturn\u001b[0m \u001b[0mshape_internal\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptimize\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mout_type\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mout_type\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    652\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    653\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\ops\\array_ops.py\u001b[0m in \u001b[0;36mshape_internal\u001b[1;34m(input, name, optimize, out_type)\u001b[0m\n\u001b[0;32m    677\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0moptimize\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0minput_shape\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_fully_defined\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    678\u001b[0m           \u001b[1;32mreturn\u001b[0m \u001b[0mconstant\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput_shape\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_list\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mout_type\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 679\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mgen_array_ops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mout_type\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mout_type\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    680\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    681\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\ops\\gen_array_ops.py\u001b[0m in \u001b[0;36mshape\u001b[1;34m(input, out_type, name)\u001b[0m\n\u001b[0;32m   9188\u001b[0m       \u001b[1;32mpass\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   9189\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 9190\u001b[1;33m       return shape_eager_fallback(\n\u001b[0m\u001b[0;32m   9191\u001b[0m           input, out_type=out_type, name=name, ctx=_ctx)\n\u001b[0;32m   9192\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0m_core\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_SymbolicException\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\ops\\gen_array_ops.py\u001b[0m in \u001b[0;36mshape_eager_fallback\u001b[1;34m(input, out_type, name, ctx)\u001b[0m\n\u001b[0;32m   9215\u001b[0m     \u001b[0mout_type\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_dtypes\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mint32\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   9216\u001b[0m   \u001b[0mout_type\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_execute\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmake_type\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mout_type\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"out_type\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 9217\u001b[1;33m   \u001b[0m_attr_T\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_execute\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0margs_to_matching_eager\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mctx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   9218\u001b[0m   \u001b[0m_inputs_flat\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   9219\u001b[0m   \u001b[0m_attrs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m(\u001b[0m\u001b[1;34m\"T\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_attr_T\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"out_type\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mout_type\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\execute.py\u001b[0m in \u001b[0;36margs_to_matching_eager\u001b[1;34m(l, ctx, allowed_dtypes, default_dtype)\u001b[0m\n\u001b[0;32m    271\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    272\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mtensor\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 273\u001b[1;33m         tensor = ops.convert_to_tensor(\n\u001b[0m\u001b[0;32m    274\u001b[0m             t, dtype, preferred_dtype=default_dtype, ctx=ctx)\n\u001b[0;32m    275\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\profiler\\trace.py\u001b[0m in \u001b[0;36mwrapped\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    161\u001b[0m         \u001b[1;32mwith\u001b[0m \u001b[0mTrace\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrace_name\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mtrace_kwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    162\u001b[0m           \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 163\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    164\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    165\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mwrapped\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\framework\\ops.py\u001b[0m in \u001b[0;36mconvert_to_tensor\u001b[1;34m(value, dtype, name, as_ref, preferred_dtype, dtype_hint, ctx, accepted_result_types)\u001b[0m\n\u001b[0;32m   1564\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1565\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mret\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1566\u001b[1;33m       \u001b[0mret\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mconversion_func\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mas_ref\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mas_ref\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1567\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1568\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mret\u001b[0m \u001b[1;32mis\u001b[0m \u001b[0mNotImplemented\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\framework\\constant_op.py\u001b[0m in \u001b[0;36m_constant_tensor_conversion_function\u001b[1;34m(v, dtype, name, as_ref)\u001b[0m\n\u001b[0;32m    344\u001b[0m                                          as_ref=False):\n\u001b[0;32m    345\u001b[0m   \u001b[0m_\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mas_ref\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 346\u001b[1;33m   \u001b[1;32mreturn\u001b[0m \u001b[0mconstant\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    347\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    348\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\framework\\constant_op.py\u001b[0m in \u001b[0;36mconstant\u001b[1;34m(value, dtype, shape, name)\u001b[0m\n\u001b[0;32m    269\u001b[0m     \u001b[0mValueError\u001b[0m\u001b[1;33m:\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mcalled\u001b[0m \u001b[0mon\u001b[0m \u001b[0ma\u001b[0m \u001b[0msymbolic\u001b[0m \u001b[0mtensor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    270\u001b[0m   \"\"\"\n\u001b[1;32m--> 271\u001b[1;33m   return _constant_impl(value, dtype, shape, name, verify_shape=False,\n\u001b[0m\u001b[0;32m    272\u001b[0m                         allow_broadcast=True)\n\u001b[0;32m    273\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\framework\\constant_op.py\u001b[0m in \u001b[0;36m_constant_impl\u001b[1;34m(value, dtype, shape, name, verify_shape, allow_broadcast)\u001b[0m\n\u001b[0;32m    281\u001b[0m       \u001b[1;32mwith\u001b[0m \u001b[0mtrace\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTrace\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"tf.constant\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    282\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0m_constant_eager_impl\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mctx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mshape\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mverify_shape\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 283\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0m_constant_eager_impl\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mctx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mshape\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mverify_shape\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    284\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    285\u001b[0m   \u001b[0mg\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_default_graph\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\framework\\constant_op.py\u001b[0m in \u001b[0;36m_constant_eager_impl\u001b[1;34m(ctx, value, dtype, shape, verify_shape)\u001b[0m\n\u001b[0;32m    306\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0m_constant_eager_impl\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mctx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mshape\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mverify_shape\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    307\u001b[0m   \u001b[1;34m\"\"\"Creates a constant on the current device.\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 308\u001b[1;33m   \u001b[0mt\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mconvert_to_eager_tensor\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mctx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    309\u001b[0m   \u001b[1;32mif\u001b[0m \u001b[0mshape\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    310\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mt\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\framework\\constant_op.py\u001b[0m in \u001b[0;36mconvert_to_eager_tensor\u001b[1;34m(value, ctx, dtype)\u001b[0m\n\u001b[0;32m    104\u001b[0m       \u001b[0mdtype\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdtypes\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_dtype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_datatype_enum\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    105\u001b[0m   \u001b[0mctx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 106\u001b[1;33m   \u001b[1;32mreturn\u001b[0m \u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mEagerTensor\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mctx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdevice_name\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    107\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    108\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Attempt to convert a value ({'input_ids': array([  101,  2998,  2850,  2696,  1012,  1047,  9189,  2226,  1012,\n        4012,  1013,  2374,  1013,  5088,  1011,  2780,  1012,  2004,\n        2361,  2595,  1029,  3931,  1027,  1013,  2951,  1013,  5088,\n        1013,  2780,  1013,  2136, 23833,  1012, 16129,   102]), 'attention_mask': array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]), 'token_type_ids': array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])}) with an unsupported type (<class 'dict'>) to a Tensor."
     ]
    }
   ],
   "source": [
    "# validation_data = convert_examples_to_tf_dataset(list(validation_InputExamples), tokenizer)\n",
    "dataset_to_numpy = list(validation_data.as_numpy_iterator())\n",
    "shape = tf.shape(dataset_to_numpy)\n",
    "print(shape)\n",
    "# validation_data = validation_data.batch(32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "b1ed9c29",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-29T19:27:45.427156Z",
     "start_time": "2021-12-29T19:27:45.414150Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(35,)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_to_numpy[0][0][\"input_ids\"].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "2115321b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-29T19:28:18.013017Z",
     "start_time": "2021-12-29T19:28:17.993023Z"
    }
   },
   "outputs": [],
   "source": [
    "validation_data = validation_data.batch(32)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c019cc22",
   "metadata": {},
   "source": [
    "### Model\n",
    "we will not finetune the bert classifier at first"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "6318009a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-29T19:13:18.634899Z",
     "start_time": "2021-12-29T19:13:18.625908Z"
    }
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import Dense\n",
    "from tensorflow.keras.models import Sequential"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0d88110",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(20),input_shape=(35,))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a6515b5d",
   "metadata": {},
   "source": [
    "### Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bcc3de56",
   "metadata": {},
   "outputs": [],
   "source": [
    "test = pd.DataFrame({DATA_COLUMN:x_test,LABEL_COLUMN:y_test})\n",
    "test_InputExamples = test.apply(lambda x: InputExample(guid=None, # Globally unique ID for bookkeeping, unused in this case\n",
    "                                                          text_a = x[DATA_COLUMN], \n",
    "                                                          text_b = None,\n",
    "                                                          label = x[LABEL_COLUMN]), axis = 1)\n",
    "test_data = convert_examples_to_tf_dataset(list(test_InputExamples), tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "311da5b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "pred = model.predict(test_data)\n",
    "print(classification_report(y_test,pred))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}