BertLayers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "from transformers import BertTokenizer, BertModel\n",
    "from torch.nn.utils.rnn import pad_sequence\n",
    "from torch import nn\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomTokenizer:\n",
    "    def __init__(self, vocabulary):\n",
    "        self.vocabulary = vocabulary\n",
    "        self.tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
    "        \n",
    "    def tokenize(self, sentences : list):\n",
    "        encoded = []\n",
    "        for sentence in sentences:\n",
    "            tokens = self.tokenizer.tokenize(sentence)\n",
    "            encoded_sentence = []\n",
    "            for token in tokens:\n",
    "                if token in self.vocabulary:\n",
    "                    encoded_sentence.append(self.vocabulary[token])\n",
    "                else:\n",
    "                    encoded_sentence.append(self.vocabulary[\"<UNK>\"])\n",
    "            encoded.append(torch.tensor(encoded_sentence))\n",
    "        encoded =  torch.tensor(pad_sequence(encoded, batch_first=True, padding_value=0))\n",
    "        return encoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BertBatchEmbedding:\n",
    "    def __init__(self):\n",
    "        self.model = BertModel.from_pretrained('bert-base-uncased', output_hidden_states=True).eval().cuda()\n",
    "        self.tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
    "    \n",
    "    def transform(self, sentences):\n",
    "        padded_sequence = self.tokenizer.batch_encode_plus(sentences, return_tensors=\"pt\", pad_to_max_length=True)\n",
    "        out = self.model(padded_sequence['input_ids'].cuda(), padded_sequence[\"attention_mask\"].cuda())\n",
    "        hidden_states = out[2]\n",
    "        token_embeddings = torch.stack(hidden_states, dim=0)\n",
    "        return token_embeddings.permute(1,2,0,3)\n",
    "    \n",
    "    def parameters(self):\n",
    "        return self.model.parameters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomBertEmbedding:\n",
    "    def __init__(self):\n",
    "        with open('datasets/embeddings/weights.pickle', 'rb') as handle:\n",
    "            weights = torch.tensor(pickle.load(handle))\n",
    "            \n",
    "        with open('datasets/embeddings/vocab.pickle', 'rb') as handle:\n",
    "            vocabulary = pickle.load(handle)\n",
    "            \n",
    "        self.model = nn.Embedding.from_pretrained(weights, padding_idx = 0, freeze = True)\n",
    "        self.tokenizer = CustomTokenizer(vocabulary)\n",
    "        \n",
    "    def embeddings(self, text):\n",
    "        tokenized_text = self.tokenizer.tokenize(text)\n",
    "        embedded_text = self.model(tokenized_text)\n",
    "        embedded_text = self.encode(embedded_text)\n",
    "        return embedded_text\n",
    "    \n",
    "    def encode(self, embedded_text):\n",
    "        shape = embedded_text.shape\n",
    "        embeddings = torch.mean(embedded_text[:,:,-3072:].reshape(shape[0],shape[1],int(3072/768), 768), dim=2)\n",
    "        return embeddings\n",
    "    \n",
    "    def parameters(self):\n",
    "        return self.model.parameters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = [\"We have a problem.\", \"What is happpening\", \"He was lying\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "custom = CustomBertEmbedding()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/filip/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:17: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "torch.Size([3, 6, 768])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "custom.embeddings(sentences).shape"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}