BERTsum.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "BERTsum",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Interpause/pseudo-text/blob/master/BERTsum.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CkYZtOVawmD9",
        "colab_type": "code",
        "outputId": "924e24f6-e293-4d61-8be2-9e1d9acfa720",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "from google.colab import drive #drive.flush_and_unmount()\n",
        "import os\n",
        "drive.mount('/content/drive')\n",
        "os.chdir('/content/drive/My Drive/CSIT Internship/code')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jwz2Zo0sCgvb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import zipfile\n",
        "zip1 = zipfile.ZipFile('/content/drive/My Drive/CSIT Internship/code/raw/DMQA/cnn.zip','r')\n",
        "zip1.extractall('/content/dmqa')\n",
        "zip1.close()\n",
        "zip2 = zipfile.ZipFile('/content/drive/My Drive/CSIT Internship/code/raw/DMQA/dailymail_stories.zip','r')\n",
        "zip2.extractall('/content/dmqa')\n",
        "zip2.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tboM88caVGHb",
        "colab_type": "code",
        "outputId": "4b6b9c84-5c0d-4881-e478-0106c4161342",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "!pip install --quiet transformers #--force-reinstall"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[?25l\r\u001b[K     |█                               | 10kB 19.5MB/s eta 0:00:01\r\u001b[K     |█▉                              | 20kB 4.4MB/s eta 0:00:01\r\u001b[K     |██▊                             | 30kB 6.0MB/s eta 0:00:01\r\u001b[K     |███▋                            | 40kB 7.6MB/s eta 0:00:01\r\u001b[K     |████▌                           | 51kB 5.1MB/s eta 0:00:01\r\u001b[K     |█████▍                          | 61kB 6.0MB/s eta 0:00:01\r\u001b[K     |██████▎                         | 71kB 6.8MB/s eta 0:00:01\r\u001b[K     |███████▏                        | 81kB 7.6MB/s eta 0:00:01\r\u001b[K     |████████                        | 92kB 8.3MB/s eta 0:00:01\r\u001b[K     |█████████                       | 102kB 6.7MB/s eta 0:00:01\r\u001b[K     |█████████▉                      | 112kB 6.7MB/s eta 0:00:01\r\u001b[K     |██████████▊                     | 122kB 6.7MB/s eta 0:00:01\r\u001b[K     |███████████▊                    | 133kB 6.7MB/s eta 0:00:01\r\u001b[K     |████████████▋                   | 143kB 6.7MB/s eta 0:00:01\r\u001b[K     |█████████████▌                  | 153kB 6.7MB/s eta 0:00:01\r\u001b[K     |██████████████▍                 | 163kB 6.7MB/s eta 0:00:01\r\u001b[K     |███████████████▎                | 174kB 6.7MB/s eta 0:00:01\r\u001b[K     |████████████████▏               | 184kB 6.7MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 194kB 6.7MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 204kB 6.7MB/s eta 0:00:01\r\u001b[K     |██████████████████▉             | 215kB 6.7MB/s eta 0:00:01\r\u001b[K     |███████████████████▊            | 225kB 6.7MB/s eta 0:00:01\r\u001b[K     |████████████████████▋           | 235kB 6.7MB/s eta 0:00:01\r\u001b[K     |█████████████████████▌          | 245kB 6.7MB/s eta 0:00:01\r\u001b[K     |██████████████████████▌         | 256kB 6.7MB/s eta 0:00:01\r\u001b[K     |███████████████████████▍        | 266kB 6.7MB/s eta 0:00:01\r\u001b[K     |████████████████████████▎       | 276kB 6.7MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▏      | 286kB 6.7MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 296kB 6.7MB/s eta 0:00:01\r\u001b[K     |███████████████████████████     | 307kB 6.7MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▉    | 317kB 6.7MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▊   | 327kB 6.7MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▋  | 337kB 6.7MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▌ | 348kB 6.7MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▍| 358kB 6.7MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 368kB 6.7MB/s \n",
            "\u001b[?25h\u001b[?25l\r\u001b[K     |▎                               | 10kB 22.6MB/s eta 0:00:01\r\u001b[K     |▋                               | 20kB 29.3MB/s eta 0:00:01\r\u001b[K     |█                               | 30kB 33.9MB/s eta 0:00:01\r\u001b[K     |█▎                              | 40kB 37.3MB/s eta 0:00:01\r\u001b[K     |█▋                              | 51kB 38.8MB/s eta 0:00:01\r\u001b[K     |██                              | 61kB 40.7MB/s eta 0:00:01\r\u001b[K     |██▏                             | 71kB 41.5MB/s eta 0:00:01\r\u001b[K     |██▌                             | 81kB 40.6MB/s eta 0:00:01\r\u001b[K     |██▉                             | 92kB 41.1MB/s eta 0:00:01\r\u001b[K     |███▏                            | 102kB 42.4MB/s eta 0:00:01\r\u001b[K     |███▌                            | 112kB 42.4MB/s eta 0:00:01\r\u001b[K     |███▉                            | 122kB 42.4MB/s eta 0:00:01\r\u001b[K     |████                            | 133kB 42.4MB/s eta 0:00:01\r\u001b[K     |████▍                           | 143kB 42.4MB/s eta 0:00:01\r\u001b[K     |████▊                           | 153kB 42.4MB/s eta 0:00:01\r\u001b[K     |█████                           | 163kB 42.4MB/s eta 0:00:01\r\u001b[K     |█████▍                          | 174kB 42.4MB/s eta 0:00:01\r\u001b[K     |█████▊                          | 184kB 42.4MB/s eta 0:00:01\r\u001b[K     |██████                          | 194kB 42.4MB/s eta 0:00:01\r\u001b[K     |██████▎                         | 204kB 42.4MB/s eta 0:00:01\r\u001b[K     |██████▋                         | 215kB 42.4MB/s eta 0:00:01\r\u001b[K     |███████                         | 225kB 42.4MB/s eta 0:00:01\r\u001b[K     |███████▎                        | 235kB 42.4MB/s eta 0:00:01\r\u001b[K     |███████▋                        | 245kB 42.4MB/s eta 0:00:01\r\u001b[K     |███████▉                        | 256kB 42.4MB/s eta 0:00:01\r\u001b[K     |████████▏                       | 266kB 42.4MB/s eta 0:00:01\r\u001b[K     |████████▌                       | 276kB 42.4MB/s eta 0:00:01\r\u001b[K     |████████▉                       | 286kB 42.4MB/s eta 0:00:01\r\u001b[K     |█████████▏                      | 296kB 42.4MB/s eta 0:00:01\r\u001b[K     |█████████▌                      | 307kB 42.4MB/s eta 0:00:01\r\u001b[K     |█████████▊                      | 317kB 42.4MB/s eta 0:00:01\r\u001b[K     |██████████                      | 327kB 42.4MB/s eta 0:00:01\r\u001b[K     |██████████▍                     | 337kB 42.4MB/s eta 0:00:01\r\u001b[K     |██████████▊                     | 348kB 42.4MB/s eta 0:00:01\r\u001b[K     |███████████                     | 358kB 42.4MB/s eta 0:00:01\r\u001b[K     |███████████▍                    | 368kB 42.4MB/s eta 0:00:01\r\u001b[K     |███████████▋                    | 378kB 42.4MB/s eta 0:00:01\r\u001b[K     |████████████                    | 389kB 42.4MB/s eta 0:00:01\r\u001b[K     |████████████▎                   | 399kB 42.4MB/s eta 0:00:01\r\u001b[K     |████████████▋                   | 409kB 42.4MB/s eta 0:00:01\r\u001b[K     |█████████████                   | 419kB 42.4MB/s eta 0:00:01\r\u001b[K     |█████████████▎                  | 430kB 42.4MB/s eta 0:00:01\r\u001b[K     |█████████████▌                  | 440kB 42.4MB/s eta 0:00:01\r\u001b[K     |█████████████▉                  | 450kB 42.4MB/s eta 0:00:01\r\u001b[K     |██████████████▏                 | 460kB 42.4MB/s eta 0:00:01\r\u001b[K     |██████████████▌                 | 471kB 42.4MB/s eta 0:00:01\r\u001b[K     |██████████████▉                 | 481kB 42.4MB/s eta 0:00:01\r\u001b[K     |███████████████▏                | 491kB 42.4MB/s eta 0:00:01\r\u001b[K     |███████████████▍                | 501kB 42.4MB/s eta 0:00:01\r\u001b[K     |███████████████▊                | 512kB 42.4MB/s eta 0:00:01\r\u001b[K     |████████████████                | 522kB 42.4MB/s eta 0:00:01\r\u001b[K     |████████████████▍               | 532kB 42.4MB/s eta 0:00:01\r\u001b[K     |████████████████▊               | 542kB 42.4MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 552kB 42.4MB/s eta 0:00:01\r\u001b[K     |█████████████████▎              | 563kB 42.4MB/s eta 0:00:01\r\u001b[K     |█████████████████▋              | 573kB 42.4MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 583kB 42.4MB/s eta 0:00:01\r\u001b[K     |██████████████████▎             | 593kB 42.4MB/s eta 0:00:01\r\u001b[K     |██████████████████▋             | 604kB 42.4MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 614kB 42.4MB/s eta 0:00:01\r\u001b[K     |███████████████████▏            | 624kB 42.4MB/s eta 0:00:01\r\u001b[K     |███████████████████▌            | 634kB 42.4MB/s eta 0:00:01\r\u001b[K     |███████████████████▉            | 645kB 42.4MB/s eta 0:00:01\r\u001b[K     |████████████████████▏           | 655kB 42.4MB/s eta 0:00:01\r\u001b[K     |████████████████████▌           | 665kB 42.4MB/s eta 0:00:01\r\u001b[K     |████████████████████▉           | 675kB 42.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████▏          | 686kB 42.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████▍          | 696kB 42.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████▊          | 706kB 42.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 716kB 42.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████▍         | 727kB 42.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████▊         | 737kB 42.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 747kB 42.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████▎        | 757kB 42.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████▋        | 768kB 42.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████        | 778kB 42.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████▎       | 788kB 42.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████▋       | 798kB 42.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 808kB 42.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▏      | 819kB 42.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▌      | 829kB 42.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▉      | 839kB 42.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▏     | 849kB 42.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▌     | 860kB 42.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▉     | 870kB 42.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████     | 880kB 42.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▍    | 890kB 42.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▊    | 901kB 42.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████    | 911kB 42.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▍   | 921kB 42.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▊   | 931kB 42.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 942kB 42.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▎  | 952kB 42.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▋  | 962kB 42.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 972kB 42.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▎ | 983kB 42.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▋ | 993kB 42.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▉ | 1.0MB 42.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▏| 1.0MB 42.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▌| 1.0MB 42.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▉| 1.0MB 42.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 1.0MB 42.4MB/s \n",
            "\u001b[K     |████████████████████████████████| 860kB 47.1MB/s \n",
            "\u001b[K     |████████████████████████████████| 645kB 49.0MB/s \n",
            "\u001b[?25h  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6XT3MdHT8SQ0",
        "colab_type": "code",
        "outputId": "ee9da859-3f5b-4429-b5e6-145719a6df4f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "%tensorflow_version 2.x\n",
        "import os\n",
        "os.chdir('/content/drive/My Drive/CSIT Internship/code')\n",
        "import torch\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from torch import nn\n",
        "from torch.optim import Adam\n",
        "from transformers import PreTrainedEncoderDecoder\n",
        "#from transformers import AutoModel, AutoTokenizer, AutoConfig\n",
        "from tqdm import tqdm, trange\n",
        "import contextlib\n",
        "\n",
        "class DummyFile(object):\n",
        "  file = None\n",
        "  def __init__(self, file):\n",
        "    self.file = file\n",
        "\n",
        "  def write(self, x):\n",
        "    # Avoid print() second call (useless \\n)\n",
        "    if len(x.rstrip()) > 0:\n",
        "        tqdm.write(x, file=self.file)\n",
        "\n",
        "@contextlib.contextmanager\n",
        "def nostdout():\n",
        "    save_stdout = sys.stdout\n",
        "    sys.stdout = DummyFile(sys.stdout)\n",
        "    yield\n",
        "    sys.stdout = save_stdout"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TensorFlow 2.x selected.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HUsrblvf4HHa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#from transformers import RobertaConfig,RobertaTokenizer,RobertaModel,RobertaForMaskedLM\n",
        "#from transformers import DistilBertConfig,DistilBertTokenizer,DistilBertModel,DistilBertForMaskedLM\n",
        "from transformers import BertTokenizer, BertConfig, BertForMaskedLM, BertModel, PreTrainedEncoderDecoder"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8OFrMlcG4HAB",
        "colab_type": "code",
        "outputId": "c8fa4200-b4e8-4805-e6ae-2ba6f60a4c50",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "torch.cuda.empty_cache()\n",
        "pretrained = \"bert-base-uncased\"\n",
        "tokenizer = BertTokenizer.from_pretrained(pretrained)\n",
        "#config = DistilBertConfig.from_pretrained(pretrained)\n",
        "encoder = BertModel.from_pretrained(pretrained)\n",
        "decoder = BertForMaskedLM.from_pretrained(pretrained)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 313/313 [00:00<00:00, 153869.80B/s]\n",
            "100%|██████████| 440473133/440473133 [00:14<00:00, 29437135.16B/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PF2h5vi6paAJ",
        "colab_type": "code",
        "outputId": "8cf06dc5-e91c-4925-8063-4a2a2f905bc3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "from transformers import AlbertTokenizer, AlbertModel\n",
        "torch.cuda.empty_cache()\n",
        "tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2')\n",
        "encoder = AlbertModel.from_pretrained('albert-base-v2')\n",
        "decoder = BertForMaskedLM(BertConfig.from_pretrained('bert-base-uncased'))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "There is currently an upstream reproducibility issue with ALBERT v2 models. Please see https://github.com/google-research/google-research/issues/119 for more information.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wv6oxK-q4G0z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model = PreTrainedEncoderDecoder.from_pretrained(**{'encoder_model':encoder,'decoder_model':decoder}).to(device)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g3IVfe7d7Nmv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#https://github.com/huggingface/transformers/blob/35ff345fc9df9e777b27903f11fa213e4052595b/examples/run_summarization_finetuning.py#L139\n",
        "#modified for Albert, some inference was made & code was modified\n",
        "class BertSumOptimizer(object):\n",
        "    \"\"\" Specific optimizer for BertSum.\n",
        "    As described in [1], the authors fine-tune BertSum for abstractive\n",
        "    summarization using two Adam Optimizers with different warm-up steps and\n",
        "    learning rate. They also use a custom learning rate scheduler.\n",
        "    [1] Liu, Yang, and Mirella Lapata. \"Text summarization with pretrained encoders.\"\n",
        "        arXiv preprint arXiv:1908.08345 (2019).\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, model, lr, warmup_steps, beta_1=0.99, beta_2=0.999, eps=1e-8):\n",
        "        self.encoder = model.encoder\n",
        "        self.decoder = model.decoder\n",
        "        self.lr = lr\n",
        "        self.warmup_steps = warmup_steps\n",
        "\n",
        "        self.optimizers = {\n",
        "            \"encoder\": Adam(\n",
        "                model.encoder.parameters(),\n",
        "                lr=lr[\"encoder\"],\n",
        "                betas=(beta_1, beta_2),\n",
        "                eps=eps,\n",
        "            ),\n",
        "            \"decoder\": Adam(\n",
        "                model.decoder.parameters(),\n",
        "                lr=lr[\"decoder\"],\n",
        "                betas=(beta_1, beta_2),\n",
        "                eps=eps,\n",
        "            ),\n",
        "        }\n",
        "\n",
        "        self._step = 0\n",
        "\n",
        "    def _update_rate(self, stack):\n",
        "        return self.lr[stack] * min(\n",
        "            self._step ** (-0.5), self._step * self.warmup_steps[stack] ** (-0.5)\n",
        "        )\n",
        "\n",
        "    def zero_grad(self):\n",
        "        #There was a bug here. interestingly\n",
        "        self.optimizers['decoder'].zero_grad()\n",
        "        self.optimizers['encoder'].zero_grad()\n",
        "\n",
        "    def step(self):\n",
        "        self._step += 1\n",
        "        for stack, optimizer in self.optimizers.items():\n",
        "            new_rate = self._update_rate(stack)\n",
        "            for param_group in optimizer.param_groups:\n",
        "                param_group[\"lr\"] = new_rate\n",
        "            optimizer.step()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CtNRtgrqEK8f",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import importlib.util\n",
        "import sys\n",
        "spec = importlib.util.spec_from_file_location('summary_utils','./utils_summarization.py')\n",
        "summary_utils = importlib.util.module_from_spec(spec)\n",
        "sys.modules['summary_utils'] = summary_utils\n",
        "spec.loader.exec_module(summary_utils)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_8UOHvCaIt2w",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def collate(data, tokenizer, block_size):\n",
        "    # remove the files with empty an story/summary, encode and fit to block\n",
        "    data = filter(lambda x: not (len(x[0]) == 0 or len(x[1]) == 0), data)\n",
        "    data = [summary_utils.encode_for_summarization(story, summary, tokenizer) for story, summary in data]\n",
        "    data = [(\n",
        "            summary_utils.fit_to_block_size(story, block_size, tokenizer.pad_token_id),\n",
        "            summary_utils.fit_to_block_size(summary, block_size, tokenizer.pad_token_id),\n",
        "        ) for story, summary in data\n",
        "    ]\n",
        "\n",
        "    stories = torch.tensor([story for story, summary in data])\n",
        "    summaries = torch.tensor([summary for story, summary in data])\n",
        "    encoder_token_type_ids = summary_utils.compute_token_type_ids(stories, tokenizer.cls_token_id)\n",
        "    encoder_mask = summary_utils.build_mask(stories, tokenizer.pad_token_id)\n",
        "    decoder_mask = summary_utils.build_mask(summaries, tokenizer.pad_token_id)\n",
        "    lm_labels = summary_utils.build_lm_labels(summaries, tokenizer.pad_token_id)\n",
        "\n",
        "    return (stories,summaries,encoder_token_type_ids,encoder_mask,decoder_mask,lm_labels)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wdJxTuTe8pe9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
        "import functools\n",
        "import logging\n",
        "logger = logging.getLogger(__name__)\n",
        "df = pd.read_pickle('./raw/myNews.pkl')\n",
        "df = df.sample(frac=1) #apparently this is shuffling\n",
        "batch_size = 4\n",
        "epochs = 1\n",
        "def train(model):\n",
        "    #straight up stolen still\n",
        "    lr = {\"encoder\": 0.002, \"decoder\": 0.2}\n",
        "    warmup_steps = {\"encoder\": 20000, \"decoder\": 10000}\n",
        "    optimizer = BertSumOptimizer(model, lr, warmup_steps)\n",
        "    epoch_iter = trange(epochs,desc='Epoch',disable=True)\n",
        "\n",
        "    train_dataset = summary_utils.CNNDailyMailDataset(tokenizer, data_dir='/content/dmqa')\n",
        "    train_sampler = RandomSampler(train_dataset)\n",
        "    model_collate_fn = functools.partial(collate, tokenizer=tokenizer, block_size=512)\n",
        "    train_dataloader = DataLoader(train_dataset,sampler=train_sampler,batch_size=batch_size,collate_fn=model_collate_fn)\n",
        "    t_total = len(train_dataloader)//1*epochs\n",
        "\n",
        "    logger.info(\"***** Running training *****\")\n",
        "    logger.info(\"  Num examples = %d\", len(train_dataset))\n",
        "    logger.info(\"  Num Epochs = %d\", epochs)\n",
        "    \n",
        "    model.train()\n",
        "    global_step = 0\n",
        "    tr_loss = 0.0\n",
        "    for i in epoch_iter:\n",
        "        train_iter = tqdm(train_dataloader, desc=\"Epoch %d/%d\"%(i,epochs), unit='batches',postfix={'loss':'0.00'},file=sys.stdout,dynamic_ncols=True)\n",
        "        for step,batch in enumerate(train_iter):\n",
        "            source, target, encoder_token_type_ids, encoder_mask, decoder_mask, lm_labels = batch\n",
        "            source = source.to(device)\n",
        "            target = target.to(device)\n",
        "            encoder_token_type_ids = encoder_token_type_ids.to(device)\n",
        "            encoder_mask = encoder_mask.to(device)\n",
        "            decoder_mask = decoder_mask.to(device)\n",
        "            lm_labels = lm_labels.to(device)\n",
        "\n",
        "            model.zero_grad()\n",
        "            optimizer.zero_grad()\n",
        "\n",
        "            outputs = model(source,target,\n",
        "                encoder_token_type_ids=encoder_token_type_ids,\n",
        "                encoder_attention_mask=encoder_mask,\n",
        "                decoder_attention_mask=decoder_mask,\n",
        "                decoder_lm_labels=lm_labels\n",
        "            )\n",
        "            loss = outputs[0]\n",
        "            train_iter.set_postfix(loss='%.2f'%loss.item(),refresh=False)\n",
        "            loss.backward()\n",
        "            tr_loss += loss.item()\n",
        "\n",
        "            torch.nn.utils.clip_grad_norm_(model.parameters(),1.0)\n",
        "            optimizer.step()\n",
        "\n",
        "            global_step += 1\n",
        "    return global_step,tr_loss/global_step"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1tab8VyI-pQh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#https://www.aclweb.org/anthology/D19-1387.pdf\n",
        "train(model)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7WKdqrHPYTtO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#model.eval()\n",
        "model.save_pretrained(\"/content/drive/My Drive/CSIT Internship/code/albertBertsum/\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i23JGjIlZbya",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model2 = PreTrainedEncoderDecoder.from_pretrained(\n",
        "    encoder_pretrained_model_name_or_path = '/content/drive/My Drive/CSIT Internship/code/albertBertsum/encoder/',\n",
        "    decoder_pretrained_model_name_or_path = '/content/drive/My Drive/CSIT Internship/code/albertBertsum/decoder/')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-Xd1slkeb1Ve",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model2."
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W10SKQ9AIz7A",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "options = {\n",
        "    'max_length':                   512,            #causes overflow\n",
        "    'stride':                       0,              #how many tokens before to start from in a overflow\n",
        "    'add_special_tokens':           True,           #Adds [CLS] & [SEP]\n",
        "    'truncation_strategy':          'only_second',  #Only body of article is cut\n",
        "    'return_token_type_ids':        True,           #when using pairs, adds the type indicator\n",
        "    'return_overflowing_tokens':    False,          #allows for overflowing tokens\n",
        "    'return_special_tokens_mask':   True,           #masking tokens\n",
        "    'return_tensors':               'pt'            #return as torch tensor\n",
        "}\n",
        "pad_c = tokenizer.special_tokens_map['pad_token']\n",
        "sep_c = tokenizer.special_tokens_map['sep_token']\n",
        "options['max_length'] += 1\n",
        "\n",
        "def processInput(articles):\n",
        "    id_batch = []\n",
        "    type_batch = []\n",
        "    mask_batch = []\n",
        "    for title,text in articles:\n",
        "        enc = tokenizer.encode_plus(title,text+sep_c+pad_c*options['max_length'],**options)\n",
        "\n",
        "        token_id = enc['input_ids'][0][:options['max_length']-1]\n",
        "        token_type = enc['token_type_ids'][0][:options['max_length']-1]\n",
        "        mask = token_id.clone()\n",
        "        mask[mask != pad_c] = 1\n",
        "\n",
        "        id_batch.append(token_id.to(device))\n",
        "        type_batch.append(token_type.to(device))\n",
        "        mask_batch.append(mask.to(device))\n",
        "    return (torch.stack(id_batch),torch.stack(type_batch),torch.stack(mask_batch))\n",
        "#processInput(df.head(5).apply(lambda row: (row.title,row.text),axis=1))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4lW8bwIQ-M9p",
        "colab_type": "code",
        "outputId": "106fab98-ba33-4120-a2c6-5400230426d4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "pretrained = \"albert-base-v2\"\n",
        "tokenizer = AutoTokenizer.from_pretrained(pretrained)\n",
        "encoder = AutoModel.from_pretrained(pretrained)\n",
        "decoder = AutoModel.from_pretrained(pretrained) #tbh shouldnt this be zero-initialized\n",
        "encoder = encoder.to(device)\n",
        "decoder = decoder.to(device)\n",
        "#encoder.train(), decoder.train()\n",
        "df = pd.read_pickle('./raw/myNews.pkl')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 760289/760289 [00:00<00:00, 1829727.12B/s]\n",
            "There is currently an upstream reproducibility issue with ALBERT v2 models. Please see https://github.com/google-research/google-research/issues/119 for more information.\n",
            "100%|██████████| 483/483 [00:00<00:00, 227189.51B/s]\n",
            "100%|██████████| 47376696/47376696 [00:02<00:00, 20413045.95B/s]\n",
            "There is currently an upstream reproducibility issue with ALBERT v2 models. Please see https://github.com/google-research/google-research/issues/119 for more information.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "70Xtib-e-Nvm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class albertVAE(nn.Module):\n",
        "    def __init__(self,encoder,decoder,projection_size=8192,vocab_size=30000,device=device):\n",
        "        super(albertVAE,self).__init__()\n",
        "        self.encoder = encoder.to(device)\n",
        "        self.decoder = decoder.to(device)\n",
        "\n",
        "        self.in_lin = nn.Linear(768*512,int(768*512/64)).to(device)\n",
        "        self.std_lin = nn.Linear(int(768*512/64),projection_size).to(device)\n",
        "        self.eps_lin = nn.Linear(int(768*512/64),projection_size).to(device)\n",
        "        self.proj_lin = nn.Linear(projection_size,int(768*512/64)).to(device)\n",
        "        self.repa_lin = nn.Linear(int(768*512/64),768*512).to(device)\n",
        "        self.logit_lin = nn.Linear(768,vocab_size).to(device)\n",
        "\n",
        "    def encode(self,x):\n",
        "        h = torch.relu(self.in_lin(x))\n",
        "        return (self.eps_lin(h), self.std_lin(h))\n",
        "\n",
        "    def reparameterize(self,mu,logvar):\n",
        "        std = torch.exp(0.5*logvar)\n",
        "        eps = torch.randn_like(std)\n",
        "        return mu+eps*std\n",
        "\n",
        "    def decode(self, x):\n",
        "        h1 = torch.relu(self.proj_lin(x))\n",
        "        h2 = torch.sigmoid(self.repa_lin(h1))\n",
        "        return self.decoder(inputs_embeds=h2.reshape(-1,512,768))\n",
        "\n",
        "    def forward(input_ids,type_ids,masks):\n",
        "        x1 = self.encoder(input_ids=input_ids,token_type_ids=type_ids,attention_mask=masks)\n",
        "        mu, logvar = self.encode(x1.reshape(x1.shape[0],-1))\n",
        "        x2 = self.reparameterize(mu, logvar)\n",
        "        return (self.decoder,mu,logvar)\n",
        "\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2I0sKFgKNCDt",
        "colab_type": "code",
        "outputId": "44193b17-e512-4dd8-b6d9-634dcbbba8a9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 360
        }
      },
      "source": [
        "input_ids,type_ids,masks = processInput(df.head(3).apply(lambda row: (row.title,row.text),axis=1))\n",
        "model = albertVAE(encoder,decoder).to(device)\n",
        "print(ouput)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "error",
          "ename": "RuntimeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-7-32637843ff50>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0minput_ids\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtype_ids\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmasks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mprocessInput\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhead\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mrow\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mrow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtitle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mrow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0malbertVAE\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mencoder\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdecoder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-6-62a19e005846>\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, encoder, decoder, projection_size, vocab_size, device)\u001b[0m\n\u001b[1;32m      5\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdecoder\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdecoder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0min_lin\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m768\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m512\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m768\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m512\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0;36m64\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstd_lin\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m768\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m512\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0;36m64\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mprojection_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meps_lin\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m768\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m512\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0;36m64\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mprojection_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36mto\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    424\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_floating_point\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_blocking\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    425\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 426\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconvert\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    427\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    428\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_backward_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_apply\u001b[0;34m(self, fn)\u001b[0m\n\u001b[1;32m    222\u001b[0m                 \u001b[0;31m# `with torch.no_grad():`\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    223\u001b[0m                 \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 224\u001b[0;31m                     \u001b[0mparam_applied\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparam\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    225\u001b[0m                 \u001b[0mshould_use_set_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompute_should_use_set_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparam\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparam_applied\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    226\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mshould_use_set_data\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36mconvert\u001b[0;34m(t)\u001b[0m\n\u001b[1;32m    422\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    423\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mconvert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 424\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_floating_point\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_blocking\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    425\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    426\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconvert\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mRuntimeError\u001b[0m: CUDA out of memory. Tried to allocate 9.00 GiB (GPU 0; 7.43 GiB total capacity; 89.17 MiB already allocated; 6.80 GiB free; 24.83 MiB cached)"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nfcq_LUAnVFE",
        "colab_type": "code",
        "outputId": "678e4dad-57a0-4ccb-fe3e-9f454209100f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "tokenizer.vocab_size"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "30000"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RX5EVBPAjG1Q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "torch.cuda.empty_cache()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FpUkvuw6N5iC",
        "colab_type": "code",
        "outputId": "a14d66e4-4d6b-4888-987c-60e0c53a1f85",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "#TODO: Remove the japanese and chinese in my news dataset like how did it get there\n",
        "#print(df.text[df.text.str.split().str.len() == max(df.text.str.split().str.len())].item())\n",
        "lmao = df.text.str.split().str.len()\n",
        "lol = df.text[(lmao <= 512) & (lmao > 256)] #no 2000 word essays please\n",
        "sum(lol.str.split().str.len())/len(lol.str.split().str.len())\n",
        "len(lol)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2328"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 117
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xMhJiaZTMxD_",
        "colab_type": "text"
      },
      "source": [
        "# Indexing\n",
        "Dataset is laaarge.\n",
        "* Sentence embeddings from word embeddings, sparse PCA, clustering, vectors for all documents stored in table, easily calculate distance to query.\n",
        "* Extract noun phrases, match nouns in query to selected phrases for each document.\n",
        "\n",
        "I am not sure about the fastest approach, nonetheless, minimize false negatives >>> minimizing false positives. Better last stage algorithm then further minimizes false positives.\n",
        "\n",
        "* Conclusion: i will use ALBERT finetuned on abstractive summarization directly, using the encoded values as the indexing for the large dataset... theoretically this is already damn good, making it then use the original texts for comparison is overkill.\n",
        "* damnit im reusing the same ALBERT for the paraphraser too later on lol"
      ]
    }
  ]
}