BertExtractiveTrain.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "BertExtractiveTrain.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Santosh-Gupta/ScientificSummarizationDataSets/blob/master/BertExtractiveTrain.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AAUP2VJ2Gewc",
        "colab_type": "code",
        "outputId": "d1e5f08e-0c4c-468c-84a8-b135891781f8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LkO9j8kyGetm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import requests\n",
        "import os\n",
        "import random\n",
        "import tarfile\n",
        "import pandas as pd\n",
        "import zipfile\n",
        "import multiprocess\n",
        "import json\n",
        "from glob import glob"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7_6pCT0eGep7",
        "colab_type": "code",
        "outputId": "4f337160-c89f-4ea7-c836-e48372029706",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 433
        }
      },
      "source": [
        "!pip install torch\n",
        "!pip install pytorch-pretrained-bert\n",
        "!pip install tensorboardX\n",
        "!pip install pyrouge"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: torch in /usr/local/lib/python3.6/dist-packages (1.2.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from torch) (1.16.5)\n",
            "Requirement already satisfied: pytorch-pretrained-bert in /usr/local/lib/python3.6/dist-packages (0.6.2)\n",
            "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from pytorch-pretrained-bert) (1.9.236)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from pytorch-pretrained-bert) (4.28.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from pytorch-pretrained-bert) (1.16.5)\n",
            "Requirement already satisfied: torch>=0.4.1 in /usr/local/lib/python3.6/dist-packages (from pytorch-pretrained-bert) (1.2.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from pytorch-pretrained-bert) (2.21.0)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.6/dist-packages (from pytorch-pretrained-bert) (2019.8.19)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch-pretrained-bert) (0.9.4)\n",
            "Requirement already satisfied: s3transfer<0.3.0,>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch-pretrained-bert) (0.2.1)\n",
            "Requirement already satisfied: botocore<1.13.0,>=1.12.236 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch-pretrained-bert) (1.12.236)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch-pretrained-bert) (2019.9.11)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch-pretrained-bert) (3.0.4)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch-pretrained-bert) (1.24.3)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch-pretrained-bert) (2.8)\n",
            "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.13.0,>=1.12.236->boto3->pytorch-pretrained-bert) (0.15.2)\n",
            "Requirement already satisfied: python-dateutil<3.0.0,>=2.1; python_version >= \"2.7\" in /usr/local/lib/python3.6/dist-packages (from botocore<1.13.0,>=1.12.236->boto3->pytorch-pretrained-bert) (2.5.3)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil<3.0.0,>=2.1; python_version >= \"2.7\"->botocore<1.13.0,>=1.12.236->boto3->pytorch-pretrained-bert) (1.12.0)\n",
            "Requirement already satisfied: tensorboardX in /usr/local/lib/python3.6/dist-packages (1.8)\n",
            "Requirement already satisfied: protobuf>=3.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorboardX) (3.7.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from tensorboardX) (1.16.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from tensorboardX) (1.12.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.2.0->tensorboardX) (41.2.0)\n",
            "Requirement already satisfied: pyrouge in /usr/local/lib/python3.6/dist-packages (0.1.3)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8aqCs4Kd5qLo",
        "colab_type": "text"
      },
      "source": [
        "Download and Unzip Scibert weights"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sdUAXalv5dO9",
        "colab_type": "code",
        "outputId": "a1e5a386-be9d-4f6c-d926-7ca555c63ba4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "#Download SciBert weights and vocab file\n",
        "import urllib.request\n",
        "# Download the file from `url` and save it locally under `file_name`:\n",
        "urllib.request.urlretrieve('https://s3-us-west-2.amazonaws.com/ai2-s2-research/scibert/pytorch_models/scibert_scivocab_uncased.tar', 'scibert.tar')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('scibert.tar', <http.client.HTTPMessage at 0x7fb455b18f60>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gyR5k9ye5ueS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tarfile\n",
        "tar = tarfile.open('scibert.tar', \"r:\")\n",
        "tar.extractall()\n",
        "tar.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QthrkM-N5wXi",
        "colab_type": "code",
        "outputId": "827aa55c-a7f7-41f2-a067-addd02eeb161",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "os.listdir('scibert_scivocab_uncased')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['vocab.txt', 'weights.tar.gz']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9DrF8jAr5ye8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tar = tarfile.open('scibert_scivocab_uncased/weights.tar.gz', \"r:gz\")\n",
        "tar.extractall('scibert_scivocab_uncased')\n",
        "tar.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CJh2DoxwKEZr",
        "colab_type": "code",
        "outputId": "f9de9102-bf12-49fe-e2de-ec5f9315d652",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "os.path.abspath('../../scibert_scivocab_uncased/pytorch_model.bin')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'/content/scibert_scivocab_uncased/pytorch_model.bin'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GhoSwvGYKGFe",
        "colab_type": "code",
        "outputId": "40849542-2221-4e06-e38b-c2f5569cb9f2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "os.path.abspath('scibert_scivocab_uncased/bert_config.json')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'/content/BertSum/src/scibert_scivocab_uncased/bert_config.json'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xqGbAJEpGvdy",
        "colab_type": "code",
        "outputId": "99d86ec9-6c38-432a-b718-1e72a956dfac",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 133
        }
      },
      "source": [
        "!git clone https://github.com/Santosh-Gupta/BertSum.git"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'BertSum'...\n",
            "remote: Enumerating objects: 78, done.\u001b[K\n",
            "remote: Counting objects:   1% (1/78)\u001b[K\rremote: Counting objects:   2% (2/78)\u001b[K\rremote: Counting objects:   3% (3/78)\u001b[K\rremote: Counting objects:   5% (4/78)\u001b[K\rremote: Counting objects:   6% (5/78)\u001b[K\rremote: Counting objects:   7% (6/78)\u001b[K\rremote: Counting objects:   8% (7/78)\u001b[K\rremote: Counting objects:  10% (8/78)\u001b[K\rremote: Counting objects:  11% (9/78)\u001b[K\rremote: Counting objects:  12% (10/78)\u001b[K\rremote: Counting objects:  14% (11/78)\u001b[K\rremote: Counting objects:  15% (12/78)\u001b[K\rremote: Counting objects:  16% (13/78)\u001b[K\rremote: Counting objects:  17% (14/78)\u001b[K\rremote: Counting objects:  19% (15/78)\u001b[K\rremote: Counting objects:  20% (16/78)\u001b[K\rremote: Counting objects:  21% (17/78)\u001b[K\rremote: Counting objects:  23% (18/78)\u001b[K\rremote: Counting objects:  24% (19/78)\u001b[K\rremote: Counting objects:  25% (20/78)\u001b[K\rremote: Counting objects:  26% (21/78)\u001b[K\rremote: Counting objects:  28% (22/78)\u001b[K\rremote: Counting objects:  29% (23/78)\u001b[K\rremote: Counting objects:  30% (24/78)\u001b[K\rremote: Counting objects:  32% (25/78)\u001b[K\rremote: Counting objects:  33% (26/78)\u001b[K\rremote: Counting objects:  34% (27/78)\u001b[K\rremote: Counting objects:  35% (28/78)\u001b[K\rremote: Counting objects:  37% (29/78)\u001b[K\rremote: Counting objects:  38% (30/78)\u001b[K\rremote: Counting objects:  39% (31/78)\u001b[K\rremote: Counting objects:  41% (32/78)\u001b[K\rremote: Counting objects:  42% (33/78)\u001b[K\rremote: Counting objects:  43% (34/78)\u001b[K\rremote: Counting objects:  44% (35/78)\u001b[K\rremote: Counting objects:  46% (36/78)\u001b[K\rremote: Counting objects:  47% (37/78)\u001b[K\rremote: Counting objects:  48% (38/78)\u001b[K\rremote: Counting objects:  50% (39/78)\u001b[K\rremote: Counting objects:  51% (40/78)\u001b[K\rremote: Counting objects:  52% (41/78)\u001b[K\rremote: Counting objects:  53% (42/78)\u001b[K\rremote: Counting objects:  55% (43/78)\u001b[K\rremote: Counting objects:  56% (44/78)\u001b[K\rremote: Counting objects:  57% (45/78)\u001b[K\rremote: Counting objects:  58% (46/78)\u001b[K\rremote: Counting objects:  60% (47/78)\u001b[K\rremote: Counting objects:  61% (48/78)\u001b[K\rremote: Counting objects:  62% (49/78)\u001b[K\rremote: Counting objects:  64% (50/78)\u001b[K\rremote: Counting objects:  65% (51/78)\u001b[K\rremote: Counting objects:  66% (52/78)\u001b[K\rremote: Counting objects:  67% (53/78)\u001b[K\rremote: Counting objects:  69% (54/78)\u001b[K\rremote: Counting objects:  70% (55/78)\u001b[K\rremote: Counting objects:  71% (56/78)\u001b[K\rremote: Counting objects:  73% (57/78)\u001b[K\rremote: Counting objects:  74% (58/78)\u001b[K\rremote: Counting objects:  75% (59/78)\u001b[K\rremote: Counting objects:  76% (60/78)\u001b[K\rremote: Counting objects:  78% (61/78)\u001b[K\rremote: Counting objects:  79% (62/78)\u001b[K\rremote: Counting objects:  80% (63/78)\u001b[K\rremote: Counting objects:  82% (64/78)\u001b[K\rremote: Counting objects:  83% (65/78)\u001b[K\rremote: Counting objects:  84% (66/78)\u001b[K\rremote: Counting objects:  85% (67/78)\u001b[K\rremote: Counting objects:  87% (68/78)\u001b[K\rremote: Counting objects:  88% (69/78)\u001b[K\rremote: Counting objects:  89% (70/78)\u001b[K\rremote: Counting objects:  91% (71/78)\u001b[K\rremote: Counting objects:  92% (72/78)\u001b[K\rremote: Counting objects:  93% (73/78)\u001b[K\rremote: Counting objects:  94% (74/78)\u001b[K\rremote: Counting objects:  96% (75/78)\u001b[K\rremote: Counting objects:  97% (76/78)\u001b[K\rremote: Counting objects:  98% (77/78)\u001b[K\rremote: Counting objects: 100% (78/78)\u001b[K\rremote: Counting objects: 100% (78/78), done.\u001b[K\n",
            "remote: Compressing objects: 100% (78/78), done.\u001b[K\n",
            "remote: Total 423 (delta 50), reused 0 (delta 0), pack-reused 345\u001b[K\n",
            "Receiving objects: 100% (423/423), 15.07 MiB | 17.55 MiB/s, done.\n",
            "Resolving deltas: 100% (259/259), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xn8HSc5mGvU6",
        "colab_type": "code",
        "outputId": "13354982-225f-4106-f67d-a9a73d550861",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "%cd BertSum/src"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/BertSum/src/BertSum/src\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "57EhHSucJUsC",
        "colab_type": "code",
        "outputId": "a298da4a-1a49-4dcd-954a-59bfa1a00b4e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "os.listdir( )"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['models', 'train.py', 'others', 'distributed.py', 'prepro', 'preprocess.py']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o7Yui_wyJd0A",
        "colab_type": "code",
        "outputId": "69808676-a445-4045-c62c-4d1716a5b54e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 164
        }
      },
      "source": [
        "os.listdir( '../../gdrive/My Drive/FinalBertSummarizationDataPart1')[0:10]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "error",
          "ename": "FileNotFoundError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-44-f161e89a0e49>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlistdir\u001b[0m\u001b[0;34m(\u001b[0m \u001b[0;34m'../../gdrive/My Drive/FinalBertSummarizationDataPart1'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '../../gdrive/My Drive/FinalBertSummarizationDataPart1'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2jqTZfH8GOCq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#batch size 1024 with 50000 went over 12 hours, maybe 12:25\n",
        "\n",
        "# 4096"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yKa8qvIuMmHe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "os.path.abspath('../../gdrive/My Drive/BertSumCheckPointFiles')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vvwOq61hD_wh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#first time training no, 'train_from'. Updated for SciBert\n",
        "\n",
        "# !python train.py -mode train -encoder transformer -dropout 0.2 -bert_data_path  '/content/gdrive/My Drive/FinalBertSummarizationDataPart1/Files' -model_path '/content/gdrive/My Drive/BertSumCheckPointFiles' -bert_config_path '/content/scibert_scivocab_uncased/bert_config.json' -pretrained_dir '/content/scibert_scivocab_uncased/weights.tar.gz' -lr 2e-3 -visible_gpus 0 -gpu_ranks 0 -world_size 1 -report_every 500 -save_checkpoint_steps 50000 -batch_size 512 -decay_method noam -train_steps 50000 -accum_count 2 -log_file bert_transformer -use_interval true -warmup_steps 100 -ff_size 2048 -inter_layers 2 -heads 8"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vlhhR8tcM-yo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "os.listdir()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vwULA5nVPuZL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Figure out which GPU used\n",
        "!nvidia-smi"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hpPmHNyYQxD-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kzp3hn-xJrLb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Training from checkpoint. Updated for the SciBert pretrained and config. \n",
        "\n",
        "#There are one of three training files directories \n",
        "# bertDataPath = '/content/gdrive/My Drive/FinalBertSummarizationDataPart1/Files'\n",
        "bertDataPath = '/content/gdrive/My Drive/FinalBertSummarizationDataPart2/Files'\n",
        "# bertDataPath = '/content/gdrive/My Drive/FinalBertSummarizationDataPart3/Files'\n",
        "\n",
        "# Try os.environ['trainSave'] = 160000\n",
        "\n",
        "# trainSave = 180000\n",
        "\n",
        "modelPath = '/content/gdrive/My Drive/BertSumCheckPointFiles'\n",
        "\n",
        "bertConfigPath = '/content/scibert_scivocab_uncased/bert_config.json'\n",
        "\n",
        "pretrainedDir = '/content/scibert_scivocab_uncased/weights.tar.gz'\n",
        "\n",
        "trainFrom = '/content/gdrive/My Drive/BertSumCheckPointFiles/model_step_280000.pt'\n",
        "\n",
        "# batchSize = 8192\n",
        "\n",
        "!python train.py -mode train -encoder transformer -dropout 0.2 -bert_data_path '$bertDataPath' \\\n",
        "-model_path '$modelPath' -bert_config_path '$bertConfigPath' -pretrained_dir '$pretrainedDir' \\\n",
        "-train_from '$trainFrom' -lr 2e-3 -visible_gpus 0 -gpu_ranks 0 -world_size 1 \\\n",
        "-report_every 1000 -save_checkpoint_steps 285000 -batch_size 4096 -decay_method noam -train_steps 285000 \\\n",
        "-accum_count 2 -log_file bert_transformer -use_interval true -warmup_steps 100 -ff_size 2048 -inter_layers 2 -heads 8"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XpWZdn5x-ZvW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from tensorflow.python.client import device_lib\n",
        "\n",
        "device_lib.list_local_devices()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8NQaDIUI-I6a",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1OzCL4mbp4QZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# !python train.py -mode train -encoder transformer -dropout 0.2 -bert_data_path  '../../gdrive/My Drive/FinalBertSummarizationDataPart1/Files' -model_path '../../gdrive/My Drive/BertSumCheckPointFiles' -train_from 'model_step_5000.pt' -lr 2e-3 -visible_gpus 0 -gpu_ranks 0 -world_size 1 -report_every 100 -save_checkpoint_steps 50 -batch_size 512 -decay_method noam -train_steps 50 -accum_count 2 -log_file bert_transformer -use_interval true -warmup_steps 100 -ff_size 2048 -inter_layers 2 -heads 8"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rf8a5rfj9hxQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip show pyrouge"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z4GkehNh_XC_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pyrouge_set_rouge_path '/usr/local/lib/python3.6/dist-packages/pyrouge/tests'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J6fXF8XUKRpe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!git clone https://github.com/andersjo/pyrouge.git"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A6pKAsz8KjRX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pyrouge_set_rouge_path 'pyrouge/tools/ROUGE-1.5.5'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jXWooplLF61L",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#'/content/gdrive/My Drive/BertSumCheckPointFiles'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kn2F_nx4OgDl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# bertDataPath = '/content/gdrive/My Drive/MiniTest/Files'\n",
        "\n",
        "# trainSave = 150000\n",
        "\n",
        "# modelPath = '/content/gdrive/My Drive/BertSumCheckPointFiles/'\n",
        "\n",
        "# bertConfigPath = '/content/scibert_scivocab_uncased/bert_config.json'\n",
        "\n",
        "# pretrainedDir = '/content/scibert_scivocab_uncased/weights.tar.gz'\n",
        "\n",
        "# trainFrom = '/content/gdrive/My Drive/BertSumCheckPointFiles/model_step_130000.pt'\n",
        "\n",
        "# logFile = '../logs/bert_transformer'\n",
        "\n",
        "# testFrom = '/content/gdrive/My Drive/BertSumCheckPointFiles/model_step_130000.pt'\n",
        "\n",
        "# resultsPath = '/content/gdrive/My Drive/BertSumResults/'\n",
        "\n",
        "# !python train.py -mode test -bert_data_path '$bertDataPath' -model_path '$modelPath' \\\n",
        "# -bert_config_path '$bertConfigPath' -pretrained_dir '$pretrainedDir' -visible_gpus 0 -gpu_ranks 0 \\\n",
        "# -batch_size 2000 -log_file '$logFile' -result_path '$resultsPath' \\\n",
        "# -block_trigram true -test_from '$testFrom'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JE_hUEg1wa0L",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!apt-get install libmagic-dev\n",
        "!pip install python-magic\n",
        "import magic\n",
        "mime = magic.Magic(mime=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xAegtSgIwas7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mime.from_file('/content/gdrive/My Drive/BertSumResults_step60000.candidate')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LoaQnMcLxYqz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open('/content/gdrive/My Drive/BertSumResults_step130000.candidate') as myfile:\n",
        "    for i in range(10):\n",
        "        print( next(myfile))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "mfAGRAjiyPzy",
        "colab": {}
      },
      "source": [
        "with open('/content/gdrive/My Drive/BertSumResults_step130000.gold') as myfile:\n",
        "    for i in range(10):\n",
        "        print( next(myfile))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tnZmkAq8waiN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "os.listdir('../temp/tmpvw1am0ff/model')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-V0L_Gp-SbbC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j66MgZH32zeq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "dataset = torch.load('../../gdrive/My Drive/MiniTest/Files.test.12.bert.pt')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k73ymAyxSf0p",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(dataset[13])"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}