finetune_kogpt_trinity.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define Paths that is reconcilable both at GPU server and at Local"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/opt/ml/kor-3-line-poetry\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "# root path\n",
    "ROOT_PATH = os.path.abspath(\".\") # this makes compatible absolute path both for local and server\n",
    "\n",
    "# designate root path for the data\n",
    "DATA_ROOT_PATH = os.path.join(ROOT_PATH, 'data')\n",
    "\n",
    "# designate path for each dataset files\n",
    "LYRIC_PATH = os.path.join(DATA_ROOT_PATH, \"lyrics_kor.txt\")\n",
    "BILLBOARD_PATH = os.path.join(DATA_ROOT_PATH, \"rawdata_김지훈_201500844.tsv\")\n",
    "GEULSTAGRAM_PATH = os.path.join(DATA_ROOT_PATH, \"geulstagram.csv\")\n",
    "\n",
    "print(ROOT_PATH)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define Train & Evaluation configruation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'DEBUG': False, 'num_workers': 4, 'train_batch_size': 16, 'user_name': 'snoop2head', 'file_base_name': 'snoop2head_1115_17:20', 'model_dir': 'skt/ko-gpt-trinity-1.2B-v0.5', 'max_token_length': 42, 'learning_rate': 5e-05, 'weight_decay': 0.01, 'save_steps': 500, 'early_stopping_patience': 5, 'warmup_steps': 500, 'logging_steps': 100, 'evaluation_strategy': 'epoch', 'evaluation_steps': 500, 'result_dir': '/opt/ml/kor-3-line-poetry/results', 'saved_model_dir': '/opt/ml/kor-3-line-poetry/best_models', 'logging_dir': '/opt/ml/kor-3-line-poetry/logs', 'baseline_dir': '/opt/ml/kor-3-line-poetry/baseline-code'}\n"
     ]
    }
   ],
   "source": [
    "from datetime import datetime\n",
    "from easydict import EasyDict\n",
    "\n",
    "# Initialize configuration\n",
    "CFG = EasyDict()\n",
    "\n",
    "# Dataset Config as constants\n",
    "CFG.DEBUG = False\n",
    "CFG.num_workers = 4\n",
    "CFG.train_batch_size = 16\n",
    "\n",
    "# Train configuration\n",
    "CFG.user_name = \"snoop2head\"\n",
    "today = datetime.now().strftime(\"%m%d_%H:%M\")\n",
    "CFG.file_base_name = f\"{CFG.user_name}_{today}\"\n",
    "CFG.model_dir = \"skt/ko-gpt-trinity-1.2B-v0.5\" # designate the model's name registered on huggingface: https://huggingface.co/skt/ko-gpt-trinity-1.2B-v0.5\n",
    "CFG.max_token_length = 42\n",
    "CFG.learning_rate = 5e-5\n",
    "CFG.weight_decay = 1e-2 # https://paperswithcode.com/method/weight-decay and https://github.com/clovaai/AdamP\n",
    "\n",
    "# training steps configurations\n",
    "CFG.save_steps = 500\n",
    "CFG.early_stopping_patience = 5\n",
    "CFG.warmup_steps = 500\n",
    "CFG.logging_steps = 100\n",
    "CFG.evaluation_strategy = 'epoch'\n",
    "CFG.evaluation_steps = 500\n",
    "\n",
    "# Directory configuration\n",
    "CFG.result_dir = os.path.join(ROOT_PATH, \"results\")\n",
    "CFG.saved_model_dir = os.path.join(ROOT_PATH, \"best_models\")\n",
    "CFG.logging_dir = os.path.join(ROOT_PATH, \"logs\")\n",
    "CFG.baseline_dir = os.path.join(ROOT_PATH, 'baseline-code')\n",
    "\n",
    "print(CFG)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import torch\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"true\"\n",
    "os.environ[\"CUDA_LAUNCH_BLOCKING\"] = \"1\"\n",
    "os.environ[\"WANDB_NOTEBOOK_NAME\"] = \"kor-3-line-poetry\"\n",
    "\n",
    "def seed_everything(seed) :\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.cuda.manual_seed_all(seed) # if use multi-GPU\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    torch.backends.cudnn.benchmark = False\n",
    "    np.random.seed(seed)\n",
    "    random.seed(seed)\n",
    "seed_everything(42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read txt file from line by line\n",
    "def read_txt(path):\n",
    "    with open(path, 'r', encoding='utf-8') as f:\n",
    "        lines = f.readlines()\n",
    "    return lines\n",
    "\n",
    "# make sampling function from the list\n",
    "def sampling(list_lines:list, n:int) -> list:\n",
    "    # sampling\n",
    "    list_lines = np.random.choice(list_lines, n)\n",
    "    list_lines = list(list_lines)\n",
    "    return list_lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total dataset length: 82989\n"
     ]
    }
   ],
   "source": [
    "# read dataset from poetic_sentences_kor.txt\n",
    "path = os.path.join(DATA_ROOT_PATH, \"poetic_sentences_kor.txt\")\n",
    "list_loaded = read_txt(path)\n",
    "print(\"total dataset length:\", len(list_loaded))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of dataset that is in between 5 and 52 is 66872\n",
      "This is 80.58 % of total dataset\n",
      "Below are 20 samples of data to use\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['힘듦을 앞에 두고서 왜 무게를 재는 거예요.\\n',\n",
       " '다리 끝에 건너오지 못한 한 사람이 있습니다\\n',\n",
       " '잃고 나서야 가졌다는 걸 알았다.\\n',\n",
       " '가시 난 내 모습이 더 깊숙이 파고들 거야\\n',\n",
       " '오로지 너에게로 가는 길에 길을 내렴..\\n',\n",
       " '사랑하는 건 늦진 않았지\\n',\n",
       " '이 세상에 힘들지 않은 일 하나 없다지만 내가 제일 힘들다\\n',\n",
       " '그 꽃을 피우는 일에만 전념하면 돼요.\\n',\n",
       " '그러니 그가 옅게 웃었다.\\n',\n",
       " '나와 둘이서 다정했었던\\n',\n",
       " '할 수 있어서 다행이다\\n',\n",
       " '같이 있을 때는 누구나 당장 앞에 있는 사람에게 잘 할 수 있습니다.\\n',\n",
       " '우리 기약 없는 약속은 하지 말아요\\n',\n",
       " '아무 일도 없던 것처럼 내 마음 이리 덮이면 좋겠습니다.\\n',\n",
       " '목요일 카카오톡 지금 진석 이렇게 예쁘고 사랑스러운 네가 그동안 참 힘들었겠다.\\n',\n",
       " '어디 한번 둘러봐\\n',\n",
       " '나 자신에게 관심을 가지세요.\\n',\n",
       " '꽃에게 좋은 말을 해주면 나쁜 말을 해준 꽃보다 더 잘 자란대\\n',\n",
       " '완전 반해 반해 버렸어요\\n',\n",
       " '매장에 나가봤어요\\n']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get proportion of sentences where length is between 1 and 52\n",
    "min_char_length = 5\n",
    "max_char_length = 52\n",
    "list_to_use = list(filter(lambda x: len(x) > min_char_length and len(x) < max_char_length, list_loaded))\n",
    "\n",
    "print(f\"Length of dataset that is in between {min_char_length} and {max_char_length} is\" , len(list_to_use))\n",
    "print(f\"This is {round(len(list_to_use) / len(list_loaded) * 100, 2)} % of total dataset\")\n",
    "\n",
    "n = 20\n",
    "print(f\"Below are {n} samples of data to use\")\n",
    "sampling(list_to_use, n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "stopwords = []\n",
    "\n",
    "# make post_process function\n",
    "def post_process(list_lines:list, stopwords) -> list:\n",
    "    # remove \\n\n",
    "    removed_lines = [line.strip() for line in list_lines]\n",
    "\n",
    "    # filter stopwords from the line item in list_lines using regex\n",
    "    if len(stopwords) > 0:\n",
    "        removed_lines = []\n",
    "        for line in list_lines:\n",
    "            for stopword in stopwords:\n",
    "                line = re.sub(stopword, '', line)\n",
    "            removed_lines.append(line)\n",
    "\n",
    "    # remove newlines\n",
    "    removed_lines = [sentence.replace('\\n', '') for sentence in removed_lines]\n",
    "\n",
    "    # strip whitespace\n",
    "    removed_lines = [sentence.strip() for sentence in removed_lines]\n",
    "\n",
    "    # remove one letter items\n",
    "    removed_lines = [sentence for sentence in removed_lines if len(sentence) > 1]\n",
    "\n",
    "    return removed_lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "66872\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['당신이 떠나시던 그 밤에',\n",
       " '인생의 목적은 누가 그것을 지배하든 주변의 사랑할 만한 사람을 사랑하는 것이다.',\n",
       " '사소한 말 한마디에 깨져버린 마음 애써 끼워 맞추려 하지 말아요',\n",
       " '내 사랑은 잔에 남은 커피향 실체 없는 추억에 하는 입맞춤 잔향 열어지는 것',\n",
       " '그것 말과.는 기대하지 않았던 수하는 가 모라. 세이지 조개 헤드 있었으니 말이다.',\n",
       " '오늘은 집으로 돌아와 울어 버렸어요.',\n",
       " '모든 것들이 빈 들녘의 바람처럼 세월을 몰고 다만 멀어져 갔다',\n",
       " '그리고 너는 너무 오랫동안 안 보여 슬픔 따위는 아무것도 아니야',\n",
       " '이제 그대에게 말해줄게요',\n",
       " '거긴 미래도']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list_post_processed = post_process(list_to_use, stopwords)\n",
    "print(len(list_post_processed))\n",
    "\n",
    "n = 10\n",
    "sampling(list_post_processed, n)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### In order to create uniform length batch, determine maximum token length\n",
    "- max_token_length is different from max_char_length.\n",
    "- max_token_length is based on input_ids of pretrained GPT2Tokenizer\n",
    "- pad until the max_token_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The tokenizer class you load from this checkpoint is not the same type as the class this function is called from. It may result in unexpected tokenization. \n",
      "The tokenizer class you load from this checkpoint is 'GPT2Tokenizer'. \n",
      "The class this function is called from is 'PreTrainedTokenizerFast'.\n"
     ]
    }
   ],
   "source": [
    "from transformers import GPT2Tokenizer, PreTrainedTokenizerFast\n",
    "\n",
    "# Load the Tokenizer: \"Fast\" means that the tokenizer code is written in Rust Lang\n",
    "tokenizer = PreTrainedTokenizerFast.from_pretrained(\n",
    "    CFG.model_dir,\n",
    "    bos_token=\"<s>\",\n",
    "    eos_token=\"</s>\",\n",
    "    unk_token=\"<unk>\",\n",
    "    pad_token=\"<pad>\",\n",
    "    mask_token=\"<mask>\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the length of the individual items in tokenized input_ids\n",
    "tokenized_input_ids = tokenizer(list_post_processed).input_ids\n",
    "tokenized_length = [len(item) for item in tokenized_input_ids]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "41\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAEGCAYAAACkQqisAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAfFUlEQVR4nO3dfZhdZXnv8e+P8BZATCAj0iQ4CLGcaDGEEegBLcIxBKgmVrRwrETLRWoJKmpbgqeXvJkeqEdRqlCjRIJVQwCRHIzGCKFoFZIBQkgCHEYIJTGQSHiRqsHgff5Y98Bi2DOzszJ77xnn97mufc1a93rWWvdaSebOetnPo4jAzMysip1anYCZmQ1dLiJmZlaZi4iZmVXmImJmZpW5iJiZWWU7tzqBZhszZky0t7e3Og0zsyHlrrvu+mVEtPWMD7si0t7eTmdnZ6vTMDMbUiQ9Wivu21lmZlaZi4iZmVXmImJmZpW5iJiZWWUuImZmVpmLiJmZVeYiYmZmlbmImJlZZS4iZmZW2bD7xvqOaJ/9vVanMKSsu+TkVqdgZg3W8CsRSSMk3SPp5pw/UNKdkrokXStp14zvlvNduby9tI3zMv6gpBNK8akZ65I0u9HHYmZmL9eM21kfA+4vzV8KXBYRBwNPAWdk/AzgqYxflu2QNBE4FXgjMBW4IgvTCODLwInAROC0bGtmZk3S0CIiaRxwMvC1nBdwHHB9NpkPTM/paTlPLj8+208DFkTE1oh4BOgCjshPV0Q8HBHPAwuyrZmZNUmjr0S+APwD8Puc3xd4OiK25fx6YGxOjwUeA8jlz2T7F+M91ukt/gqSZkrqlNS5efPmHTwkMzPr1rAiIunPgU0RcVej9lGviJgbER0R0dHW9oru8M3MrKJGvp11NPAuSScBuwN7A18ERknaOa82xgEbsv0GYDywXtLOwKuBJ0vxbuV1eoubmVkTNOxKJCLOi4hxEdFO8WD81oh4P7AMOCWbzQBuyulFOU8uvzUiIuOn5ttbBwITgOXACmBCvu21a+5jUaOOx8zMXqkV3xM5F1gg6TPAPcBVGb8K+IakLmALRVEgItZIWgisBbYBsyLiBQBJZwNLgBHAvIhY09QjMTMb5ppSRCLiNuC2nH6Y4s2qnm1+C7y3l/XnAHNqxBcDiwcwVTMz2w7u9sTMzCpzETEzs8pcRMzMrDIXETMzq8xFxMzMKnMRMTOzylxEzMysMhcRMzOrzEXEzMwqcxExM7PKXETMzKwyFxEzM6vMRcTMzCpzETEzs8pcRMzMrDIXETMzq6xhRUTS7pKWS7pX0hpJF2b8akmPSFqZn0kZl6TLJXVJWiVpcmlbMyQ9lJ8Zpfjhku7LdS6XpEYdj5mZvVIjRzbcChwXEc9J2gX4iaTv57K/j4jre7Q/kWL89AnAkcCVwJGS9gHOBzqAAO6StCginso2ZwJ3UoxwOBX4PmZm1hQNuxKJwnM5u0t+oo9VpgHX5Hp3AKMk7Q+cACyNiC1ZOJYCU3PZ3hFxR0QEcA0wvVHHY2Zmr9TQZyKSRkhaCWyiKAR35qI5ecvqMkm7ZWws8Fhp9fUZ6yu+vka8Vh4zJXVK6ty8efOOHpaZmaWGFpGIeCEiJgHjgCMkvQk4DzgEeAuwD3BuI3PIPOZGREdEdLS1tTV6d2Zmw0ZT3s6KiKeBZcDUiNiYt6y2Al8HjshmG4DxpdXGZayv+LgacTMza5JGvp3VJmlUTo8E3gE8kM8yyDeppgOrc5VFwOn5ltZRwDMRsRFYAkyRNFrSaGAKsCSXPSvpqNzW6cBNjToeMzN7pUa+nbU/MF/SCIpitTAibpZ0q6Q2QMBK4MPZfjFwEtAF/Br4EEBEbJF0MbAi210UEVty+izgamAkxVtZfjPLzKyJGlZEImIVcFiN+HG9tA9gVi/L5gHzasQ7gTftWKZmZlaVv7FuZmaVuYiYmVllLiJmZlaZi4iZmVXmImJmZpW5iJiZWWUuImZmVpmLiJmZVeYiYmZmlbmImJlZZS4iZmZWmYuImZlV5iJiZmaVuYiYmVllLiJmZlaZi4iZmVXWyOFxd5e0XNK9ktZIujDjB0q6U1KXpGsl7Zrx3XK+K5e3l7Z1XsYflHRCKT41Y12SZjfqWMzMrLZGXolsBY6LiDcDk4CpOXb6pcBlEXEw8BRwRrY/A3gq45dlOyRNBE4F3ghMBa6QNCKH3f0ycCIwETgt25qZWZM0rIhE4bmc3SU/ARwHXJ/x+cD0nJ6W8+Ty4yUp4wsiYmtEPEIxBvsR+emKiIcj4nlgQbY1M7MmaegzkbxiWAlsApYCPweejoht2WQ9MDanxwKPAeTyZ4B9y/Ee6/QWNzOzJmloEYmIFyJiEjCO4srhkEburzeSZkrqlNS5efPmVqRgZvYHqSlvZ0XE08Ay4E+BUZJ2zkXjgA05vQEYD5DLXw08WY73WKe3eK39z42IjojoaGtrG4hDMjMzGvt2VpukUTk9EngHcD9FMTklm80AbsrpRTlPLr81IiLjp+bbWwcCE4DlwApgQr7ttSvFw/dFjToeMzN7pZ37b1LZ/sD8fItqJ2BhRNwsaS2wQNJngHuAq7L9VcA3JHUBWyiKAhGxRtJCYC2wDZgVES8ASDobWAKMAOZFxJoGHo+ZmfXQsCISEauAw2rEH6Z4PtIz/lvgvb1saw4wp0Z8MbB4h5M1M7NK/I11MzOrzEXEzMwqcxExM7PKXETMzKwyFxEzM6vMRcTMzCpzETEzs8pcRMzMrDIXETMzq8xFxMzMKnMRMTOzylxEzMysMhcRMzOrzEXEzMwqcxExM7PK6ioikv6k0YmYmdnQU++VyBWSlks6S9KrG5qRmZkNGXUVkYh4K/B+YDxwl6RvSXpHX+tIGi9pmaS1ktZI+ljGL5C0QdLK/JxUWuc8SV2SHpR0Qik+NWNdkmaX4gdKujPj1+ZY62Zm1iR1PxOJiIeAfwTOBf4MuFzSA5L+opdVtgGfjIiJwFHALEkTc9llETEpP4sBctmpwBuBqRRXPyNyjPYvAycCE4HTStu5NLd1MPAUcEbdR25mZjus3mcih0q6DLgfOA54Z0T8t5y+rNY6EbExIu7O6V/lumP72M00YEFEbI2IR4AuirHYjwC6IuLhiHgeWABMk6Tc//W5/nxgej3HY2ZmA6PeK5F/Ae4G3hwRs0rF4RcUVyd9ktQOHAbcmaGzJa2SNE/S6IyNBR4rrbY+Y73F9wWejohtPeK19j9TUqekzs2bN/d7sGZmVp96i8jJwLci4jcAknaStAdARHyjrxUl7QXcAJwTEc8CVwIHAZOAjcDnqqVev4iYGxEdEdHR1tbW6N2ZmQ0b9RaRHwEjS/N7ZKxPknahKCDfjIjvAETEExHxQkT8Hvgqxe0qgA0UD+67jctYb/EngVGSdu4RNzOzJqm3iOweEc91z+T0Hn2tkM8srgLuj4jPl+L7l5q9G1id04uAUyXtJulAYAKwHFgBTMg3sXalePi+KCICWAackuvPAG6q83jMzGwA7Nx/EwD+S9Lk7mchkg4HftPPOkcDHwDuk7QyY5+ieLtqEhDAOuBvACJijaSFwFqKN7tmRcQLub+zgSXACGBeRKzJ7Z0LLJD0GeAeiqJlZmZNUm8ROQe4TtIvAAGvBf6yrxUi4ifZtqfFfawzB5hTI7641noR8TAv3Q4zM7Mmq6uIRMQKSYcAf5yhByPid41Ly8zMhoJ6r0QA3gK05zqTJRER1zQkK/uD0D77e61OYUhZd8nJrU7BbLvVVUQkfYPitdyVwAsZDsBFxMxsGKv3SqQDmJhvRJmZmQH1v+K7muJhupmZ2YvqvRIZA6yVtBzY2h2MiHc1JCszMxsS6i0iFzQyCTMzG5rqfcX33yW9DpgQET/KfrNGNDY1MzMb7OrtCv5Mii7Xv5KhscB3G5STmZkNEfU+WJ9F0Y3Js/DiAFWvaVRSZmY2NNRbRLbmgFAAZM+5ft3XzGyYq7eI/LukTwEjc2z164D/27i0zMxsKKi3iMwGNgP3UfS6u5g6RjQ0M7M/bPW+ndU9gNRXG5uOmZkNJfX2nfUINZ6BRMTrBzwjMzMbMran76xuuwPvBfYZ+HTMzGwoqeuZSEQ8WfpsiIgvAH32Wy1pvKRlktZKWiPpYxnfR9JSSQ/lz9EZl6TLJXVJWiVpcmlbM7L9Q5JmlOKHS7ov17k8h+Q1M7MmqffLhpNLnw5JH6b/q5htwCcjYiJwFDBL0kSKh/S3RMQE4JacBziRYlz1CcBM4Mrc9z7A+cCRFKMYnt9deLLNmaX1ptZzPGZmNjDqvZ31udL0Noqx0d/X1woRsRHYmNO/knQ/xTfdpwHHZrP5wG0UY6VPA67J7ubvkDRK0v7ZdmlEbAGQtBSYKuk2YO+IuCPj1wDTge/XeUxmZraD6n076+07shNJ7cBhwJ3AfllgAB4H9svpscBjpdXWZ6yv+Poa8Vr7n0lxdcMBBxywA0diZmZl9b6d9Ym+lkfE5/tYdy/gBuCciHi2/NgiIkJSw7/5HhFzgbkAHR0d/qa9mdkAqffLhh3A3/LSFcCHgcnAq/JTk6RdKArINyPiOxl+Im9TkT83ZXwDML60+riM9RUfVyNuZmZNUm8RGQdMjohPRsQngcOBAyLiwoi4sNYK+abUVcD9Pa5UFgHdb1jNAG4qxU/Pt7SOAp7J215LgCmSRucD9SnAklz2rKSjcl+nl7ZlZmZNUO+D9f2A50vzz/PSs4zeHA18ALhP0sqMfQq4BFgo6QzgUV56QL8YOAnoAn4NfAggIrZIuhhYke0u6n7IDpwFXA2MpHig7ofqZmZNVG8RuQZYLunGnJ9O8WZVryLiJ0Bv39s4vkb7oOhyvta25gHzasQ7gTf1lYeZmTVOvW9nzZH0feCtGfpQRNzTuLTMzGwoqPeZCMAewLMR8UVgvaQDG5STmZkNEfV+Y/18ii8EnpehXYB/a1RSZmY2NNR7JfJu4F3AfwFExC/o49VeMzMbHuotIs/ng+8AkLRn41IyM7Ohot4islDSV4BRks4EfoQHqDIzG/b6fTsrv8h3LXAI8Czwx8CnI2Jpg3MzM7NBrt8ikv1bLY6IPwFcOMzM7EX13s66W9JbGpqJmZkNOfV+Y/1I4K8kraN4Q0sUFymHNioxMzMb/PosIpIOiIj/BE5oUj5mZjaE9Hcl8l2K3nsflXRDRLynCTmZmdkQ0d8zkXIHiq9vZCJmZjb09FdEopdpMzOzfm9nvVnSsxRXJCNzGl56sL53Q7MzM7NBrc8iEhEjmpWImZkNPdvTFbyZmdnLNKyISJonaZOk1aXYBZI2SFqZn5NKy86T1CXpQUknlOJTM9YlaXYpfqCkOzN+raRdG3UsZmZWWyOvRK4GptaIXxYRk/KzGEDSROBU4I25zhWSRkgaAXwZOBGYCJyWbQEuzW0dDDwFnNHAYzEzsxoaVkQi4nZgS53NpwELImJrRDwCdAFH5KcrIh6OiOeBBcC07BTyOOD6XH8+xbjvZmbWRK14JnK2pFV5u2t0xsYCj5XarM9Yb/F9gacjYluPeE2SZkrqlNS5efPmgToOM7Nhr9lF5ErgIGASsBH4XDN2GhFzI6IjIjra2tqasUszs2Gh3g4YB0REPNE9LemrwM05uwEYX2o6LmP0En+SYoCsnfNqpNzezMyapKlXIpL2L82+G+h+c2sRcKqk3SQdCEwAlgMrgAn5JtauFA/fF+VQvcuAU3L9GcBNzTgGMzN7ScOuRCR9GzgWGCNpPXA+cKykSRRdqKwD/gYgItZIWgisBbYBsyLihdzO2cASYAQwLyLW5C7OBRZI+gxwD3BVo47FzMxqa1gRiYjTaoR7/UUfEXOAOTXii4HFNeIPU7y9ZWZmLeJvrJuZWWUuImZmVpmLiJmZVeYiYmZmlbmImJlZZS4iZmZWmYuImZlV5iJiZmaVuYiYmVllLiJmZlaZi4iZmVXmImJmZpW5iJiZWWUuImZmVpmLiJmZVeYiYmZmlTWsiEiaJ2mTpNWl2D6Slkp6KH+OzrgkXS6pS9IqSZNL68zI9g9JmlGKHy7pvlzncklq1LGYmVltjbwSuRqY2iM2G7glIiYAt+Q8wIkU46pPAGYCV0JRdCiG1T2SYhTD87sLT7Y5s7Rez32ZmVmDNayIRMTtwJYe4WnA/JyeD0wvxa+Jwh3AKEn7AycASyNiS0Q8BSwFpuayvSPijogI4JrStszMrEma/Uxkv4jYmNOPA/vl9FjgsVK79RnrK76+RrwmSTMldUrq3Lx5844dgZmZvahlD9bzCiKatK+5EdERER1tbW3N2KWZ2bDQ7CLyRN6KIn9uyvgGYHyp3biM9RUfVyNuZmZN1OwisgjofsNqBnBTKX56vqV1FPBM3vZaAkyRNDofqE8BluSyZyUdlW9lnV7alpmZNcnOjdqwpG8DxwJjJK2neMvqEmChpDOAR4H3ZfPFwElAF/Br4EMAEbFF0sXAimx3UUR0P6w/i+INsJHA9/NjZmZN1LAiEhGn9bLo+BptA5jVy3bmAfNqxDuBN+1IjmZmtmP8jXUzM6vMRcTMzCpzETEzs8pcRMzMrDIXETMzq8xFxMzMKnMRMTOzylxEzMysMhcRMzOrzEXEzMwqcxExM7PKXETMzKwyFxEzM6vMRcTMzCpzETEzs8pcRMzMrLKWFBFJ6yTdJ2mlpM6M7SNpqaSH8ufojEvS5ZK6JK2SNLm0nRnZ/iFJM3rbn5mZNUYrr0TeHhGTIqIj52cDt0TEBOCWnAc4EZiQn5nAlVAUHYohd48EjgDO7y48ZmbWHIPpdtY0YH5Ozweml+LXROEOYJSk/YETgKURsSUingKWAlObnLOZ2bDWqiISwA8l3SVpZsb2i4iNOf04sF9OjwUeK627PmO9xV9B0kxJnZI6N2/ePFDHYGY27O3cov0eExEbJL0GWCrpgfLCiAhJMVA7i4i5wFyAjo6OAduumdlw15IrkYjYkD83ATdSPNN4Im9TkT83ZfMNwPjS6uMy1lvczMyapOlFRNKekl7VPQ1MAVYDi4DuN6xmADfl9CLg9HxL6yjgmbzttQSYIml0PlCfkjEzM2uSVtzO2g+4UVL3/r8VET+QtAJYKOkM4FHgfdl+MXAS0AX8GvgQQERskXQxsCLbXRQRW5p3GGZm1vQiEhEPA2+uEX8SOL5GPIBZvWxrHjBvoHM0M7P6DKZXfM3MbIhxETEzs8pcRMzMrDIXETMzq8xFxMzMKnMRMTOzylxEzMysMhcRMzOrrFUdMJpZD+2zv9fqFIaUdZec3OoUDF+JmJnZDnARMTOzylxEzMysMhcRMzOrzEXEzMwqcxExM7PKXETMzKwyFxEzM6tsyBcRSVMlPSipS9LsVudjZjacDOkiImkE8GXgRGAicJqkia3Nysxs+BjSRQQ4AuiKiIcj4nlgATCtxTmZmQ0bQ73vrLHAY6X59cCRPRtJmgnMzNnnJD1YcX9jgF9WXLeRnNf2cV7bZ1DmpUsHZ14M0vPFjuf1ulrBoV5E6hIRc4G5O7odSZ0R0TEAKQ0o57V9nNf2cV7bZ7jlNdRvZ20Axpfmx2XMzMyaYKgXkRXABEkHStoVOBVY1OKczMyGjSF9Oysitkk6G1gCjADmRcSaBu5yh2+JNYjz2j7Oa/s4r+0zrPJSRDRiu2ZmNgwM9dtZZmbWQi4iZmZWmYtIHQZz1yqS1km6T9JKSZ0tzGOepE2SVpdi+0haKumh/Dl6kOR1gaQNec5WSjqpBXmNl7RM0lpJayR9LOMtPWd95NXScyZpd0nLJd2beV2Y8QMl3Zn/Nq/NF2wGQ15XS3qkdL4mNTOvzGGEpHsk3ZzzjTlXEeFPHx+KB/Y/B14P7ArcC0xsdV6l/NYBYwZBHm8DJgOrS7F/Bmbn9Gzg0kGS1wXA37X4fO0PTM7pVwH/j6Lrnpaesz7yauk5AwTsldO7AHcCRwELgVMz/q/A3w6SvK4GTmnx37FPAN8Cbs75hpwrX4n0z12r1CEibge29AhPA+bn9HxgejNzgl7zarmI2BgRd+f0r4D7KXpgaOk56yOvlorCczm7S34COA64PuOtOF+95dVSksYBJwNfy3nRoHPlItK/Wl2rtPwfVUkAP5R0V3bvMpjsFxEbc/pxYL9WJtPD2ZJW5e2upt9mK5PUDhxG8b/YQXPOeuQFLT5neXtmJbAJWEpxh+DpiNiWTVryb7NnXhHRfb7m5Pm6TNJuTU7rC8A/AL/P+X1p0LlyERn6jomIyRQ9Gc+S9LZWJ1RLFNfQLf8fWroSOAiYBGwEPteqRCTtBdwAnBMRz5aXtfKc1cir5ecsIl6IiEkUPVMcARzS7Bxq6ZmXpDcB51Hk9xZgH+DcZuUj6c+BTRFxVzP25yLSv0HdtUpEbMifm4AbKf5xDRZPSNofIH9uanE+AETEE/kP//fAV2nROZO0C8Uv6m9GxHcy3PJzViuvwXLOMpengWXAnwKjJHV/abql/zZLeU3N24IREVuBr9Pc83U08C5J6yhuvx8HfJEGnSsXkf4N2q5VJO0p6VXd08AUYHXfazXVImBGTs8AbmphLi/q/iWd3k0Lzlneo74KuD8iPl9a1NJz1lterT5nktokjcrpkcA7KJ7XLANOyWatOF+18nqg9B8BUTx7aNr5iojzImJcRLRT/L66NSLeT6POVSvfHhgqH+AkirdUfg78r1bnU8rr9RRvi90LrGllbsC3KW5z/I7ifusZFPdhbwEeAn4E7DNI8voGcB+wiuKX9v4tyOsYiltVq4CV+Tmp1eesj7xaes6AQ4F7cv+rgU9n/PXAcqALuA7YbZDkdWuer9XAv5FvcLXg79mxvPR2VkPOlbs9MTOzynw7y8zMKnMRMTOzylxEzMysMhcRMzOrzEXEzMwqcxGxHSbpuf5b7dD2z5G0x0DsT9Jukn6UPav+ZY9lH5T0R6X5dZLGVNxPu6T/WUe7D0r6UpV99NjOuzQAPUxLuk1Sx45up5dtj5J0ViO2ba3jImJDwTnAHv01qtNhABExKSKu7bHsg8AfvWKNatqBfovIQImIRRFxSbP2V9EowEXkD4yLiDWEpIMk/SA7hvyxpEMyfrWkyyX9VNLDkk7J+E6SrpD0gIpxNBZLOkXSRyl+sS+TtKy0/Tk5hsMdkl7RSaGKcTm+mx3g3SHpUEmvofji11vySuSgUvtTgA7gm7lsZC76iKS7VYzZ0n0Me2YnhMtzvIZavTpfArw1t/VxFeNOfD23c4+kt9fI+WRJP5M0RtKUnL5b0nXZl1X31dGFNXJ68YpGL41hsVLSbyT9WW85SxopaYGk+yXdCIzsmVe2u0TFGCOrJP2fjLVJukHSivwcnfELcl+35Z/xR0vn5KDM67PZ9u9z3VV6aSyO9sznqyrG6Phh95+HpIPzSvLePAcH9bYda5JWfIvSnz+sD/BcjdgtwIScPpKi6wUoxlm4juI/MBMputmHojuGxRl/LfAUOR4DPcZMofhG9Ttz+p+Bf6yx/38Bzs/p44CVOX0s+Q3eGuvcBnSU5tcBH8nps4Cv5fQ/AX+V06MoejPYs8e2XrYf4JPAvJw+BPhPYHeKq58vUXQl8mNgNDAGuL17mxSd9326n5w+CHypRw7vzG3u0lvOFGNOdOd1KLCtfA4yvi/wILz45eRR+fNbFB2AAhxA0VUKFGOP/BTYLY/lycyhnZeP6zIFmEsxJsdOwM0U47+0Zx6Tst3CUu53Au/O6d0prlBrbqfV/y6Gy6e7My6zAZP/a/7vwHWSusPlrrC/G0VHfmtLVxHHANdl/PHyVUcNz1P8ogC4i6K/op6OAd4DEBG3StpX0t4VDqe7Y8S7gL/I6SkUHdz9Xc7vTv4S7WM7x1AUNiLiAUmPAm/IZcdRXAVNiYhnVfTCOhH4jzx/uwI/6yenl5E0Afgs8PaI+J2k3nJ+G3B55rVK0qoam3sG+C1wlYpR8rrP/f8AJpb+jPfuvmICvhdF54NbJW2idpf2U/JzT87vBUygKLCPRMTK0nG2q+gnbmxE3Jj5/jaPtbft3F7r3NjAchGxRtiJYuyCSb0s31qaVi9t+vK7yP+KAi/Q2L/H3bmW9yPgPRHx4ADto3vkzDcAnbn9pRFx2nbk9KL8Rb4QODNeGpukZs6lAtCriNgm6QjgeIorxrMpCt9OwFHdv8x7bLP8Z9zbn5GA/x0RX+mxfnuN9WveZutrO9YcfiZiAy6K8ScekfReKHoylfTmflb7D+A9Kp6N7EdxO6jbryiGat0ePwben/s/Fvhl9Bivo4Z697OE4lmJcvuH1bGtcj5voLgK6P6F/ijFVdM1kt4I3AEcLengbL9nrlOvecDXI+LHdeR8O/kCgIpxMA7tubEsSq+OiMXAx4HuP8sfAh8ptZvUT149z8kS4K9Lz3vGqnhuVVMUIy2ulzQ92++m4q297dqODSwXERsIe0haX/p8guIX5hmSunsY7m9I4RsoetldS/Hw+26K2yhQ3O/+QT+3uHq6ADg8b89cwkvdq/flauBf9fIH67VcTHGPf5WkNTnf0yrghXwA/HHgCmAnSfcB1wIfzNs9QHGLi+KcXQfsTfGM49uZ/8+ocwAmSa+juFr469LD9Y4+cr4S2EvS/cBFFLeOenoVcHPm8hOK5ygAHwU68mH2WuDDfeUWEU9S3KJbLemzEfFDiucqP8vzcj39F/EPAB/NXH4KvLbidmyAuBdfGzQk7RURz0nal6LL6qMj4vFW52VmvfMzERtMblYxwM+uwMUuIGaDn69EzMysMj8TMTOzylxEzMysMhcRMzOrzEXEzMwqcxExM7PK/j/wTxgw5f2c1wAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "# check the length distribution of the list with x ticks divided by 10 tokens\n",
    "plt.hist(tokenized_length, bins=np.arange(0, max(tokenized_length)+1, 10))\n",
    "print(max(tokenized_length)+1)\n",
    "plt.xlabel(\"Length of the tokenized sentence\")\n",
    "plt.ylabel(\"Frequency\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "66872 0.8057935389027462\n"
     ]
    }
   ],
   "source": [
    "# get proportion of sentences where length is between 0 and 32\n",
    "min_token_length = 0\n",
    "max_token_length = 42\n",
    "list_between = list(filter(lambda x: x > min_token_length and x < max_token_length, tokenized_length))\n",
    "print(len(list_between) , len(list_between) / len(list_loaded))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# you can overwrite with the length you desire\n",
    "max_token_length = 42\n",
    "CFG.max_token_length = max_token_length"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fine-tuning "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The tokenizer class you load from this checkpoint is not the same type as the class this function is called from. It may result in unexpected tokenization. \n",
      "The tokenizer class you load from this checkpoint is 'GPT2Tokenizer'. \n",
      "The class this function is called from is 'PreTrainedTokenizerFast'.\n"
     ]
    }
   ],
   "source": [
    "# https://huggingface.co/transformers/preprocessing.html\n",
    "# Load the Tokenizer: \"Fast\" means that the tokenizer code is written in Rust Lang\n",
    "tokenizer = PreTrainedTokenizerFast.from_pretrained(\n",
    "    CFG.model_dir,\n",
    "    max_len = CFG.max_token_length,\n",
    "    padding='max_length',\n",
    "    add_special_tokens = True,\n",
    "    return_tensors=\"pt\",\n",
    "    truncation = True,\n",
    "    bos_token = \"<s>\",\n",
    "    eos_token = \"</s>\",\n",
    "    unk_token = \"<unk>\",\n",
    "    pad_token = \"<pad>\",\n",
    "    mask_token = \"<mask>\",\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define model configuration and model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GPT2Config {\n",
       "  \"activation_function\": \"gelu_new\",\n",
       "  \"architectures\": [\n",
       "    \"GPT2LMHeadModel\"\n",
       "  ],\n",
       "  \"attn_pdrop\": 0.1,\n",
       "  \"bos_token_id\": 0,\n",
       "  \"embd_pdrop\": 0.1,\n",
       "  \"eos_token_id\": 8,\n",
       "  \"gradient_checkpointing\": false,\n",
       "  \"initializer_range\": 0.02,\n",
       "  \"layer_norm_epsilon\": 1e-05,\n",
       "  \"model_type\": \"gpt2\",\n",
       "  \"n_ctx\": 1024,\n",
       "  \"n_embd\": 1920,\n",
       "  \"n_head\": 16,\n",
       "  \"n_inner\": 7680,\n",
       "  \"n_layer\": 24,\n",
       "  \"n_positions\": 1024,\n",
       "  \"pad_token_id\": 8,\n",
       "  \"reorder_and_upcast_attn\": false,\n",
       "  \"resid_pdrop\": 0.1,\n",
       "  \"scale_attn_by_inverse_layer_idx\": false,\n",
       "  \"scale_attn_weights\": true,\n",
       "  \"summary_activation\": null,\n",
       "  \"summary_first_dropout\": 0.1,\n",
       "  \"summary_proj_to_labels\": true,\n",
       "  \"summary_type\": \"cls_index\",\n",
       "  \"summary_use_proj\": true,\n",
       "  \"transformers_version\": \"4.12.3\",\n",
       "  \"use_cache\": true,\n",
       "  \"vocab_size\": 51200\n",
       "}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import GPT2Config\n",
    "\n",
    "# set config and override with custom configuration\n",
    "config = GPT2Config.from_pretrained(CFG.model_dir)\n",
    "config"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```\n",
    "**labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:\n",
    "    Labels for language modeling.\n",
    "    Note that the labels **are shifted** inside the model, i.e. you can set ``lm_labels = input_ids``\n",
    "    Indices are selected in ``[-100, 0, ..., config.vocab_size]``\n",
    "    All labels set to ``-100`` are ignored (masked), the loss is only\n",
    "    computed for labels in ``[0, ..., config.vocab_size]``\n",
    "\n",
    "Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:\n",
    "**loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:\n",
    "    Language modeling loss.\n",
    "**prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)``\n",
    "    Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).\n",
    "**past**:\n",
    "    list of ``torch.FloatTensor`` (one for each layer) of shape ``(2, batch_size, num_heads, sequence_length, embed_size_per_head)``:\n",
    "    that contains pre-computed hidden-states (key and values in the attention blocks).\n",
    "    Can be used (see `past` input) to speed up sequential decoding. The token ids which have their past given to this model\n",
    "    should not be passed as input ids as they have already been computed.\n",
    "**hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)\n",
    "    list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)\n",
    "    of shape ``(batch_size, sequence_length, hidden_size)``:\n",
    "    Hidden-states of the model at the output of each layer plus the initial embedding outputs.\n",
    "**attentions**: (`optional`, returned when ``config.output_attentions=True``)\n",
    "    list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:\n",
    "    Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.\n",
    "\n",
    "Examples::\n",
    "\n",
    "    import torch\n",
    "    from transformers import GPT2Tokenizer, GPT2LMHeadModel\n",
    "\n",
    "    tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n",
    "    model = GPT2LMHeadModel.from_pretrained('gpt2')\n",
    "\n",
    "    input_ids = torch.tensor(tokenizer.encode(\"Hello, my dog is cute\", add_special_tokens=True)).unsqueeze(0)  # Batch size 1\n",
    "    outputs = model(input_ids, labels=input_ids)\n",
    "    loss, logits = outputs[:2]\n",
    "\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import GPT2LMHeadModel\n",
    "\n",
    "# Attach Language model Head to the pretrained GPT model\n",
    "model = GPT2LMHeadModel.from_pretrained(CFG.model_dir) # KoGPT3 shares the same structure as KoGPT2. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "current device that is training on: cuda:0\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GPT2LMHeadModel(\n",
       "  (transformer): GPT2Model(\n",
       "    (wte): Embedding(51200, 1920)\n",
       "    (wpe): Embedding(1024, 1920)\n",
       "    (drop): Dropout(p=0.1, inplace=False)\n",
       "    (h): ModuleList(\n",
       "      (0): GPT2Block(\n",
       "        (ln_1): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): GPT2Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): GPT2MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (1): GPT2Block(\n",
       "        (ln_1): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): GPT2Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): GPT2MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (2): GPT2Block(\n",
       "        (ln_1): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): GPT2Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): GPT2MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (3): GPT2Block(\n",
       "        (ln_1): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): GPT2Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): GPT2MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (4): GPT2Block(\n",
       "        (ln_1): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): GPT2Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): GPT2MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (5): GPT2Block(\n",
       "        (ln_1): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): GPT2Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): GPT2MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (6): GPT2Block(\n",
       "        (ln_1): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): GPT2Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): GPT2MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (7): GPT2Block(\n",
       "        (ln_1): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): GPT2Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): GPT2MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (8): GPT2Block(\n",
       "        (ln_1): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): GPT2Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): GPT2MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (9): GPT2Block(\n",
       "        (ln_1): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): GPT2Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): GPT2MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (10): GPT2Block(\n",
       "        (ln_1): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): GPT2Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): GPT2MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (11): GPT2Block(\n",
       "        (ln_1): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): GPT2Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): GPT2MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (12): GPT2Block(\n",
       "        (ln_1): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): GPT2Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): GPT2MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (13): GPT2Block(\n",
       "        (ln_1): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): GPT2Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): GPT2MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (14): GPT2Block(\n",
       "        (ln_1): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): GPT2Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): GPT2MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (15): GPT2Block(\n",
       "        (ln_1): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): GPT2Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): GPT2MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (16): GPT2Block(\n",
       "        (ln_1): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): GPT2Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): GPT2MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (17): GPT2Block(\n",
       "        (ln_1): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): GPT2Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): GPT2MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (18): GPT2Block(\n",
       "        (ln_1): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): GPT2Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): GPT2MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (19): GPT2Block(\n",
       "        (ln_1): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): GPT2Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): GPT2MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (20): GPT2Block(\n",
       "        (ln_1): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): GPT2Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): GPT2MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (21): GPT2Block(\n",
       "        (ln_1): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): GPT2Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): GPT2MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (22): GPT2Block(\n",
       "        (ln_1): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): GPT2Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): GPT2MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (23): GPT2Block(\n",
       "        (ln_1): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): GPT2Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): GPT2MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (ln_f): LayerNorm((1920,), eps=1e-05, elementwise_affine=True)\n",
       "  )\n",
       "  (lm_head): Linear(in_features=1920, out_features=51200, bias=False)\n",
       ")"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "# move the model to device\n",
    "if torch.cuda.is_available() and CFG.DEBUG == False:\n",
    "    device = torch.device(\"cuda:0\")\n",
    "elif CFG.DEBUG == True:\n",
    "    device = torch.device(\"cpu\")\n",
    "\n",
    "print(f\"current device that is training on: {device}\")\n",
    "model = model.to(device)\n",
    "model.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mon Nov 15 17:21:50 2021       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 450.80.02    Driver Version: 450.80.02    CUDA Version: 11.0     |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|                               |                      |               MIG M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  Tesla V100-PCIE...  Off  | 00000000:00:05.0 Off |                  Off |\n",
      "| N/A   50C    P0    46W / 250W |   5823MiB / 32510MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                                  |\n",
      "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
      "|        ID   ID                                                   Usage      |\n",
      "|=============================================================================|\n",
      "+-----------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reference: https://github.com/shbictai/narrativeKoGPT2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://huggingface.co/transformers/custom_datasets.html\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "class CustomDataset(Dataset):\n",
    "    \"\"\" CustomDataset class for poetic sentences \"\"\"\n",
    "    def __init__(self, list_dataset, tokenizer):\n",
    "\n",
    "        self.list_dataset = list_dataset\n",
    "        self.tokenizer = tokenizer\n",
    "        self.tokenized_sentences = self.tokenizer(\n",
    "            list_dataset,\n",
    "            return_tensors=\"pt\",\n",
    "            padding=\"max_length\",\n",
    "            truncation=True,\n",
    "            max_length=CFG.max_token_length,\n",
    "            add_special_tokens=True,\n",
    "            return_token_type_ids=False,\n",
    "            )\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        encoded_dict = {key: val[idx] for key, val in self.tokenized_sentences.items()}\n",
    "        encoded_dict[\"labels\"] = encoded_dict[\"input_ids\"].clone() # gpt has same labels as input_ids: https://github.com/huggingface/notebooks/blob/master/examples/language_modeling.ipynb\n",
    "        return encoded_dict\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.list_dataset)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "66872\n"
     ]
    }
   ],
   "source": [
    "kor_poetic_dataset = CustomDataset(list_post_processed, tokenizer)\n",
    "kor_eval_dataset_for_debugging = CustomDataset(list_post_processed[:100], tokenizer) # small dataset for debugging purpose\n",
    "print(len(kor_poetic_dataset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'input_ids': tensor([30203, 31802, 33055, 27028, 32167, 30093, 24672, 22244,     3,     3,\n",
      "            3,     3,     3,     3,     3,     3,     3,     3,     3,     3,\n",
      "            3,     3,     3,     3,     3,     3,     3,     3,     3,     3,\n",
      "            3,     3,     3,     3,     3,     3,     3,     3,     3,     3,\n",
      "            3,     3]), 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), 'labels': tensor([30203, 31802, 33055, 27028, 32167, 30093, 24672, 22244,     3,     3,\n",
      "            3,     3,     3,     3,     3,     3,     3,     3,     3,     3,\n",
      "            3,     3,     3,     3,     3,     3,     3,     3,     3,     3,\n",
      "            3,     3,     3,     3,     3,     3,     3,     3,     3,     3,\n",
      "            3,     3])}\n"
     ]
    }
   ],
   "source": [
    "print(kor_poetic_dataset[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# training 35000 step according to https://jeinalog.tistory.com/entry/AI-x-Bookathon-%EC%9D%B8%EA%B3%B5%EC%A7%80%EB%8A%A5%EC%9D%84-%EC%88%98%ED%95%84-%EC%9E%91%EA%B0%80%EB%A1%9C-%ED%95%99%EC%8A%B5%EC%8B%9C%EC%BC%9C%EB%B3%B4%EC%9E%90\n",
    "CFG.num_epochs = 17 # One epoch is 2090(= round(dataset_size: 64/ batch_size: 32)) steps. 5 epoch is 10450 step, 17 epoch is 35000 step.\n",
    "CFG.evaluation_steps = 1400 # evaluate every 1400 steps -> 35000/1400 = 25.\n",
    "CFG.learning_rate = 2e-5 # https://jeinalog.tistory.com/entry/AI-x-Bookathon-%EC%9D%B8%EA%B3%B5%EC%A7%80%EB%8A%A5%EC%9D%84-%EC%88%98%ED%95%84-%EC%9E%91%EA%B0%80%EB%A1%9C-%ED%95%99%EC%8A%B5%EC%8B%9C%EC%BC%9C%EB%B3%B4%EC%9E%90\n",
    "CFG.train_batch_size = 32 # batch size of 32, max_length = 42 takes up 30.5GB of memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://github.com/huggingface/transformers/blob/master/examples/pytorch/language-modeling/run_clm.py\n",
    "from transformers import Trainer, TrainingArguments\n",
    "\n",
    "# https://huggingface.co/transformers/_modules/transformers/training_args.html\n",
    "training_args = TrainingArguments(\n",
    "    # train configs\n",
    "    num_train_epochs=CFG.num_epochs,  # total number of training epochs\n",
    "    per_device_train_batch_size=CFG.train_batch_size,  # batch size per device during training \n",
    "    per_device_eval_batch_size=CFG.train_batch_size,\n",
    "    learning_rate=CFG.learning_rate,  # learning rate,\n",
    "    # warmup_steps=CFG.warmup_steps,  # number of warmup steps for learning rate scheduler\n",
    "    weight_decay=CFG.weight_decay,  # strength of weight decay\n",
    "    \n",
    "    # evaluating\n",
    "    evaluation_strategy = \"steps\",\n",
    "    eval_steps = CFG.evaluation_steps,\n",
    "    load_best_model_at_end = True,\n",
    "    metric_for_best_model = 'loss',\n",
    "    \n",
    "    # logging\n",
    "    logging_steps=CFG.evaluation_steps,              # log saving step.\n",
    "    logging_dir=\"./logs\",  # directory for storing logs, \n",
    "    report_to=\"wandb\",  # report to wandb\n",
    "    run_name = 'kogpt-trinity-poet',\n",
    "\n",
    "    # save configs\n",
    "    save_steps=CFG.evaluation_steps,\n",
    "    output_dir=\"./results\",  # output directory\n",
    "    save_total_limit= 4,              # number of total save model.\n",
    ")\n",
    "\n",
    "# https://github.com/huggingface/transformers/blob/master/src/transformers/trainer.py\n",
    "trainer = Trainer(\n",
    "    model=model,  # the instantiated 🤗 Transformers model to be trained\n",
    "    args=training_args,  # training arguments, defined above\n",
    "    train_dataset=kor_poetic_dataset,  # training dataset, no evaluation dataset\n",
    "    eval_dataset=kor_poetic_dataset,  # evaluation dataset is same as training dataset\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running training *****\n",
      "  Num examples = 66872\n",
      "  Num Epochs = 17\n",
      "  Instantaneous batch size per device = 32\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 32\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 35530\n",
      "Automatic Weights & Biases logging enabled, to disable set os.environ[\"WANDB_DISABLED\"] = \"true\"\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find kor-3-line-poetry\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33msnoop2head\u001b[0m (use `wandb login --relogin` to force relogin)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "                    Syncing run <strong><a href=\"https://wandb.ai/snoop2head/huggingface/runs/303yd9c0\" target=\"_blank\">kogpt-trinity-poet</a></strong> to <a href=\"https://wandb.ai/snoop2head/huggingface\" target=\"_blank\">Weights & Biases</a> (<a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">docs</a>).<br/>\n",
       "\n",
       "                "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='35530' max='35530' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [35530/35530 17:48:06, Epoch 17/17]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1400</td>\n",
       "      <td>0.949900</td>\n",
       "      <td>0.744963</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2800</td>\n",
       "      <td>0.778600</td>\n",
       "      <td>0.544095</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4200</td>\n",
       "      <td>0.666800</td>\n",
       "      <td>0.375864</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5600</td>\n",
       "      <td>0.442500</td>\n",
       "      <td>0.287678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7000</td>\n",
       "      <td>0.375100</td>\n",
       "      <td>0.221534</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8400</td>\n",
       "      <td>0.313200</td>\n",
       "      <td>0.174947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9800</td>\n",
       "      <td>0.231000</td>\n",
       "      <td>0.168612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11200</td>\n",
       "      <td>0.217600</td>\n",
       "      <td>0.161072</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12600</td>\n",
       "      <td>0.207400</td>\n",
       "      <td>0.149079</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14000</td>\n",
       "      <td>0.181900</td>\n",
       "      <td>0.146354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15400</td>\n",
       "      <td>0.176700</td>\n",
       "      <td>0.142527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16800</td>\n",
       "      <td>0.175300</td>\n",
       "      <td>0.134185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18200</td>\n",
       "      <td>0.158600</td>\n",
       "      <td>0.132836</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19600</td>\n",
       "      <td>0.155700</td>\n",
       "      <td>0.129664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>21000</td>\n",
       "      <td>0.155400</td>\n",
       "      <td>0.123533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22400</td>\n",
       "      <td>0.143800</td>\n",
       "      <td>0.122523</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23800</td>\n",
       "      <td>0.141300</td>\n",
       "      <td>0.120619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25200</td>\n",
       "      <td>0.142100</td>\n",
       "      <td>0.115307</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>26600</td>\n",
       "      <td>0.132800</td>\n",
       "      <td>0.115008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>28000</td>\n",
       "      <td>0.129900</td>\n",
       "      <td>0.113422</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29400</td>\n",
       "      <td>0.130700</td>\n",
       "      <td>0.109155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30800</td>\n",
       "      <td>0.122800</td>\n",
       "      <td>0.108694</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>32200</td>\n",
       "      <td>0.120400</td>\n",
       "      <td>0.107193</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>33600</td>\n",
       "      <td>0.119600</td>\n",
       "      <td>0.105050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>35000</td>\n",
       "      <td>0.113200</td>\n",
       "      <td>0.104590</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running Evaluation *****\n",
      "  Num examples = 66872\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./results/checkpoint-1400\n",
      "Configuration saved in ./results/checkpoint-1400/config.json\n",
      "Model weights saved in ./results/checkpoint-1400/pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 66872\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./results/checkpoint-2800\n",
      "Configuration saved in ./results/checkpoint-2800/config.json\n",
      "Model weights saved in ./results/checkpoint-2800/pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 66872\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./results/checkpoint-4200\n",
      "Configuration saved in ./results/checkpoint-4200/config.json\n",
      "Model weights saved in ./results/checkpoint-4200/pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 66872\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./results/checkpoint-5600\n",
      "Configuration saved in ./results/checkpoint-5600/config.json\n",
      "Model weights saved in ./results/checkpoint-5600/pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 66872\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./results/checkpoint-7000\n",
      "Configuration saved in ./results/checkpoint-7000/config.json\n",
      "Model weights saved in ./results/checkpoint-7000/pytorch_model.bin\n",
      "Deleting older checkpoint [results/checkpoint-1400] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 66872\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./results/checkpoint-8400\n",
      "Configuration saved in ./results/checkpoint-8400/config.json\n",
      "Model weights saved in ./results/checkpoint-8400/pytorch_model.bin\n",
      "Deleting older checkpoint [results/checkpoint-2800] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 66872\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./results/checkpoint-9800\n",
      "Configuration saved in ./results/checkpoint-9800/config.json\n",
      "Model weights saved in ./results/checkpoint-9800/pytorch_model.bin\n",
      "Deleting older checkpoint [results/checkpoint-4200] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 66872\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./results/checkpoint-11200\n",
      "Configuration saved in ./results/checkpoint-11200/config.json\n",
      "Model weights saved in ./results/checkpoint-11200/pytorch_model.bin\n",
      "Deleting older checkpoint [results/checkpoint-5600] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 66872\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./results/checkpoint-12600\n",
      "Configuration saved in ./results/checkpoint-12600/config.json\n",
      "Model weights saved in ./results/checkpoint-12600/pytorch_model.bin\n",
      "Deleting older checkpoint [results/checkpoint-7000] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 66872\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./results/checkpoint-14000\n",
      "Configuration saved in ./results/checkpoint-14000/config.json\n",
      "Model weights saved in ./results/checkpoint-14000/pytorch_model.bin\n",
      "Deleting older checkpoint [results/checkpoint-8400] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 66872\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./results/checkpoint-15400\n",
      "Configuration saved in ./results/checkpoint-15400/config.json\n",
      "Model weights saved in ./results/checkpoint-15400/pytorch_model.bin\n",
      "Deleting older checkpoint [results/checkpoint-9800] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 66872\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./results/checkpoint-16800\n",
      "Configuration saved in ./results/checkpoint-16800/config.json\n",
      "Model weights saved in ./results/checkpoint-16800/pytorch_model.bin\n",
      "Deleting older checkpoint [results/checkpoint-11200] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 66872\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./results/checkpoint-18200\n",
      "Configuration saved in ./results/checkpoint-18200/config.json\n",
      "Model weights saved in ./results/checkpoint-18200/pytorch_model.bin\n",
      "Deleting older checkpoint [results/checkpoint-12600] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 66872\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./results/checkpoint-19600\n",
      "Configuration saved in ./results/checkpoint-19600/config.json\n",
      "Model weights saved in ./results/checkpoint-19600/pytorch_model.bin\n",
      "Deleting older checkpoint [results/checkpoint-14000] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 66872\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./results/checkpoint-21000\n",
      "Configuration saved in ./results/checkpoint-21000/config.json\n",
      "Model weights saved in ./results/checkpoint-21000/pytorch_model.bin\n",
      "Deleting older checkpoint [results/checkpoint-15400] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 66872\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./results/checkpoint-22400\n",
      "Configuration saved in ./results/checkpoint-22400/config.json\n",
      "Model weights saved in ./results/checkpoint-22400/pytorch_model.bin\n",
      "Deleting older checkpoint [results/checkpoint-16800] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 66872\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./results/checkpoint-23800\n",
      "Configuration saved in ./results/checkpoint-23800/config.json\n",
      "Model weights saved in ./results/checkpoint-23800/pytorch_model.bin\n",
      "Deleting older checkpoint [results/checkpoint-18200] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 66872\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./results/checkpoint-25200\n",
      "Configuration saved in ./results/checkpoint-25200/config.json\n",
      "Model weights saved in ./results/checkpoint-25200/pytorch_model.bin\n",
      "Deleting older checkpoint [results/checkpoint-19600] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 66872\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./results/checkpoint-26600\n",
      "Configuration saved in ./results/checkpoint-26600/config.json\n",
      "Model weights saved in ./results/checkpoint-26600/pytorch_model.bin\n",
      "Deleting older checkpoint [results/checkpoint-21000] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 66872\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./results/checkpoint-28000\n",
      "Configuration saved in ./results/checkpoint-28000/config.json\n",
      "Model weights saved in ./results/checkpoint-28000/pytorch_model.bin\n",
      "Deleting older checkpoint [results/checkpoint-22400] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 66872\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./results/checkpoint-29400\n",
      "Configuration saved in ./results/checkpoint-29400/config.json\n",
      "Model weights saved in ./results/checkpoint-29400/pytorch_model.bin\n",
      "Deleting older checkpoint [results/checkpoint-23800] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 66872\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./results/checkpoint-30800\n",
      "Configuration saved in ./results/checkpoint-30800/config.json\n",
      "Model weights saved in ./results/checkpoint-30800/pytorch_model.bin\n",
      "Deleting older checkpoint [results/checkpoint-25200] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 66872\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./results/checkpoint-32200\n",
      "Configuration saved in ./results/checkpoint-32200/config.json\n",
      "Model weights saved in ./results/checkpoint-32200/pytorch_model.bin\n",
      "Deleting older checkpoint [results/checkpoint-26600] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 66872\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./results/checkpoint-33600\n",
      "Configuration saved in ./results/checkpoint-33600/config.json\n",
      "Model weights saved in ./results/checkpoint-33600/pytorch_model.bin\n",
      "Deleting older checkpoint [results/checkpoint-28000] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 66872\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./results/checkpoint-35000\n",
      "Configuration saved in ./results/checkpoint-35000/config.json\n",
      "Model weights saved in ./results/checkpoint-35000/pytorch_model.bin\n",
      "Deleting older checkpoint [results/checkpoint-29400] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./results/checkpoint-35000 (score: 0.10458999872207642).\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=35530, training_loss=0.2571327281407561, metrics={'train_runtime': 64094.8963, 'train_samples_per_second': 17.737, 'train_steps_per_second': 0.554, 'total_flos': 3.0432334227369984e+17, 'train_loss': 0.2571327281407561, 'epoch': 17.0})"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://wandb.ai/snoop2head/huggingface/runs/303yd9c0?workspace=user-snoop2head\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tue Nov 16 11:10:12 2021       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 450.80.02    Driver Version: 450.80.02    CUDA Version: 11.0     |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|                               |                      |               MIG M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  Tesla V100-PCIE...  Off  | 00000000:00:05.0 Off |                  Off |\n",
      "| N/A   59C    P0    43W / 250W |  30513MiB / 32510MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                                  |\n",
      "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
      "|        ID   ID                                                   Usage      |\n",
      "|=============================================================================|\n",
      "+-----------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# first, download and use git-lfs: https://askubuntu.com/questions/799341/how-to-install-git-lfs-on-ubuntu-16-04\n",
    "model.push_to_hub(\"KoGPT-Joong-2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "f9100c3c266b1cd71851c62951119892bad84809058d01f3493295f06be191a8"
  },
  "kernelspec": {
   "display_name": "Python 3.8.3 64-bit ('poet': pyenv)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}