notebooks/MLDoc-JA-multifit_fp16.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: CUDA_VISIBLE_DEVICES=1\n"
     ]
    }
   ],
   "source": [
    "%env CUDA_VISIBLE_DEVICES=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/pczapla/workspace/ulmfit-multilingual\n"
     ]
    }
   ],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "%cd ..\n",
    "from fastai.text import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from multifit.datasets import ULMFiTDataset, Dataset\n",
    "from multifit import ULMFiT, multifit1552_fp16, multifit1552_fp32"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Wikipedia Pretraining"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "def multifit1552_fp16():\n",
      "    return multifit1552_fp32(bs=128).replace_(fp16=True, name=multifit1552_fp16.__name__)\n",
      "\n",
      "def multifit1552_fp32(bs=64):\n",
      "    self = ULMFiT()\n",
      "    self.replace_(\n",
      "        label_smoothing_eps=0.0,\n",
      "        true_wd=True,\n",
      "        wd=0.1,\n",
      "        seed=0,\n",
      "        fp16=False,\n",
      "        bs=bs,\n",
      "        use_adam_08=False,\n",
      "        early_stopping=None,\n",
      "        name=multifit1552_fp32.__name__\n",
      "    )\n",
      "    self.arch.replace_(\n",
      "        tokenizer='fsp',\n",
      "        max_vocab=15000,\n",
      "        qrnn=True,\n",
      "        n_layers=4,\n",
      "        n_hid=1552\n",
      "    )\n",
      "    self.pretrain_lm.replace_(num_epochs=10, drop_mult=0.5, lr=(1e-2 * bs / 48))\n",
      "    self.finetuine_lm.replace_(num_epochs=10, drop_mult=1.0, lr=(1e-3 * bs / 48))\n",
      "    self.classifier.replace_(num_epochs=8, drop_mult=0.5, bs=20, label_smoothing_eps=0.1)\n",
      "    return self\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import inspect\n",
    "print(inspect.getsource(multifit1552_fp16))\n",
    "print(inspect.getsource(multifit1552_fp32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp = multifit1552_fp16()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "wiki_dataset = exp.arch.dataset(Path(f'data/wiki/ja-100'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data lm-notst, trn: 120037, val: 63\n",
      "Size of vocabulary: 15000\n",
      "First 20 words in vocab: ['▁xxunk', '▁xxpad', '▁xxbos', '▁xxeos', '▁xxfld', '▁xxmaj', '▁xxup', '▁xxrep', '▁xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', '▁は', '▁・', '▁(']\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>idx</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>▁月 ▁9 ▁日 ▁) ▁は ▁、 ▁日 ▁本 ▁の ▁ 漫 ▁画 ▁家 ▁、 ▁ アニメーター ▁、 ▁ アニメーション ▁監 ▁ 督 ▁。 ▁大 ▁ 阪 ▁ 帝 ▁国 ▁大 ▁学 ▁ 附 ▁ 属 ▁ 医 ▁学 ▁ 専 ▁ 門 ▁部 ▁を ▁ 卒 ▁業 ▁、 ▁ 医 ▁ 師 ▁ 免 ▁ 許 ▁取 ▁ 得 ▁、 ▁ のち ▁ 医 ▁学 ▁ 博 ▁ 士 ▁(</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>▁では ▁小 ▁学 ▁校 ▁の ▁教 ▁ 室 ▁が ▁ 不 ▁ 足 ▁するなど ▁、 ▁人 ▁ 口 ▁が ▁ 急 ▁ 増 ▁している ▁。 ▁これらの ことから ▁、 ▁ 老 ▁年 ▁人 ▁ 口 ▁の ▁ 比 ▁ 率 ▁が ▁大 ▁ 阪 ▁市 ▁24 ▁区 ▁の ▁中 ▁で ▁最 ▁も ▁ 低 ▁くなっている ▁。 ▁地 ▁理 ▁. ▁区 ▁ 域 ▁は ▁東 ▁を ▁西 ▁ 横 ▁ 堀 ▁川</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>トラック などに ▁手 ▁ 荷 ▁物 ▁が ▁ 残 ▁ っ ていても それが ▁ 忘 ▁れ ▁物 ▁である とは ▁ 認 ▁ 識 ▁しない ▁ 可 ▁ 能 ▁性 ▁が ▁高 ▁い ▁。 ▁また ▁ 忘 ▁れ ▁物 ▁として ▁手 ▁ 荷 ▁物 ▁を ▁発 ▁見 ▁し ていても ▁、 ▁それが ▁ 爆 ▁発 ▁物 ▁である ▁事 ▁には ▁ 気 ▁ 付 ▁かない ▁ 可 ▁ 能 ▁性 ▁もあるが ▁、 ▁</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>郎 ▁は ▁ 淀 ▁野 ▁と ▁ 近 ▁所 ▁を ▁ 散 ▁ 歩 ▁中 ▁、 ▁「 ▁東 ▁京 ▁の ▁ 横 ▁ 光 ▁は どう や ▁? ▁」 ▁と ▁ 質 ▁ 問 ▁し ▁、 ▁ 勢 ▁いの あった ▁ 横 ▁ 光 ▁ 利 ▁一 ▁を ライバル ▁ 視 ▁していた ▁。 ▁12 ▁月 ▁下 ▁ 旬 ▁、 ▁ 母 ▁が ▁ 阿 ▁ 倍 ▁野 ▁の ▁小 ▁間</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>▁ 速 ▁に アラブ ▁化 ▁・ ▁ イスラム ▁化 ▁した ▁。 ▁8 ▁世 ▁ 紀 ▁には アッ バース ▁ 朝 ▁の カリフ が バグ ダー ド に ▁ 都 ▁を ▁ 造 ▁ 営 ▁し ▁、 ▁ アッ バース ▁ 朝 ▁が ▁ 滅 ▁びる まで イスラム ▁世 ▁ 界 ▁の ▁ 精 ▁神 ▁的 ▁中 ▁ 心 ▁として ▁ 栄 ▁えた ▁。 ▁10 ▁世 ▁ 紀 ▁ 末 ▁に</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "wiki_dataset.load_lm_databunch(bs=128,bptt=70).show_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Setting LM weights seed seed to 0\n",
      "Training args:  {'drop_mult': 0.5, 'true_wd': True, 'wd': 0.1, 'pretrained': False} config:  {'emb_sz': 400, 'n_hid': 1552, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.1, 'hidden_p': 0.15, 'input_p': 0.25, 'embed_p': 0.02, 'weight_p': 0.2, 'tie_weights': True, 'out_bias': True}\n",
      "Data lm-notst, trn: 120037, val: 63\n",
      "Size of vocabulary: 15000\n",
      "First 20 words in vocab: ['▁xxunk', '▁xxpad', '▁xxbos', '▁xxeos', '▁xxfld', '▁xxmaj', '▁xxup', '▁xxrep', '▁xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', '▁は', '▁・', '▁(']\n",
      "Setting LM training seed seed to 0\n",
      "Experiment data/wiki/ja-100/models/fsp15k/multifit1552_fp16\n",
      "Training lm from random weights\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Total time: 9:30:58 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>3.107674</td>\n",
       "      <td>3.097375</td>\n",
       "      <td>0.411802</td>\n",
       "      <td>57:20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>3.331074</td>\n",
       "      <td>3.364228</td>\n",
       "      <td>0.382917</td>\n",
       "      <td>57:24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>4.564233</td>\n",
       "      <td>4.470573</td>\n",
       "      <td>0.283381</td>\n",
       "      <td>57:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3.628411</td>\n",
       "      <td>3.552323</td>\n",
       "      <td>0.364708</td>\n",
       "      <td>56:55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>3.489929</td>\n",
       "      <td>3.459749</td>\n",
       "      <td>0.370721</td>\n",
       "      <td>56:50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>3.310685</td>\n",
       "      <td>3.333506</td>\n",
       "      <td>0.383702</td>\n",
       "      <td>56:54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>3.154821</td>\n",
       "      <td>3.179965</td>\n",
       "      <td>0.401726</td>\n",
       "      <td>57:02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>2.972626</td>\n",
       "      <td>2.931468</td>\n",
       "      <td>0.432523</td>\n",
       "      <td>57:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>2.831486</td>\n",
       "      <td>2.714095</td>\n",
       "      <td>0.461959</td>\n",
       "      <td>57:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>2.760642</td>\n",
       "      <td>2.598436</td>\n",
       "      <td>0.481062</td>\n",
       "      <td>57:07</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "this Learner object self-destroyed - it still exists, but no longer usable\n",
      "Language model saved to data/wiki/ja-100/models/fsp15k/multifit1552_fp16\n",
      "Saving dump to data/wiki/ja-100/models/fsp15k/multifit1552_fp16/pretraining.json\n"
     ]
    }
   ],
   "source": [
    "exp.pretrain_lm.train_(wiki_dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MLDoc Trainig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "mldoc_dataset = exp.arch.dataset(Path('data/mldoc/ja-1'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data lm-notst, trn: 9900, val: 1100\n",
      "Size of vocabulary: 15000\n",
      "First 20 words in vocab: ['▁xxunk', '▁xxpad', '▁xxbos', '▁xxeos', '▁xxfld', '▁xxmaj', '▁xxup', '▁xxrep', '▁xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', '▁は', '▁・', '▁(']\n",
      "Data cls, trn: 1000, val: 1000\n",
      "Data tst, trn: 1000, val: 4000\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>▁xxbos ▁xxfld ▁1 ▁ [ ▁東 ▁京 ▁25 ▁日 ▁ ロイ ター ▁ ] ▁野 ▁ 村 ▁ 総 ▁ 研 ▁が ▁、 ▁9 ▁月 ▁24 ▁日 ▁ 付 ▁で まとめた ▁ 香 ▁ 港 ▁・ ▁ シン ガ ポー ▁ ル ▁・ ▁ マレーシア ▁市 ▁場 ▁の ▁主 ▁要 ▁株 ▁式 ▁の ▁ 投 ▁ 資 ▁ 判 ▁ 断 ▁は ▁、 ▁以 ▁下 ▁の ▁通 ▁り ▁。 ▁(</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>▁xxbos ▁xxfld ▁1 ▁ [ ▁東 ▁京 ▁25 ▁日 ▁ ロイ ター ▁ ] ▁野 ▁ 村 ▁ 総 ▁ 研 ▁が ▁、 ▁11 ▁月 ▁25 ▁日 ▁ 付 ▁で まとめた ▁ 香 ▁ 港 ▁・ ▁ シン ガ ポ ▁ ール ▁・ ▁ マレーシア ▁市 ▁場 ▁の ▁主 ▁要 ▁株 ▁式 ▁の ▁ 投 ▁ 資 ▁ 判 ▁ 断 ▁は ▁、 ▁以 ▁下 ▁の ▁通 ▁り ▁。 ▁(</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>▁xxbos ▁xxfld ▁1 ▁ 直 ▁ 近 ▁上 ▁ 値 ▁ 抵 ▁ 抗 ▁ 短 ▁期 ▁上 ▁ 値 ▁ 抵 ▁ 抗 ▁ 直 ▁ 近 ▁下 ▁ 値 ▁ 支 ▁ 持 ▁ 短 ▁期 ▁下 ▁ 値 ▁ 支 ▁ 持 ▁ ドル ▁ / ▁ 円 ▁ 125 ▁. ▁ 60 ▁12 6 ▁. ▁00 ▁ 124 ▁. ▁ 60 ▁ 124 ▁. ▁ 40</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>▁xxbos ▁xxfld ▁1 ▁ [ ▁東 ▁京 ▁26 ▁日 ▁ ロイ ター ▁ ] ▁ 今 ▁ 週 ▁は ▁、 ▁28 ▁日 ▁発 ▁表 ▁の 8 ▁月 ▁ 調 ▁ 査 ▁日 ▁ 銀 ▁ 短 ▁ 観 ▁( ▁ 企 ▁業 ▁ 短 ▁期 ▁経 ▁ 済 ▁ 観 ▁ 測 ▁ 調 ▁ 査 ▁) ▁、 ▁30 ▁日 ▁に ▁発 ▁表 ▁される 7 ▁月 ▁ 鉱 ▁ 工</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>▁xxbos ▁xxfld ▁1 ▁ レート は ▁ 終 ▁ 値 ▁( ▁前 ▁日 ▁ 比 ▁または ▁前 ▁ 週 ▁ 末 ▁ 比 ▁) ▁、 ▁ 安 ▁ 値 ▁ 〜 ▁高 ▁ 値 ▁ ━ ▁ ━ ▁ ━ ▁ ━ ▁ ━ ▁ ━ ▁ ━ ▁ ━ ▁ ━ ▁ ━ ▁ ━ ▁ ━ ▁ ━ ▁ ━ ▁ ━ ▁ ━ ▁ ━ ▁ ━</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "mldoc_dataset.load_clas_databunch(bs=128)[0].show_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Setting LM weights seed seed to 0\n",
      "Training args:  {'drop_mult': 1.0, 'true_wd': True, 'wd': 0.1, 'pretrained': True, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/fsp15k/multifit1552_fp16/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/fsp15k/itos')]} config:  {'emb_sz': 400, 'n_hid': 1552, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.1, 'hidden_p': 0.15, 'input_p': 0.25, 'embed_p': 0.02, 'weight_p': 0.2, 'tie_weights': True, 'out_bias': True}\n",
      "Data lm-notst, trn: 9900, val: 1100\n",
      "Size of vocabulary: 15000\n",
      "First 20 words in vocab: ['▁xxunk', '▁xxpad', '▁xxbos', '▁xxeos', '▁xxfld', '▁xxmaj', '▁xxup', '▁xxrep', '▁xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', '▁は', '▁・', '▁(']\n",
      "Setting LM training seed seed to 0\n",
      "Experiment data/mldoc/ja-1/models/fsp15k/multifit1552_fp16\n",
      "Fitting using 2 cycle fit schedule\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Total time: 01:26 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2.219860</td>\n",
       "      <td>1.833544</td>\n",
       "      <td>0.606933</td>\n",
       "      <td>01:26</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Total time: 18:17 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.944596</td>\n",
       "      <td>1.670773</td>\n",
       "      <td>0.637072</td>\n",
       "      <td>01:49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1.806380</td>\n",
       "      <td>1.528684</td>\n",
       "      <td>0.662753</td>\n",
       "      <td>01:49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1.720533</td>\n",
       "      <td>1.444982</td>\n",
       "      <td>0.677305</td>\n",
       "      <td>01:49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1.663399</td>\n",
       "      <td>1.379856</td>\n",
       "      <td>0.688376</td>\n",
       "      <td>01:49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.618090</td>\n",
       "      <td>1.334722</td>\n",
       "      <td>0.696019</td>\n",
       "      <td>01:49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>1.572266</td>\n",
       "      <td>1.292020</td>\n",
       "      <td>0.704291</td>\n",
       "      <td>01:49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>1.550954</td>\n",
       "      <td>1.254581</td>\n",
       "      <td>0.711302</td>\n",
       "      <td>01:49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>1.474405</td>\n",
       "      <td>1.226887</td>\n",
       "      <td>0.716287</td>\n",
       "      <td>01:49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>1.446200</td>\n",
       "      <td>1.212317</td>\n",
       "      <td>0.718968</td>\n",
       "      <td>01:49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>1.445466</td>\n",
       "      <td>1.209516</td>\n",
       "      <td>0.719546</td>\n",
       "      <td>01:50</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "this Learner object self-destroyed - it still exists, but no longer usable\n",
      "Language model saved to data/mldoc/ja-1/models/fsp15k/multifit1552_fp16\n",
      "Saving dump to data/mldoc/ja-1/models/fsp15k/multifit1552_fp16/finetuining.json\n"
     ]
    }
   ],
   "source": [
    "exp.finetune_lm.train_(mldoc_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading data/mldoc/ja-1/models/fsp15k/multifit1552_fp16/finetuining.json\n",
      "Loading data/wiki/ja-100/models/fsp15k/multifit1552_fp16/pretraining.json\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "ULMFiTFinetuining(seed=0, name='multifit1552_fp16', experiment_path=PosixPath('data/mldoc/ja-1/models/fsp15k/multifit1552_fp16'), dataset_path=PosixPath('data/mldoc/ja-1'), num_epochs=10, bs=128, bptt=70, drop_mult=1.0, label_smoothing_eps=0.0, use_adam_08=False, true_wd=True, wd=0.1, fp16=True, lr=0.0026666666666666666, pretrained=True)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exp.load_(Path('data/mldoc/ja-1/models/fsp15k/multifit1552_fp16')).finetune_lm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Setting Classifier weights seed seed to 0\n",
      "Data lm-notst, trn: 9900, val: 1100\n",
      "Size of vocabulary: 15000\n",
      "First 20 words in vocab: ['▁xxunk', '▁xxpad', '▁xxbos', '▁xxeos', '▁xxfld', '▁xxmaj', '▁xxup', '▁xxrep', '▁xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', '▁は', '▁・', '▁(']\n",
      "Data cls, trn: 1000, val: 1000\n",
      "Data tst, trn: 1000, val: 4000\n",
      "Training args:  {'drop_mult': 0.5, 'wd': 0.1, 'pretrained': False, 'bptt': 70, 'loss_func': FlattenedLoss of LabelSmoothingCrossEntropy()} config:  {'emb_sz': 400, 'n_hid': 1552, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.4, 'hidden_p': 0.3, 'input_p': 0.4, 'embed_p': 0.05, 'weight_p': 0.5}\n",
      "Loading pretrained model /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/fsp15k/multifit1552_fp16/enc_best\n",
      "Setting Classifier training seed seed to 0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Total time: 01:52 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.911967</td>\n",
       "      <td>0.639816</td>\n",
       "      <td>0.850000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.781325</td>\n",
       "      <td>0.646144</td>\n",
       "      <td>0.874000</td>\n",
       "      <td>00:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.703413</td>\n",
       "      <td>0.722381</td>\n",
       "      <td>0.849000</td>\n",
       "      <td>00:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.622030</td>\n",
       "      <td>0.678657</td>\n",
       "      <td>0.875000</td>\n",
       "      <td>00:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.545690</td>\n",
       "      <td>0.623689</td>\n",
       "      <td>0.890000</td>\n",
       "      <td>00:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.491453</td>\n",
       "      <td>0.628224</td>\n",
       "      <td>0.883000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.451803</td>\n",
       "      <td>0.628584</td>\n",
       "      <td>0.886000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.429028</td>\n",
       "      <td>0.627261</td>\n",
       "      <td>0.888000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classifier model saved to data/mldoc/ja-1/models/fsp15k/multifit1552_fp16\n",
      "Saving dump to data/mldoc/ja-1/models/fsp15k/multifit1552_fp16/classifier.json\n",
      "this Learner object self-destroyed - it still exists, but no longer usable\n"
     ]
    }
   ],
   "source": [
    "exp.classifier.train_(seed=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Setting Classifier weights seed seed to 1\n",
      "Data lm-notst, trn: 9900, val: 1100\n",
      "Size of vocabulary: 15000\n",
      "First 20 words in vocab: ['▁xxunk', '▁xxpad', '▁xxbos', '▁xxeos', '▁xxfld', '▁xxmaj', '▁xxup', '▁xxrep', '▁xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', '▁は', '▁・', '▁(']\n",
      "Data cls, trn: 1000, val: 1000\n",
      "Data tst, trn: 1000, val: 4000\n",
      "Training args:  {'drop_mult': 0.5, 'wd': 0.1, 'pretrained': False, 'bptt': 70, 'loss_func': FlattenedLoss of LabelSmoothingCrossEntropy()} config:  {'emb_sz': 400, 'n_hid': 1552, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.4, 'hidden_p': 0.3, 'input_p': 0.4, 'embed_p': 0.05, 'weight_p': 0.5}\n",
      "Loading pretrained model /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/fsp15k/multifit1552_fp16/enc_best\n",
      "Setting Classifier training seed seed to 1\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Total time: 01:54 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.908408</td>\n",
       "      <td>0.654319</td>\n",
       "      <td>0.860000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.790398</td>\n",
       "      <td>0.768438</td>\n",
       "      <td>0.812000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.692833</td>\n",
       "      <td>0.650352</td>\n",
       "      <td>0.883000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.607950</td>\n",
       "      <td>0.659773</td>\n",
       "      <td>0.867000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.549019</td>\n",
       "      <td>0.627178</td>\n",
       "      <td>0.886000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.497783</td>\n",
       "      <td>0.701499</td>\n",
       "      <td>0.879000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.455803</td>\n",
       "      <td>0.651905</td>\n",
       "      <td>0.870000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.430048</td>\n",
       "      <td>0.634729</td>\n",
       "      <td>0.879000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classifier model saved to data/mldoc/ja-1/models/fsp15k/multifit1552_fp16seed1\n",
      "Saving dump to data/mldoc/ja-1/models/fsp15k/multifit1552_fp16seed1/classifier.json\n",
      "this Learner object self-destroyed - it still exists, but no longer usable\n"
     ]
    }
   ],
   "source": [
    "exp.classifier.train_(seed=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Setting Classifier weights seed seed to 2\n",
      "Data lm-notst, trn: 9900, val: 1100\n",
      "Size of vocabulary: 15000\n",
      "First 20 words in vocab: ['▁xxunk', '▁xxpad', '▁xxbos', '▁xxeos', '▁xxfld', '▁xxmaj', '▁xxup', '▁xxrep', '▁xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', '▁は', '▁・', '▁(']\n",
      "Data cls, trn: 1000, val: 1000\n",
      "Data tst, trn: 1000, val: 4000\n",
      "Training args:  {'drop_mult': 0.5, 'wd': 0.1, 'pretrained': False, 'bptt': 70, 'loss_func': FlattenedLoss of LabelSmoothingCrossEntropy()} config:  {'emb_sz': 400, 'n_hid': 1552, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.4, 'hidden_p': 0.3, 'input_p': 0.4, 'embed_p': 0.05, 'weight_p': 0.5}\n",
      "Loading pretrained model /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/fsp15k/multifit1552_fp16/enc_best\n",
      "Setting Classifier training seed seed to 2\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Total time: 01:54 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.925113</td>\n",
       "      <td>0.653345</td>\n",
       "      <td>0.849000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.774887</td>\n",
       "      <td>0.632942</td>\n",
       "      <td>0.876000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.686229</td>\n",
       "      <td>0.661485</td>\n",
       "      <td>0.870000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.630340</td>\n",
       "      <td>0.630609</td>\n",
       "      <td>0.877000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.550789</td>\n",
       "      <td>0.618177</td>\n",
       "      <td>0.878000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.494387</td>\n",
       "      <td>0.636892</td>\n",
       "      <td>0.878000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.450500</td>\n",
       "      <td>0.633069</td>\n",
       "      <td>0.885000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.431031</td>\n",
       "      <td>0.637073</td>\n",
       "      <td>0.881000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classifier model saved to data/mldoc/ja-1/models/fsp15k/multifit1552_fp16seed2\n",
      "Saving dump to data/mldoc/ja-1/models/fsp15k/multifit1552_fp16seed2/classifier.json\n",
      "this Learner object self-destroyed - it still exists, but no longer usable\n"
     ]
    }
   ],
   "source": [
    "exp.classifier.train_(seed=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Setting Classifier weights seed seed to 3\n",
      "Data lm-notst, trn: 9900, val: 1100\n",
      "Size of vocabulary: 15000\n",
      "First 20 words in vocab: ['▁xxunk', '▁xxpad', '▁xxbos', '▁xxeos', '▁xxfld', '▁xxmaj', '▁xxup', '▁xxrep', '▁xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', '▁は', '▁・', '▁(']\n",
      "Data cls, trn: 1000, val: 1000\n",
      "Data tst, trn: 1000, val: 4000\n",
      "Training args:  {'drop_mult': 0.5, 'wd': 0.1, 'pretrained': False, 'bptt': 70, 'loss_func': FlattenedLoss of LabelSmoothingCrossEntropy()} config:  {'emb_sz': 400, 'n_hid': 1552, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.4, 'hidden_p': 0.3, 'input_p': 0.4, 'embed_p': 0.05, 'weight_p': 0.5}\n",
      "Loading pretrained model /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/fsp15k/multifit1552_fp16/enc_best\n",
      "Setting Classifier training seed seed to 3\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Total time: 01:54 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.903641</td>\n",
       "      <td>0.616180</td>\n",
       "      <td>0.872000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.798058</td>\n",
       "      <td>0.611768</td>\n",
       "      <td>0.882000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.692193</td>\n",
       "      <td>0.674029</td>\n",
       "      <td>0.842000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.622152</td>\n",
       "      <td>0.605841</td>\n",
       "      <td>0.889000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.545950</td>\n",
       "      <td>0.640584</td>\n",
       "      <td>0.877000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.495717</td>\n",
       "      <td>0.625205</td>\n",
       "      <td>0.885000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.454127</td>\n",
       "      <td>0.629366</td>\n",
       "      <td>0.885000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.430954</td>\n",
       "      <td>0.627262</td>\n",
       "      <td>0.891000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classifier model saved to data/mldoc/ja-1/models/fsp15k/multifit1552_fp16seed3\n",
      "Saving dump to data/mldoc/ja-1/models/fsp15k/multifit1552_fp16seed3/classifier.json\n",
      "this Learner object self-destroyed - it still exists, but no longer usable\n"
     ]
    }
   ],
   "source": [
    "exp.classifier.train_(seed=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Setting Classifier weights seed seed to 4\n",
      "Data lm-notst, trn: 9900, val: 1100\n",
      "Size of vocabulary: 15000\n",
      "First 20 words in vocab: ['▁xxunk', '▁xxpad', '▁xxbos', '▁xxeos', '▁xxfld', '▁xxmaj', '▁xxup', '▁xxrep', '▁xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', '▁は', '▁・', '▁(']\n",
      "Data cls, trn: 1000, val: 1000\n",
      "Data tst, trn: 1000, val: 4000\n",
      "Training args:  {'drop_mult': 0.5, 'wd': 0.1, 'pretrained': False, 'bptt': 70, 'loss_func': FlattenedLoss of LabelSmoothingCrossEntropy()} config:  {'emb_sz': 400, 'n_hid': 1552, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.4, 'hidden_p': 0.3, 'input_p': 0.4, 'embed_p': 0.05, 'weight_p': 0.5}\n",
      "Loading pretrained model /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/fsp15k/multifit1552_fp16/enc_best\n",
      "Setting Classifier training seed seed to 4\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Total time: 01:53 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.885045</td>\n",
       "      <td>0.623022</td>\n",
       "      <td>0.863000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.791488</td>\n",
       "      <td>0.675674</td>\n",
       "      <td>0.850000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.702259</td>\n",
       "      <td>0.796426</td>\n",
       "      <td>0.834000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.626244</td>\n",
       "      <td>0.681571</td>\n",
       "      <td>0.865000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.558977</td>\n",
       "      <td>0.633119</td>\n",
       "      <td>0.890000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.493925</td>\n",
       "      <td>0.646156</td>\n",
       "      <td>0.875000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.450324</td>\n",
       "      <td>0.630983</td>\n",
       "      <td>0.885000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.428313</td>\n",
       "      <td>0.649953</td>\n",
       "      <td>0.879000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classifier model saved to data/mldoc/ja-1/models/fsp15k/multifit1552_fp16seed4\n",
      "Saving dump to data/mldoc/ja-1/models/fsp15k/multifit1552_fp16seed4/classifier.json\n",
      "this Learner object self-destroyed - it still exists, but no longer usable\n"
     ]
    }
   ],
   "source": [
    "exp.classifier.train_(seed=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Setting Classifier weights seed seed to 5\n",
      "Data lm-notst, trn: 9900, val: 1100\n",
      "Size of vocabulary: 15000\n",
      "First 20 words in vocab: ['▁xxunk', '▁xxpad', '▁xxbos', '▁xxeos', '▁xxfld', '▁xxmaj', '▁xxup', '▁xxrep', '▁xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', '▁は', '▁・', '▁(']\n",
      "Data cls, trn: 1000, val: 1000\n",
      "Data tst, trn: 1000, val: 4000\n",
      "Training args:  {'drop_mult': 0.5, 'wd': 0.1, 'pretrained': False, 'bptt': 70, 'loss_func': FlattenedLoss of LabelSmoothingCrossEntropy()} config:  {'emb_sz': 400, 'n_hid': 1552, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.4, 'hidden_p': 0.3, 'input_p': 0.4, 'embed_p': 0.05, 'weight_p': 0.5}\n",
      "Loading pretrained model /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/fsp15k/multifit1552_fp16/enc_best\n",
      "Setting Classifier training seed seed to 5\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Total time: 01:53 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.898696</td>\n",
       "      <td>0.634723</td>\n",
       "      <td>0.865000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.787859</td>\n",
       "      <td>0.628963</td>\n",
       "      <td>0.878000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.700993</td>\n",
       "      <td>0.642409</td>\n",
       "      <td>0.855000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.619737</td>\n",
       "      <td>0.632098</td>\n",
       "      <td>0.880000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.558749</td>\n",
       "      <td>0.729376</td>\n",
       "      <td>0.861000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.489555</td>\n",
       "      <td>0.629867</td>\n",
       "      <td>0.877000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.451724</td>\n",
       "      <td>0.644491</td>\n",
       "      <td>0.876000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.429537</td>\n",
       "      <td>0.651178</td>\n",
       "      <td>0.879000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classifier model saved to data/mldoc/ja-1/models/fsp15k/multifit1552_fp16seed5\n",
      "Saving dump to data/mldoc/ja-1/models/fsp15k/multifit1552_fp16seed5/classifier.json\n",
      "this Learner object self-destroyed - it still exists, but no longer usable\n"
     ]
    }
   ],
   "source": [
    "exp.classifier.train_(seed=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Setting Classifier weights seed seed to 6\n",
      "Data lm-notst, trn: 9900, val: 1100\n",
      "Size of vocabulary: 15000\n",
      "First 20 words in vocab: ['▁xxunk', '▁xxpad', '▁xxbos', '▁xxeos', '▁xxfld', '▁xxmaj', '▁xxup', '▁xxrep', '▁xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', '▁は', '▁・', '▁(']\n",
      "Data cls, trn: 1000, val: 1000\n",
      "Data tst, trn: 1000, val: 4000\n",
      "Training args:  {'drop_mult': 0.5, 'wd': 0.1, 'pretrained': False, 'bptt': 70, 'loss_func': FlattenedLoss of LabelSmoothingCrossEntropy()} config:  {'emb_sz': 400, 'n_hid': 1552, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.4, 'hidden_p': 0.3, 'input_p': 0.4, 'embed_p': 0.05, 'weight_p': 0.5}\n",
      "Loading pretrained model /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/fsp15k/multifit1552_fp16/enc_best\n",
      "Setting Classifier training seed seed to 6\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Total time: 01:53 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.891280</td>\n",
       "      <td>0.656561</td>\n",
       "      <td>0.861000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.771049</td>\n",
       "      <td>0.648831</td>\n",
       "      <td>0.859000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.677718</td>\n",
       "      <td>0.681419</td>\n",
       "      <td>0.859000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.622475</td>\n",
       "      <td>0.635418</td>\n",
       "      <td>0.872000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.555276</td>\n",
       "      <td>0.639490</td>\n",
       "      <td>0.875000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.492825</td>\n",
       "      <td>0.653272</td>\n",
       "      <td>0.873000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.449347</td>\n",
       "      <td>0.649654</td>\n",
       "      <td>0.873000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.425822</td>\n",
       "      <td>0.639283</td>\n",
       "      <td>0.874000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classifier model saved to data/mldoc/ja-1/models/fsp15k/multifit1552_fp16seed6\n",
      "Saving dump to data/mldoc/ja-1/models/fsp15k/multifit1552_fp16seed6/classifier.json\n",
      "this Learner object self-destroyed - it still exists, but no longer usable\n"
     ]
    }
   ],
   "source": [
    "exp.classifier.train_(seed=6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Setting Classifier weights seed seed to 7\n",
      "Data lm-notst, trn: 9900, val: 1100\n",
      "Size of vocabulary: 15000\n",
      "First 20 words in vocab: ['▁xxunk', '▁xxpad', '▁xxbos', '▁xxeos', '▁xxfld', '▁xxmaj', '▁xxup', '▁xxrep', '▁xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', '▁は', '▁・', '▁(']\n",
      "Data cls, trn: 1000, val: 1000\n",
      "Data tst, trn: 1000, val: 4000\n",
      "Training args:  {'drop_mult': 0.5, 'wd': 0.1, 'pretrained': False, 'bptt': 70, 'loss_func': FlattenedLoss of LabelSmoothingCrossEntropy()} config:  {'emb_sz': 400, 'n_hid': 1552, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.4, 'hidden_p': 0.3, 'input_p': 0.4, 'embed_p': 0.05, 'weight_p': 0.5}\n",
      "Loading pretrained model /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/fsp15k/multifit1552_fp16/enc_best\n",
      "Setting Classifier training seed seed to 7\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Total time: 01:53 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.890206</td>\n",
       "      <td>0.618165</td>\n",
       "      <td>0.869000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.787146</td>\n",
       "      <td>0.614832</td>\n",
       "      <td>0.884000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.695766</td>\n",
       "      <td>0.705636</td>\n",
       "      <td>0.843000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.625174</td>\n",
       "      <td>0.611888</td>\n",
       "      <td>0.890000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.561164</td>\n",
       "      <td>0.677992</td>\n",
       "      <td>0.856000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.496124</td>\n",
       "      <td>0.623592</td>\n",
       "      <td>0.881000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.454188</td>\n",
       "      <td>0.647716</td>\n",
       "      <td>0.876000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.431855</td>\n",
       "      <td>0.633412</td>\n",
       "      <td>0.879000</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classifier model saved to data/mldoc/ja-1/models/fsp15k/multifit1552_fp16seed7\n",
      "Saving dump to data/mldoc/ja-1/models/fsp15k/multifit1552_fp16seed7/classifier.json\n",
      "this Learner object self-destroyed - it still exists, but no longer usable\n"
     ]
    }
   ],
   "source": [
    "exp.classifier.train_(seed=7)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'test loss': 0.5893779397010803, 'test FBeta': 0.9026554822921753, 'test Precision': 0.9043030738830566, 'test Recall': 0.902697741985321, 'test accuracy': 0.903249979019165, 'name': 'multifit1552_fp16seed1', 'valid loss': 0.6347294449806213, 'valid FBeta': 0.8822355270385742, 'valid Precision': 0.8312159776687622, 'valid Recall': 0.8805031180381775, 'valid accuracy': 0.9299362897872925, 'train loss': 0.3954845070838928, 'train FBeta': 0.9960629940032959, 'train Precision': 0.9978118538856506, 'train Recall': 1.0, 'train accuracy': 0.9980732202529907}\n",
      "Loading data/mldoc/ja-1/models/fsp15k/multifit1552_fp16/classifier.json\n",
      "Loading data/mldoc/ja-1/models/fsp15k/multifit1552_fp16/finetuining.json\n",
      "Loading data/wiki/ja-100/models/fsp15k/multifit1552_fp16/pretraining.json\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>seed</th>\n",
       "      <th>test accuracy</th>\n",
       "      <th>valid accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>multifit1552_fp16seed4</td>\n",
       "      <td>4</td>\n",
       "      <td>0.89675</td>\n",
       "      <td>0.918580</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>multifit1552_fp16seed6</td>\n",
       "      <td>6</td>\n",
       "      <td>0.89800</td>\n",
       "      <td>0.921776</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>multifit1552_fp16seed5</td>\n",
       "      <td>5</td>\n",
       "      <td>0.89775</td>\n",
       "      <td>0.923404</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>multifit1552_fp16seed7</td>\n",
       "      <td>7</td>\n",
       "      <td>0.90200</td>\n",
       "      <td>0.927660</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>multifit1552_fp16seed2</td>\n",
       "      <td>2</td>\n",
       "      <td>0.90275</td>\n",
       "      <td>0.929336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>multifit1552_fp16seed1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.90325</td>\n",
       "      <td>0.929936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>multifit1552_fp16</td>\n",
       "      <td>0</td>\n",
       "      <td>0.89950</td>\n",
       "      <td>0.936170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>multifit1552_fp16seed3</td>\n",
       "      <td>3</td>\n",
       "      <td>0.90250</td>\n",
       "      <td>0.937634</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     name  seed  test accuracy  valid accuracy\n",
       "5  multifit1552_fp16seed4     4        0.89675        0.918580\n",
       "1  multifit1552_fp16seed6     6        0.89800        0.921776\n",
       "3  multifit1552_fp16seed5     5        0.89775        0.923404\n",
       "2  multifit1552_fp16seed7     7        0.90200        0.927660\n",
       "0  multifit1552_fp16seed2     2        0.90275        0.929336\n",
       "6  multifit1552_fp16seed1     1        0.90325        0.929936\n",
       "7       multifit1552_fp16     0        0.89950        0.936170\n",
       "4  multifit1552_fp16seed3     3        0.90250        0.937634"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_results(exp_path):\n",
    "    exp = ULMFiT().load_(exp_path).classifier\n",
    "    results = exp.validate() \n",
    "    results.update(seed=exp.seed, fp16=exp.fp16)\n",
    "    return results\n",
    "results = [get_results(exp_path) for exp_path in Path('data/mldoc/ja-1/models/fsp15k').glob(\"multifit1552_fp16*\")]\n",
    "results_df = pd.DataFrame.from_records(results)\n",
    "results_df.sort_values([\"valid accuracy\"])[[\"name\", \"seed\", \"test accuracy\", \"valid accuracy\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}