ImageClassification_last.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python377jvsc74a57bd098b0a9b7b4eaaa670588a142fd0a9b87eaafe866f1db4228be72b4211d12040f",
   "display_name": "Python 3.7.7 64-bit ('base': conda)"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "def seed_everything(seed = 0):\n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.backends.cudnn.deterministic = False\n",
    "    torch.backends.cudnn.benchmark = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class param:\n",
    "    num_epoch = 8\n",
    "    batch_size = 64\n",
    "    num_data = 12800\n",
    "    img_size = 224\n",
    "    img_face_crop_mergin = 60\n",
    "    img_center_crop_size = 224\n",
    "    num_workers = 4\n",
    "    lr = 0.0001\n",
    "    k_fold = 5\n",
    "    train_patience = 4\n",
    "\n",
    "class path:\n",
    "    train_csv = '/opt/ml/input/data/train/train.csv'\n",
    "    train_images = '/opt/ml/input/data/train/images'\n",
    "    eval_images = '/opt/ml/input/data/eval/images'\n",
    "    model_save = '/opt/ml/code/results'\n",
    "\n",
    "class label:\n",
    "    def age(x):\n",
    "        x = int(x)\n",
    "        assert(x > 0)\n",
    "        return 0 if x < 30 else 1 if x < 60 else 2\n",
    "\n",
    "    def gender(x):\n",
    "        assert(x == 'male' or x == 'female')\n",
    "        return 0 if x == 'male' else 1\n",
    "\n",
    "    def mask(x):\n",
    "        assert(x.startswith('incorrect') or x.startswith('mask') or x.startswith('normal'))\n",
    "        return 0 if x.startswith('mask') else 1 if x.startswith('incorrect') else 2\n",
    "\n",
    "    def group(x):\n",
    "        x = os.path.basename(x).split('_')\n",
    "        return label.gender(x[1]) * 3 + label.age(x[3])\n",
    "    \n",
    "    def image(x):\n",
    "        x = os.path.split(x)\n",
    "        mask_label = label.mask(x[-1])\n",
    "        x = x[-2].split('_')\n",
    "        age_label = label.age(x[3])\n",
    "        gender_label = label.gender(x[1])\n",
    "        return mask_label * 6 + gender_label * 3 + age_label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "import torch\n",
    "import random\n",
    "import pickle\n",
    "import timm\n",
    "import numpy as np\n",
    "from torch.utils.data import Dataset, DataLoader, ConcatDataset\n",
    "from tqdm.notebook import tqdm\n",
    "from glob import glob\n",
    "from collections import defaultdict\n",
    "from pprint import pprint\n",
    "from PIL import Image\n",
    "from albumentations import Compose, Resize, HorizontalFlip, Rotate\n",
    "from adamp import AdamP\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "seed_everything()\n",
    "\n",
    "class FaceDataset(Dataset):\n",
    "    def __init__(self, imgs, transforms):\n",
    "        self.imgs = imgs\n",
    "        self.transforms = transforms\n",
    "        self.labels = [label.image(img) for img in imgs]\n",
    "        self.bboxes = [pickle.load(open(os.path.splitext(img)[0] + '.txt', 'rb')) for img in self.imgs]\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        img = Image.open(self.imgs[idx])\n",
    "        label = self.labels[idx]\n",
    "        img_size = img.size\n",
    "        box = self.bboxes[idx]\n",
    "        if box is not None:\n",
    "            margin = param.img_face_crop_mergin\n",
    "            box = [\n",
    "                int(max(box[0] - margin / 2, 0)),\n",
    "                int(max(box[1] - margin / 2, 0)),\n",
    "                int(min(box[2] + margin / 2, img_size[0])),\n",
    "                int(min(box[3] + margin / 2, img_size[1]))\n",
    "            ]\n",
    "        else:\n",
    "            margin = param.img_center_crop_size\n",
    "            box = [\n",
    "                int(img_size[0] / 2 - margin / 2),\n",
    "                int(img_size[1] / 2 - margin / 2),\n",
    "                int(img_size[0] / 2 + margin / 2),\n",
    "                int(img_size[1] / 2 + margin / 2)\n",
    "            ]\n",
    "        img = img.crop(box)\n",
    "        if self.transforms is not None:\n",
    "            img = self.transforms(image=np.array(img))['image']\n",
    "        return img, label\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.imgs)\n",
    "\n",
    "def get_dataset(images_path, transforms=None, k_fold = 6, num_data=2700):\n",
    "    #num_data : 3 * k_fold * n\n",
    "    groups = glob(images_path + '/*')\n",
    "\n",
    "    #classifying by group label\n",
    "    groups_classified = defaultdict(list)\n",
    "    for group in groups:\n",
    "        groups_classified[label.group(group)].append(group)\n",
    "    groups_classified = list(groups_classified.values())\n",
    "\n",
    "    #upsampling/downsampling to num_data\n",
    "    num_data //= 3\n",
    "    groups_flattened = []\n",
    "    num_class = len(groups_classified)\n",
    "    num_data_per_class = num_data // k_fold\n",
    "    for group_classified in groups_classified:\n",
    "        extra = random.sample(group_classified, num_data // k_fold % len(group_classified))\n",
    "        group_classified *= num_data // k_fold // len(group_classified)\n",
    "        group_classified += extra\n",
    "        groups_flattened += group_classified\n",
    "    \n",
    "    #shuffle and divide\n",
    "    random.shuffle(groups_flattened)\n",
    "    groups_fold = [groups_flattened[i * num_data_per_class : (i + 1) * num_data_per_class] for i in range(k_fold)]\n",
    "\n",
    "    #convert group path to image path\n",
    "    imgs_fold = [[] for _ in range(k_fold)]\n",
    "    for i, group_fold in enumerate(groups_fold):\n",
    "        for group in group_fold:\n",
    "            ext = [os.path.splitext(i)[1] for i in glob(group + '/*')]\n",
    "            ext = '.jpg' if '.jpg' in ext else '.jpeg' if '.jpeg' in ext else '.png'\n",
    "            imgs = ['incorrect_mask', 'mask' + str(random.randint(1, 5)), 'normal']\n",
    "            imgs = [os.path.join(group, img) + ext for img in imgs]\n",
    "            imgs_fold[i] += imgs\n",
    "\n",
    "    #create dataset\n",
    "    dataset_fold = []\n",
    "    for i in range(k_fold):\n",
    "        dataset_fold.append(FaceDataset(imgs_fold[i], transforms=transforms))\n",
    "\n",
    "    return dataset_fold\n",
    "\n",
    "transforms = Compose([\n",
    "    Resize(param.img_size, param.img_size, p=1.0),\n",
    "    HorizontalFlip(p=.5),\n",
    "    Rotate(10)\n",
    "])\n",
    "    \n",
    "dataset = get_dataset(path.train_images, transforms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(name, model, train_data, valid_data, optimizer, scheduler, criterion):\n",
    "    num_epoch = param.num_epoch\n",
    "    train_log_interval = 5\n",
    "\n",
    "    best_val_acc = 0\n",
    "    best_val_loss = np.inf\n",
    "    renew_count = 0\n",
    "\n",
    "    device = torch.device('cuda')\n",
    "    model.to(device)\n",
    "    for epoch in tqdm(range(num_epoch)):\n",
    "        model.train()\n",
    "        loss_value = 0\n",
    "        matches = 0\n",
    "        for i, data in enumerate(train_data):\n",
    "            inputs, labels = data\n",
    "            inputs = inputs.to(device)\n",
    "            labels = labels.to(device)\n",
    "\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            outputs = model(inputs.permute(0, 3, 1, 2).float())\n",
    "            preds = torch.argmax(outputs, dim=-1)\n",
    "\n",
    "            loss = criterion(outputs, labels)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            loss_value += loss.item()\n",
    "            matches += (preds == labels).sum().item()\n",
    "\n",
    "            if (i + 1) % train_log_interval == 0:\n",
    "                train_loss = loss_value / train_log_interval\n",
    "                train_acc = matches / param.batch_size / train_log_interval\n",
    "                print(f'{name}) epoch: {epoch + 1}, batch: {i + 1}, train loss: {train_loss:5.3}, train accuracy: {train_acc:5.3%}')\n",
    "\n",
    "                loss_value = 0\n",
    "                matches = 0\n",
    "        \n",
    "        model.eval()\n",
    "        loss_value = 0\n",
    "        matches = 0\n",
    "        for i, data in enumerate(valid_data):\n",
    "            inputs, labels = data\n",
    "            inputs = inputs.to(device)\n",
    "            labels = labels.to(device)\n",
    "\n",
    "            with torch.no_grad():\n",
    "                outputs = model(inputs.permute(0, 3, 1, 2).float())\n",
    "                preds = torch.argmax(outputs, dim=-1)\n",
    "\n",
    "                loss_value += criterion(outputs, labels).item()\n",
    "                matches += (preds == labels).sum().item()\n",
    "            \n",
    "        val_loss = loss_value / len(valid_data)\n",
    "        val_acc = matches / len(valid_data) / param.batch_size\n",
    "\n",
    "        if val_loss < best_val_loss:\n",
    "            best_val_loss = val_loss\n",
    "        \n",
    "        if val_acc > best_val_acc:\n",
    "            print(\"New best model for f1-score! saving the model\")\n",
    "            save_path = path.model_save + '/' + name\n",
    "            if not os.path.exists(save_path):\n",
    "                os.makedirs(save_path)\n",
    "            torch.save(model.state_dict(), save_path + f'/{epoch:03}_accuracy_{val_acc:4.2%}.ckpt')\n",
    "            best_val_acc = val_acc\n",
    "            renew_count = 0\n",
    "        else:\n",
    "            renew_count += 1\n",
    "        \n",
    "        if renew_count > param.train_patience:\n",
    "            print(\"Early stopping\")\n",
    "            break\n",
    "\n",
    "        print('val_acc:', val_acc, 'loss:', val_loss, 'best acc:', best_val_acc, 'best loss:', best_val_loss)\n",
    "        scheduler.step(val_loss)\n",
    "\n",
    "    print('Finishing Training')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "b6fc18a74e124ceba6fe4111707d8b07"
      }
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "HBox(children=(FloatProgress(value=0.0, max=8.0), HTML(value='')))",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "4372c13cc0574aa6b26bd415867185c6"
      }
     },
     "metadata": {}
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "New best model for f1-score! saving the model\n",
      "val_acc: 0.048828125 loss: 553.6003909111023 best acc: 0.048828125 best loss: 553.6003909111023\n",
      "New best model for f1-score! saving the model\n",
      "val_acc: 0.056640625 loss: 689.5002410411835 best acc: 0.056640625 best loss: 553.6003909111023\n",
      "val_acc: 0.046875 loss: 551.7532777786255 best acc: 0.056640625 best loss: 551.7532777786255\n",
      "Exception ignored in: <function _releaseLock at 0x7fe4282a88c0>\n",
      "Traceback (most recent call last):\n",
      "  File \"/opt/conda/lib/python3.7/logging/__init__.py\", line 221, in _releaseLock\n",
      "    def _releaseLock():\n",
      "KeyboardInterrupt\n",
      "\n",
      "\n"
     ]
    },
    {
     "output_type": "error",
     "ename": "RuntimeError",
     "evalue": "DataLoader worker (pid(s) 29521, 29522, 29523) exited unexpectedly",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mEmpty\u001b[0m                                     Traceback (most recent call last)",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_try_get_data\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    871\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 872\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_data_queue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    873\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/multiprocessing/queues.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(self, block, timeout)\u001b[0m\n\u001b[1;32m    104\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_poll\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 105\u001b[0;31m                         \u001b[0;32mraise\u001b[0m \u001b[0mEmpty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    106\u001b[0m                 \u001b[0;32melif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_poll\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mEmpty\u001b[0m: ",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-15-0006206b95fb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0mcriterion\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCrossEntropyLoss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0mscheduler\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptim\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlr_scheduler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mReduceLROnPlateau\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfactor\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpatience\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m     \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34mf'last_{fold}_{param.img_size}'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrain_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalid_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalid_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscheduler\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mscheduler\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcriterion\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-14-0c0b1e2183e6>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(name, model, train_data, valid_data, optimizer, scheduler, criterion)\u001b[0m\n\u001b[1;32m     41\u001b[0m         \u001b[0mloss_value\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     42\u001b[0m         \u001b[0mmatches\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 43\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalid_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     44\u001b[0m             \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     45\u001b[0m             \u001b[0minputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    433\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sampler_iter\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    434\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 435\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    436\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_yielded\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    437\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1066\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1067\u001b[0m             \u001b[0;32massert\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_shutdown\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tasks_outstanding\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1068\u001b[0;31m             \u001b[0midx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1069\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tasks_outstanding\u001b[0m \u001b[0;34m-=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1070\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_get_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1032\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1033\u001b[0m             \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1034\u001b[0;31m                 \u001b[0msuccess\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_try_get_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1035\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0msuccess\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1036\u001b[0m                     \u001b[0;32mreturn\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_try_get_data\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    883\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfailed_workers\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    884\u001b[0m                 \u001b[0mpids_str\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m', '\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mw\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpid\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mw\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfailed_workers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 885\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'DataLoader worker (pid(s) {}) exited unexpectedly'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpids_str\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    886\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mqueue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mEmpty\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    887\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: DataLoader worker (pid(s) 29521, 29522, 29523) exited unexpectedly"
     ]
    }
   ],
   "source": [
    "for fold in tqdm(range(param.k_fold)):\n",
    "    trainset = ConcatDataset([dataset[i] for i in range(param.k_fold) if i != fold])\n",
    "    validset = dataset[fold]\n",
    "    train_data = DataLoader(trainset, batch_size=param.batch_size, shuffle=True, num_workers=param.num_workers)\n",
    "    valid_data = DataLoader(validset, batch_size=param.batch_size, shuffle=True, num_workers=param.num_workers)\n",
    "    model = timm.create_model('efficientnet_b3', pretrained=True, num_classes=18)\n",
    "    optimizer = torch.optim.Adam(filter(lambda p : p.requires_grad, model.parameters()), lr=param.lr)\n",
    "    criterion = torch.nn.CrossEntropyLoss()\n",
    "    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.1, patience=5)\n",
    "    train(name=f'last_{fold}_{param.img_size}', model=model, train_data=train_data, valid_data=valid_data, optimizer=optimizer, scheduler=scheduler, criterion=criterion)"
   ]
  }
 ]
}