training/somu-yolov5s6-train1-base-model-1-of-5-folds.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.026372,
     "end_time": "2022-02-05T14:04:21.096138",
     "exception": false,
     "start_time": "2022-02-05T14:04:21.069766",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Forked from: https://www.kaggle.com/steamedsheep/yolov5-high-resolution-training but extensively modified using my private notebooks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.023728,
     "end_time": "2022-02-05T14:04:21.145683",
     "exception": false,
     "start_time": "2022-02-05T14:04:21.121955",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Yolov5 training with Albumentations\n",
    "\n",
    "## Why this notebook?\n",
    "* There are no notebooks that show how to integrate custom augmentations into the Yolov5 training pipeline\n",
    "* Others bake the augmentations to the data prior to training.\n",
    "* This notebook shows you how to apply augmentations from the Albumentations package on the fly using yolov5's Albumentations integration"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.026612,
     "end_time": "2022-02-05T14:04:21.196593",
     "exception": false,
     "start_time": "2022-02-05T14:04:21.169981",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Log:\n",
    "|version|config|\n",
    "|-------|------|\n",
    "|v1|img_size=3000,epoch=11,bs=2,val fold = 1|\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.028883,
     "end_time": "2022-02-05T14:04:21.253959",
     "exception": false,
     "start_time": "2022-02-05T14:04:21.225076",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Modules Required"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2022-02-05T14:04:21.322033Z",
     "iopub.status.busy": "2022-02-05T14:04:21.321213Z",
     "iopub.status.idle": "2022-02-05T14:04:23.949718Z",
     "shell.execute_reply": "2022-02-05T14:04:23.949035Z",
     "shell.execute_reply.started": "2022-02-05T13:20:55.645481Z"
    },
    "papermill": {
     "duration": 2.66736,
     "end_time": "2022-02-05T14:04:23.949919",
     "exception": false,
     "start_time": "2022-02-05T14:04:21.282559",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import ast\n",
    "import os\n",
    "import json\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import torch\n",
    "import importlib\n",
    "import cv2 \n",
    "\n",
    "import shutil\n",
    "from shutil import copyfile, make_archive\n",
    "from tqdm.notebook import tqdm\n",
    "tqdm.pandas()\n",
    "from sklearn.model_selection import GroupKFold\n",
    "from PIL import Image\n",
    "from string import Template\n",
    "from IPython.display import display\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "TRAIN_PATH = '/kaggle/input/tensorflow-great-barrier-reef'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.025982,
     "end_time": "2022-02-05T14:04:24.001102",
     "exception": false,
     "start_time": "2022-02-05T14:04:23.97512",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Yolov5 Setup and Albumentations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T14:04:24.058233Z",
     "iopub.status.busy": "2022-02-05T14:04:24.057372Z",
     "iopub.status.idle": "2022-02-05T14:04:26.250212Z",
     "shell.execute_reply": "2022-02-05T14:04:26.249672Z",
     "shell.execute_reply.started": "2022-02-05T13:20:58.262247Z"
    },
    "papermill": {
     "duration": 2.223613,
     "end_time": "2022-02-05T14:04:26.250342",
     "exception": false,
     "start_time": "2022-02-05T14:04:24.026729",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# !git clone https://github.com/ultralytics/yolov5.git\n",
    "!git clone https://github.com/soumya997/yolov5-w-f2-mod.git\n",
    "!mv ./yolov5-w-f2-mod ./yolov5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.025842,
     "end_time": "2022-02-05T14:04:27.045675",
     "exception": false,
     "start_time": "2022-02-05T14:04:27.019833",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Setting your custom Albumentations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T14:04:27.105868Z",
     "iopub.status.busy": "2022-02-05T14:04:27.10421Z",
     "iopub.status.idle": "2022-02-05T14:04:27.106436Z",
     "shell.execute_reply": "2022-02-05T14:04:27.10684Z",
     "shell.execute_reply.started": "2022-02-05T13:21:02.563537Z"
    },
    "papermill": {
     "duration": 0.035261,
     "end_time": "2022-02-05T14:04:27.106988",
     "exception": false,
     "start_time": "2022-02-05T14:04:27.071727",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# # Specify your custom set of Albumentations here\n",
    "# # See https://www.kaggle.com/alexchwong/cots-albumentations-gallery for some ideas on what to include\n",
    "# # Ensure there are at least 16 spaces after each new line, for correct indentations consistent with yolov5's augmentations.py script\n",
    "\n",
    "# ################ 16 hashes here\n",
    "\n",
    "# # ALBUMENTATIONS = '''[\n",
    "# #                 # Add your Albumentations after this line\n",
    "# #                     A.OneOf([\n",
    "# #                         A.MotionBlur(p=.2),\n",
    "# #                         A.MedianBlur(blur_limit=3, p=0.3),\n",
    "# #                         A.Blur(blur_limit=3, p=0.1),\n",
    "# #                     ], p=0.3),\n",
    "# #                     A.OneOf([\n",
    "# #                         A.CLAHE(clip_limit=2),\n",
    "# #                         A.RandomBrightnessContrast(),            \n",
    "# #                     ], p=0.3),\n",
    "# #                 # Do not edit past this line\n",
    "# #                 ],\n",
    "# # # '''\n",
    "# ALBUMENTATIONS = '''[\n",
    "#                 # Add your Albumentations after this line\n",
    "#                     A.OneOf([\n",
    "#                         A.HueSaturationValue(hue_shift_limit = 0.2, \n",
    "#                                              sat_shift_limit = 0.2,\n",
    "#                                              val_shift_limit = 0.2,\n",
    "#                                              p = 0.3), \n",
    "\n",
    "#                         A.RandomBrightnessContrast(brightness_limit = 0.2,                                             \n",
    "#                                                    contrast_limit = 0.2,\n",
    "#                                                    p = 0.3),\n",
    "#                         A.RGBShift(r_shift_limit = 20/255, \n",
    "#                                    g_shift_limit = 20/255, \n",
    "#                                    b_shift_limit = 10/255,\n",
    "#                                    p = 0.3)\n",
    "#                     ], \n",
    "#                     p = 0.2),\n",
    "#                 A.OneOf([\n",
    "#                             A.RandomGamma(gamma_limit = (80, 120),p = 0.3),\n",
    "#                             A.Blur(p = 0.6),\n",
    "#                             A.GaussNoise(var_limit = (0.01, 0.05), mean = 0, p = 0.05),\n",
    "#                             A.ToGray(p = 0.05),\n",
    "#                             A.Solarize(p=0.3)\n",
    "\n",
    "#                         ],\n",
    "#                         p = 0.1),\n",
    "#                 A.ToGray(p=0.01),\n",
    "#                 A.RandomFog(fog_coef_lower = 0.1,\n",
    "#                                 fog_coef_upper = 0.2,\n",
    "#                                 p = 0.03),          \n",
    "#                 # Do not edit past this line\n",
    "#                 ],\n",
    "# '''\n",
    "\n",
    "# ################ If you are not convinced whether your Albumentations are working, uncomment the following line to make all the training pictures gray at the end:\n",
    "\n",
    "# # ALBUMENTATIONS = \"[A.ToGray(p=1.0),],\"\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.025995,
     "end_time": "2022-02-05T14:04:27.159068",
     "exception": false,
     "start_time": "2022-02-05T14:04:27.133073",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Yolov5's hyperparameters file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T14:04:27.219362Z",
     "iopub.status.busy": "2022-02-05T14:04:27.21775Z",
     "iopub.status.idle": "2022-02-05T14:04:27.22004Z",
     "shell.execute_reply": "2022-02-05T14:04:27.220439Z",
     "shell.execute_reply.started": "2022-02-05T13:21:02.573518Z"
    },
    "papermill": {
     "duration": 0.034738,
     "end_time": "2022-02-05T14:04:27.220602",
     "exception": false,
     "start_time": "2022-02-05T14:04:27.185864",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# From yolov5/data/hyps/hyp.scratch.yaml\n",
    "\n",
    "# Change yolov5's built-in augmentations / training hyperparameters here\n",
    "\n",
    "hyps = '''\n",
    "\n",
    "# YOLOv5 🚀 by Ultralytics, GPL-3.0 license\n",
    "# Hyperparameters for COCO training from scratch\n",
    "# python train.py --batch 40 --cfg yolov5m.yaml --weights '' --data coco.yaml --img 640 --epochs 300\n",
    "# See tutorials for hyperparameter evolution https://github.com/ultralytics/yolov5#tutorials\n",
    "\n",
    "lr0: 0.01  # initial learning rate (SGD=1E-2, Adam=1E-3)\n",
    "lrf: 0.1  # final OneCycleLR learning rate (lr0 * lrf)\n",
    "momentum: 0.937  # SGD momentum/Adam beta1\n",
    "weight_decay: 0.0005  # optimizer weight decay 5e-4\n",
    "warmup_epochs: 3.0  # warmup epochs (fractions ok)\n",
    "warmup_momentum: 0.8  # warmup initial momentum\n",
    "warmup_bias_lr: 0.1  # warmup initial bias lr\n",
    "box: 0.05  # box loss gain\n",
    "cls: 0.5  # cls loss gain\n",
    "cls_pw: 1.0  # cls BCELoss positive_weight\n",
    "obj: 1.0  # obj loss gain (scale with pixels)\n",
    "obj_pw: 1.0  # obj BCELoss positive_weight\n",
    "iou_t: 0.20  # IoU training threshold\n",
    "anchor_t: 4.0  # anchor-multiple threshold\n",
    "# anchors: 3  # anchors per output layer (0 to ignore)\n",
    "fl_gamma: 0.0  # focal loss gamma (efficientDet default gamma=1.5)\n",
    "hsv_h: 0.015  # image HSV-Hue augmentation (fraction)\n",
    "hsv_s: 0.7  # image HSV-Saturation augmentation (fraction)\n",
    "hsv_v: 0.4  # image HSV-Value augmentation (fraction)\n",
    "degrees: 0.0  # image rotation (+/- deg)\n",
    "translate: 0.1  # image translation (+/- fraction)\n",
    "scale: 0.5  # image scale (+/- gain)\n",
    "shear: 0.0  # image shear (+/- deg)\n",
    "perspective: 0.0  # image perspective (+/- fraction), range 0-0.001\n",
    "flipud: 0.0  # image flip up-down (probability)\n",
    "fliplr: 0.5  # image flip left-right (probability)\n",
    "mosaic: 1.0  # image mosaic (probability)\n",
    "mixup: 0.0  # image mixup (probability)\n",
    "copy_paste: 0.0  # segment copy-paste (probability)\n",
    "\n",
    "'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.025813,
     "end_time": "2022-02-05T14:04:27.27276",
     "exception": false,
     "start_time": "2022-02-05T14:04:27.246947",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Define Model YAML for 1 class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T14:04:27.332461Z",
     "iopub.status.busy": "2022-02-05T14:04:27.331675Z",
     "iopub.status.idle": "2022-02-05T14:04:28.635189Z",
     "shell.execute_reply": "2022-02-05T14:04:28.634628Z",
     "shell.execute_reply.started": "2022-02-05T13:21:02.593491Z"
    },
    "papermill": {
     "duration": 1.336691,
     "end_time": "2022-02-05T14:04:28.635329",
     "exception": false,
     "start_time": "2022-02-05T14:04:27.298638",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# edit number of classes to 1\n",
    "copyfile(\"/kaggle/working/yolov5/models/yolov5s.yaml\", \"/kaggle/working/yolov5/yolov5s.yaml\")   \n",
    "!sed -i 's/nc: 80/nc: 1/g' /kaggle/working/yolov5/yolov5s.yaml\n",
    "!cat '/kaggle/working/yolov5/yolov5s.yaml'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.026299,
     "end_time": "2022-02-05T14:04:28.68878",
     "exception": false,
     "start_time": "2022-02-05T14:04:28.662481",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Write yaml files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T14:04:28.746784Z",
     "iopub.status.busy": "2022-02-05T14:04:28.746201Z",
     "iopub.status.idle": "2022-02-05T14:04:28.749096Z",
     "shell.execute_reply": "2022-02-05T14:04:28.748662Z",
     "shell.execute_reply.started": "2022-02-05T13:21:04.198886Z"
    },
    "papermill": {
     "duration": 0.033924,
     "end_time": "2022-02-05T14:04:28.749218",
     "exception": false,
     "start_time": "2022-02-05T14:04:28.715294",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "cots_data = '''\n",
    "train: ../images/train\n",
    "val: ../images/valid\n",
    "\n",
    "nc: 1\n",
    "names: [cots]\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T14:04:28.807617Z",
     "iopub.status.busy": "2022-02-05T14:04:28.806968Z",
     "iopub.status.idle": "2022-02-05T14:04:28.809673Z",
     "shell.execute_reply": "2022-02-05T14:04:28.809251Z",
     "shell.execute_reply.started": "2022-02-05T13:21:04.204844Z"
    },
    "papermill": {
     "duration": 0.033945,
     "end_time": "2022-02-05T14:04:28.80978",
     "exception": false,
     "start_time": "2022-02-05T14:04:28.775835",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "with open('./yolov5/data/cots.yaml', 'w') as fp:\n",
    "    fp.write(cots_data)\n",
    "with open('./yolov5/data/hyps/hyps_custom.yaml', 'w') as fp:\n",
    "    fp.write(hyps)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.026035,
     "end_time": "2022-02-05T14:04:28.862213",
     "exception": false,
     "start_time": "2022-02-05T14:04:28.836178",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Bake your custom albumentations into Yolov5's augmentations.py script"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true,
    "execution": {
     "iopub.execute_input": "2022-02-05T14:04:28.928589Z",
     "iopub.status.busy": "2022-02-05T14:04:28.918766Z",
     "iopub.status.idle": "2022-02-05T14:04:28.930911Z",
     "shell.execute_reply": "2022-02-05T14:04:28.930472Z",
     "shell.execute_reply.started": "2022-02-05T13:21:04.221916Z"
    },
    "jupyter": {
     "source_hidden": true
    },
    "papermill": {
     "duration": 0.042667,
     "end_time": "2022-02-05T14:04:28.931043",
     "exception": false,
     "start_time": "2022-02-05T14:04:28.888376",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# augmentations_template = '''\n",
    "# # YOLOv5 🚀 by Ultralytics, GPL-3.0 license\n",
    "# \"\"\"\n",
    "# Image augmentation functions\n",
    "# \"\"\"\n",
    "\n",
    "# import math\n",
    "# import random\n",
    "\n",
    "# import cv2\n",
    "# import numpy as np\n",
    "\n",
    "# from utils.general import LOGGER, check_version, colorstr, resample_segments, segment2box\n",
    "# from utils.metrics import bbox_ioa\n",
    "\n",
    "\n",
    "# class Albumentations:\n",
    "#     # YOLOv5 Albumentations class (optional, only used if package is installed)\n",
    "#     def __init__(self):\n",
    "#         self.transform = None\n",
    "#         try:\n",
    "#             import albumentations as A\n",
    "#             # check_version(A.__version__, '1.0.3', hard=True)  # version requirement\n",
    "\n",
    "#             self.transform = A.Compose($albumentations\n",
    "#                 bbox_params=A.BboxParams(format='yolo', label_fields=['class_labels']))\n",
    "\n",
    "#             LOGGER.info(colorstr('albumentations: ') + ', '.join(f'{x}' for x in self.transform.transforms if x.p))\n",
    "#         except ImportError:  # package not installed, skip\n",
    "#             pass\n",
    "#         except Exception as e:\n",
    "#             LOGGER.info(colorstr('albumentations: ') + f'{e}')\n",
    "\n",
    "#     def __call__(self, im, labels, p=1.0):\n",
    "#         if self.transform and random.random() < p:\n",
    "#             new = self.transform(image=im, bboxes=labels[:, 1:], class_labels=labels[:, 0])  # transformed\n",
    "#             im, labels = new['image'], np.array([[c, *b] for c, b in zip(new['class_labels'], new['bboxes'])])\n",
    "#         return im, labels\n",
    "\n",
    "\n",
    "# def augment_hsv(im, hgain=0.5, sgain=0.5, vgain=0.5):\n",
    "#     # HSV color-space augmentation\n",
    "#     if hgain or sgain or vgain:\n",
    "#         r = np.random.uniform(-1, 1, 3) * [hgain, sgain, vgain] + 1  # random gains\n",
    "#         hue, sat, val = cv2.split(cv2.cvtColor(im, cv2.COLOR_BGR2HSV))\n",
    "#         dtype = im.dtype  # uint8\n",
    "\n",
    "#         x = np.arange(0, 256, dtype=r.dtype)\n",
    "#         lut_hue = ((x * r[0]) % 180).astype(dtype)\n",
    "#         lut_sat = np.clip(x * r[1], 0, 255).astype(dtype)\n",
    "#         lut_val = np.clip(x * r[2], 0, 255).astype(dtype)\n",
    "\n",
    "#         im_hsv = cv2.merge((cv2.LUT(hue, lut_hue), cv2.LUT(sat, lut_sat), cv2.LUT(val, lut_val)))\n",
    "#         cv2.cvtColor(im_hsv, cv2.COLOR_HSV2BGR, dst=im)  # no return needed\n",
    "\n",
    "\n",
    "# def hist_equalize(im, clahe=True, bgr=False):\n",
    "#     # Equalize histogram on BGR image 'im' with im.shape(n,m,3) and range 0-255\n",
    "#     yuv = cv2.cvtColor(im, cv2.COLOR_BGR2YUV if bgr else cv2.COLOR_RGB2YUV)\n",
    "#     if clahe:\n",
    "#         c = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))\n",
    "#         yuv[:, :, 0] = c.apply(yuv[:, :, 0])\n",
    "#     else:\n",
    "#         yuv[:, :, 0] = cv2.equalizeHist(yuv[:, :, 0])  # equalize Y channel histogram\n",
    "#     return cv2.cvtColor(yuv, cv2.COLOR_YUV2BGR if bgr else cv2.COLOR_YUV2RGB)  # convert YUV image to RGB\n",
    "\n",
    "\n",
    "# def replicate(im, labels):\n",
    "#     # Replicate labels\n",
    "#     h, w = im.shape[:2]\n",
    "#     boxes = labels[:, 1:].astype(int)\n",
    "#     x1, y1, x2, y2 = boxes.T\n",
    "#     s = ((x2 - x1) + (y2 - y1)) / 2  # side length (pixels)\n",
    "#     for i in s.argsort()[:round(s.size * 0.5)]:  # smallest indices\n",
    "#         x1b, y1b, x2b, y2b = boxes[i]\n",
    "#         bh, bw = y2b - y1b, x2b - x1b\n",
    "#         yc, xc = int(random.uniform(0, h - bh)), int(random.uniform(0, w - bw))  # offset x, y\n",
    "#         x1a, y1a, x2a, y2a = [xc, yc, xc + bw, yc + bh]\n",
    "#         im[y1a:y2a, x1a:x2a] = im[y1b:y2b, x1b:x2b]  # im4[ymin:ymax, xmin:xmax]\n",
    "#         labels = np.append(labels, [[labels[i, 0], x1a, y1a, x2a, y2a]], axis=0)\n",
    "\n",
    "#     return im, labels\n",
    "\n",
    "\n",
    "# def letterbox(im, new_shape=(640, 640), color=(114, 114, 114), auto=True, scaleFill=False, scaleup=True, stride=32):\n",
    "#     # Resize and pad image while meeting stride-multiple constraints\n",
    "#     shape = im.shape[:2]  # current shape [height, width]\n",
    "#     if isinstance(new_shape, int):\n",
    "#         new_shape = (new_shape, new_shape)\n",
    "\n",
    "#     # Scale ratio (new / old)\n",
    "#     r = min(new_shape[0] / shape[0], new_shape[1] / shape[1])\n",
    "#     if not scaleup:  # only scale down, do not scale up (for better val mAP)\n",
    "#         r = min(r, 1.0)\n",
    "\n",
    "#     # Compute padding\n",
    "#     ratio = r, r  # width, height ratios\n",
    "#     new_unpad = int(round(shape[1] * r)), int(round(shape[0] * r))\n",
    "#     dw, dh = new_shape[1] - new_unpad[0], new_shape[0] - new_unpad[1]  # wh padding\n",
    "#     if auto:  # minimum rectangle\n",
    "#         dw, dh = np.mod(dw, stride), np.mod(dh, stride)  # wh padding\n",
    "#     elif scaleFill:  # stretch\n",
    "#         dw, dh = 0.0, 0.0\n",
    "#         new_unpad = (new_shape[1], new_shape[0])\n",
    "#         ratio = new_shape[1] / shape[1], new_shape[0] / shape[0]  # width, height ratios\n",
    "\n",
    "#     dw /= 2  # divide padding into 2 sides\n",
    "#     dh /= 2\n",
    "\n",
    "#     if shape[::-1] != new_unpad:  # resize\n",
    "#         im = cv2.resize(im, new_unpad, interpolation=cv2.INTER_LINEAR)\n",
    "#     top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1))\n",
    "#     left, right = int(round(dw - 0.1)), int(round(dw + 0.1))\n",
    "#     im = cv2.copyMakeBorder(im, top, bottom, left, right, cv2.BORDER_CONSTANT, value=color)  # add border\n",
    "#     return im, ratio, (dw, dh)\n",
    "\n",
    "\n",
    "# def random_perspective(im, targets=(), segments=(), degrees=10, translate=.1, scale=.1, shear=10, perspective=0.0,\n",
    "#                        border=(0, 0)):\n",
    "#     # torchvision.transforms.RandomAffine(degrees=(-10, 10), translate=(0.1, 0.1), scale=(0.9, 1.1), shear=(-10, 10))\n",
    "#     # targets = [cls, xyxy]\n",
    "\n",
    "#     height = im.shape[0] + border[0] * 2  # shape(h,w,c)\n",
    "#     width = im.shape[1] + border[1] * 2\n",
    "\n",
    "#     # Center\n",
    "#     C = np.eye(3)\n",
    "#     C[0, 2] = -im.shape[1] / 2  # x translation (pixels)\n",
    "#     C[1, 2] = -im.shape[0] / 2  # y translation (pixels)\n",
    "\n",
    "#     # Perspective\n",
    "#     P = np.eye(3)\n",
    "#     P[2, 0] = random.uniform(-perspective, perspective)  # x perspective (about y)\n",
    "#     P[2, 1] = random.uniform(-perspective, perspective)  # y perspective (about x)\n",
    "\n",
    "#     # Rotation and Scale\n",
    "#     R = np.eye(3)\n",
    "#     a = random.uniform(-degrees, degrees)\n",
    "#     # a += random.choice([-180, -90, 0, 90])  # add 90deg rotations to small rotations\n",
    "#     s = random.uniform(1 - scale, 1 + scale)\n",
    "#     # s = 2 ** random.uniform(-scale, scale)\n",
    "#     R[:2] = cv2.getRotationMatrix2D(angle=a, center=(0, 0), scale=s)\n",
    "\n",
    "#     # Shear\n",
    "#     S = np.eye(3)\n",
    "#     S[0, 1] = math.tan(random.uniform(-shear, shear) * math.pi / 180)  # x shear (deg)\n",
    "#     S[1, 0] = math.tan(random.uniform(-shear, shear) * math.pi / 180)  # y shear (deg)\n",
    "\n",
    "#     # Translation\n",
    "#     T = np.eye(3)\n",
    "#     T[0, 2] = random.uniform(0.5 - translate, 0.5 + translate) * width  # x translation (pixels)\n",
    "#     T[1, 2] = random.uniform(0.5 - translate, 0.5 + translate) * height  # y translation (pixels)\n",
    "\n",
    "#     # Combined rotation matrix\n",
    "#     M = T @ S @ R @ P @ C  # order of operations (right to left) is IMPORTANT\n",
    "#     if (border[0] != 0) or (border[1] != 0) or (M != np.eye(3)).any():  # image changed\n",
    "#         if perspective:\n",
    "#             im = cv2.warpPerspective(im, M, dsize=(width, height), borderValue=(114, 114, 114))\n",
    "#         else:  # affine\n",
    "#             im = cv2.warpAffine(im, M[:2], dsize=(width, height), borderValue=(114, 114, 114))\n",
    "\n",
    "#     # Visualize\n",
    "#     # import matplotlib.pyplot as plt\n",
    "#     # ax = plt.subplots(1, 2, figsize=(12, 6))[1].ravel()\n",
    "#     # ax[0].imshow(im[:, :, ::-1])  # base\n",
    "#     # ax[1].imshow(im2[:, :, ::-1])  # warped\n",
    "\n",
    "#     # Transform label coordinates\n",
    "#     n = len(targets)\n",
    "#     if n:\n",
    "#         use_segments = any(x.any() for x in segments)\n",
    "#         new = np.zeros((n, 4))\n",
    "#         if use_segments:  # warp segments\n",
    "#             segments = resample_segments(segments)  # upsample\n",
    "#             for i, segment in enumerate(segments):\n",
    "#                 xy = np.ones((len(segment), 3))\n",
    "#                 xy[:, :2] = segment\n",
    "#                 xy = xy @ M.T  # transform\n",
    "#                 xy = xy[:, :2] / xy[:, 2:3] if perspective else xy[:, :2]  # perspective rescale or affine\n",
    "\n",
    "#                 # clip\n",
    "#                 new[i] = segment2box(xy, width, height)\n",
    "\n",
    "#         else:  # warp boxes\n",
    "#             xy = np.ones((n * 4, 3))\n",
    "#             xy[:, :2] = targets[:, [1, 2, 3, 4, 1, 4, 3, 2]].reshape(n * 4, 2)  # x1y1, x2y2, x1y2, x2y1\n",
    "#             xy = xy @ M.T  # transform\n",
    "#             xy = (xy[:, :2] / xy[:, 2:3] if perspective else xy[:, :2]).reshape(n, 8)  # perspective rescale or affine\n",
    "\n",
    "#             # create new boxes\n",
    "#             x = xy[:, [0, 2, 4, 6]]\n",
    "#             y = xy[:, [1, 3, 5, 7]]\n",
    "#             new = np.concatenate((x.min(1), y.min(1), x.max(1), y.max(1))).reshape(4, n).T\n",
    "\n",
    "#             # clip\n",
    "#             new[:, [0, 2]] = new[:, [0, 2]].clip(0, width)\n",
    "#             new[:, [1, 3]] = new[:, [1, 3]].clip(0, height)\n",
    "\n",
    "#         # filter candidates\n",
    "#         i = box_candidates(box1=targets[:, 1:5].T * s, box2=new.T, area_thr=0.01 if use_segments else 0.10)\n",
    "#         targets = targets[i]\n",
    "#         targets[:, 1:5] = new[i]\n",
    "\n",
    "#     return im, targets\n",
    "\n",
    "\n",
    "# def copy_paste(im, labels, segments, p=0.5):\n",
    "#     # Implement Copy-Paste augmentation https://arxiv.org/abs/2012.07177, labels as nx5 np.array(cls, xyxy)\n",
    "#     n = len(segments)\n",
    "#     if p and n:\n",
    "#         h, w, c = im.shape  # height, width, channels\n",
    "#         im_new = np.zeros(im.shape, np.uint8)\n",
    "#         for j in random.sample(range(n), k=round(p * n)):\n",
    "#             l, s = labels[j], segments[j]\n",
    "#             box = w - l[3], l[2], w - l[1], l[4]\n",
    "#             ioa = bbox_ioa(box, labels[:, 1:5])  # intersection over area\n",
    "#             if (ioa < 0.30).all():  # allow 30% obscuration of existing labels\n",
    "#                 labels = np.concatenate((labels, [[l[0], *box]]), 0)\n",
    "#                 segments.append(np.concatenate((w - s[:, 0:1], s[:, 1:2]), 1))\n",
    "#                 cv2.drawContours(im_new, [segments[j].astype(np.int32)], -1, (255, 255, 255), cv2.FILLED)\n",
    "\n",
    "#         result = cv2.bitwise_and(src1=im, src2=im_new)\n",
    "#         result = cv2.flip(result, 1)  # augment segments (flip left-right)\n",
    "#         i = result > 0  # pixels to replace\n",
    "#         # i[:, :] = result.max(2).reshape(h, w, 1)  # act over ch\n",
    "#         im[i] = result[i]  # cv2.imwrite('debug.jpg', im)  # debug\n",
    "\n",
    "#     return im, labels, segments\n",
    "\n",
    "\n",
    "# def cutout(im, labels, p=0.5):\n",
    "#     # Applies image cutout augmentation https://arxiv.org/abs/1708.04552\n",
    "#     if random.random() < p:\n",
    "#         h, w = im.shape[:2]\n",
    "#         scales = [0.5] * 1 + [0.25] * 2 + [0.125] * 4 + [0.0625] * 8 + [0.03125] * 16  # image size fraction\n",
    "#         for s in scales:\n",
    "#             mask_h = random.randint(1, int(h * s))  # create random masks\n",
    "#             mask_w = random.randint(1, int(w * s))\n",
    "\n",
    "#             # box\n",
    "#             xmin = max(0, random.randint(0, w) - mask_w // 2)\n",
    "#             ymin = max(0, random.randint(0, h) - mask_h // 2)\n",
    "#             xmax = min(w, xmin + mask_w)\n",
    "#             ymax = min(h, ymin + mask_h)\n",
    "\n",
    "#             # apply random color mask\n",
    "#             im[ymin:ymax, xmin:xmax] = [random.randint(64, 191) for _ in range(3)]\n",
    "\n",
    "#             # return unobscured labels\n",
    "#             if len(labels) and s > 0.03:\n",
    "#                 box = np.array([xmin, ymin, xmax, ymax], dtype=np.float32)\n",
    "#                 ioa = bbox_ioa(box, labels[:, 1:5])  # intersection over area\n",
    "#                 labels = labels[ioa < 0.60]  # remove >60% obscured labels\n",
    "\n",
    "#     return labels\n",
    "\n",
    "\n",
    "# def mixup(im, labels, im2, labels2):\n",
    "#     # Applies MixUp augmentation https://arxiv.org/pdf/1710.09412.pdf\n",
    "#     r = np.random.beta(32.0, 32.0)  # mixup ratio, alpha=beta=32.0\n",
    "#     im = (im * r + im2 * (1 - r)).astype(np.uint8)\n",
    "#     labels = np.concatenate((labels, labels2), 0)\n",
    "#     return im, labels\n",
    "\n",
    "\n",
    "# def box_candidates(box1, box2, wh_thr=2, ar_thr=100, area_thr=0.1, eps=1e-16):  # box1(4,n), box2(4,n)\n",
    "#     # Compute candidate boxes: box1 before augment, box2 after augment, wh_thr (pixels), aspect_ratio_thr, area_ratio\n",
    "#     w1, h1 = box1[2] - box1[0], box1[3] - box1[1]\n",
    "#     w2, h2 = box2[2] - box2[0], box2[3] - box2[1]\n",
    "#     ar = np.maximum(w2 / (h2 + eps), h2 / (w2 + eps))  # aspect ratio\n",
    "#     return (w2 > wh_thr) & (h2 > wh_thr) & (w2 * h2 / (w1 * h1 + eps) > area_thr) & (ar < ar_thr)  # candidates\n",
    "\n",
    "# '''\n",
    "\n",
    "# augmentations = Template(augmentations_template).substitute(\n",
    "#     albumentations = ALBUMENTATIONS,\n",
    "# )\n",
    "\n",
    "# with open('/kaggle/working/yolov5/utils/augmentations.py', 'w') as f:\n",
    "#     f.write(augmentations)\n",
    "\n",
    "# !cat /kaggle/working/yolov5/utils/augmentations.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.026348,
     "end_time": "2022-02-05T14:04:28.983699",
     "exception": false,
     "start_time": "2022-02-05T14:04:28.957351",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Data Prep\n",
    "\n",
    "* This data prep uses 5-fold split using \"sequence\"\n",
    "* Fold 4 is used as validation dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T14:04:29.042047Z",
     "iopub.status.busy": "2022-02-05T14:04:29.041167Z",
     "iopub.status.idle": "2022-02-05T14:04:29.043015Z",
     "shell.execute_reply": "2022-02-05T14:04:29.043448Z",
     "shell.execute_reply.started": "2022-02-05T13:21:04.235991Z"
    },
    "papermill": {
     "duration": 0.032996,
     "end_time": "2022-02-05T14:04:29.043588",
     "exception": false,
     "start_time": "2022-02-05T14:04:29.010592",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Control Panel\n",
    "\n",
    "N_FOLDS = 5\n",
    "VALIDATION_FOLD = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true,
    "execution": {
     "iopub.execute_input": "2022-02-05T14:04:29.102192Z",
     "iopub.status.busy": "2022-02-05T14:04:29.10138Z",
     "iopub.status.idle": "2022-02-05T14:04:29.10335Z",
     "shell.execute_reply": "2022-02-05T14:04:29.10375Z",
     "shell.execute_reply.started": "2022-02-05T13:21:04.255828Z"
    },
    "papermill": {
     "duration": 0.033893,
     "end_time": "2022-02-05T14:04:29.103907",
     "exception": false,
     "start_time": "2022-02-05T14:04:29.070014",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "TRAIN_PATH = '/kaggle/input/tensorflow-great-barrier-reef'\n",
    "\n",
    "def get_bbox(annots):\n",
    "    bboxes = [list(annot.values()) for annot in annots]\n",
    "    return bboxes\n",
    "\n",
    "def get_path(row):\n",
    "    row['image_path'] = f'{TRAIN_PATH}/train_images/video_{row.video_id}/{row.video_frame}.jpg'\n",
    "    return row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true,
    "execution": {
     "iopub.execute_input": "2022-02-05T14:04:29.159442Z",
     "iopub.status.busy": "2022-02-05T14:04:29.158652Z",
     "iopub.status.idle": "2022-02-05T14:04:29.166387Z",
     "shell.execute_reply": "2022-02-05T14:04:29.165986Z",
     "shell.execute_reply.started": "2022-02-05T13:21:04.269909Z"
    },
    "papermill": {
     "duration": 0.03634,
     "end_time": "2022-02-05T14:04:29.166497",
     "exception": false,
     "start_time": "2022-02-05T14:04:29.130157",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# hide\n",
    "\n",
    "def coco2yolo(image_height, image_width, bboxes):\n",
    "    \"\"\"\n",
    "    coco => [xmin, ymin, w, h]\n",
    "    yolo => [xmid, ymid, w, h] (normalized)\n",
    "    \"\"\"\n",
    "    \n",
    "    bboxes = bboxes.copy().astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n",
    "    \n",
    "    for bbox in bboxes:\n",
    "        if bbox[0] + bbox[2] >= image_width:\n",
    "            bbox[2] = image_width - bbox[0] - 1\n",
    "        if bbox[1] + bbox[3] >= image_height:\n",
    "            bbox[3] = image_height - bbox[1] - 1\n",
    "    \n",
    "    # normolizinig\n",
    "    bboxes[..., [0, 2]]= bboxes[..., [0, 2]]/ image_width\n",
    "    bboxes[..., [1, 3]]= bboxes[..., [1, 3]]/ image_height\n",
    "    \n",
    "    # converstion (xmin, ymin) => (xmid, ymid)\n",
    "    bboxes[..., [0, 1]] = bboxes[..., [0, 1]] + bboxes[..., [2, 3]]/2\n",
    "    \n",
    "    return bboxes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true,
    "execution": {
     "iopub.execute_input": "2022-02-05T14:04:29.22603Z",
     "iopub.status.busy": "2022-02-05T14:04:29.225505Z",
     "iopub.status.idle": "2022-02-05T14:04:32.841533Z",
     "shell.execute_reply": "2022-02-05T14:04:32.840595Z",
     "shell.execute_reply.started": "2022-02-05T13:21:04.286274Z"
    },
    "papermill": {
     "duration": 3.648463,
     "end_time": "2022-02-05T14:04:32.841665",
     "exception": false,
     "start_time": "2022-02-05T14:04:29.193202",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"/kaggle/input/tensorflow-great-barrier-reef/train.csv\")\n",
    "\n",
    "# Taken only annotated photos\n",
    "df[\"num_bbox\"] = df['annotations'].apply(lambda x: str.count(x, 'x'))\n",
    "df_train = df[df[\"num_bbox\"]>0]\n",
    "\n",
    "# Annotations \n",
    "df_train['annotations'] = df_train['annotations'].progress_apply(lambda x: ast.literal_eval(x))\n",
    "df_train['bboxes'] = df_train.annotations.progress_apply(get_bbox)\n",
    "\n",
    "# Images resolution\n",
    "df_train[\"width\"] = 1280\n",
    "df_train[\"height\"] = 720\n",
    "\n",
    "#Path of images\n",
    "df_train = df_train.progress_apply(get_path, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true,
    "execution": {
     "iopub.execute_input": "2022-02-05T14:04:32.904404Z",
     "iopub.status.busy": "2022-02-05T14:04:32.903493Z",
     "iopub.status.idle": "2022-02-05T14:04:32.931665Z",
     "shell.execute_reply": "2022-02-05T14:04:32.932153Z",
     "shell.execute_reply.started": "2022-02-05T13:21:08.156279Z"
    },
    "papermill": {
     "duration": 0.062196,
     "end_time": "2022-02-05T14:04:32.932304",
     "exception": false,
     "start_time": "2022-02-05T14:04:32.870108",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "kf = GroupKFold(n_splits = N_FOLDS) \n",
    "df_train = df_train.reset_index(drop=True)\n",
    "df_train['fold'] = -1\n",
    "for fold, (train_idx, val_idx) in enumerate(kf.split(df_train, y = df_train.video_id.tolist(), groups=df_train.sequence)):\n",
    "#     print(df_train.loc[val_idx, 'fold'])\n",
    "    df_train.loc[val_idx, 'fold'] = fold\n",
    "\n",
    "df_train.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T14:04:32.991794Z",
     "iopub.status.busy": "2022-02-05T14:04:32.991014Z",
     "iopub.status.idle": "2022-02-05T14:04:32.999294Z",
     "shell.execute_reply": "2022-02-05T14:04:32.998837Z",
     "shell.execute_reply.started": "2022-02-05T13:21:08.192177Z"
    },
    "papermill": {
     "duration": 0.038969,
     "end_time": "2022-02-05T14:04:32.999404",
     "exception": false,
     "start_time": "2022-02-05T14:04:32.960435",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_train[\"fold\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true,
    "execution": {
     "iopub.execute_input": "2022-02-05T14:04:33.100099Z",
     "iopub.status.busy": "2022-02-05T14:04:33.089743Z",
     "iopub.status.idle": "2022-02-05T14:05:33.457796Z",
     "shell.execute_reply": "2022-02-05T14:05:33.458235Z",
     "shell.execute_reply.started": "2022-02-05T13:21:08.204122Z"
    },
    "papermill": {
     "duration": 60.43066,
     "end_time": "2022-02-05T14:05:33.458403",
     "exception": false,
     "start_time": "2022-02-05T14:04:33.027743",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "IMAGE_DIR = \"/kaggle/working/images\"\n",
    "LABEL_DIR = \"/kaggle/working/labels\"\n",
    "\n",
    "!mkdir -p {IMAGE_DIR}\n",
    "!mkdir -p {LABEL_DIR}\n",
    "\n",
    "!mkdir -p {IMAGE_DIR + '/train'}\n",
    "!mkdir -p {IMAGE_DIR + '/valid'}\n",
    "!mkdir -p {LABEL_DIR + '/train'}\n",
    "!mkdir -p {LABEL_DIR + '/valid'}\n",
    "\n",
    "def writeLabels(bboxes, destfile, image_width = 1280, image_height = 720):\n",
    "    bboxes_coco  = np.array(bboxes).astype(np.float32).copy()\n",
    "    num_bbox     = len(bboxes_coco)\n",
    "    names        = ['cots'] * num_bbox\n",
    "    labels       = [0] * num_bbox\n",
    "    with open(destfile, 'w') as f:\n",
    "        if num_bbox<1:\n",
    "            annot = ''\n",
    "            f.write(annot)\n",
    "        else:\n",
    "            bboxes_yolo  = coco2yolo(image_height, image_width, bboxes_coco)\n",
    "            for bbox_idx in range(len(bboxes_yolo)):\n",
    "                annot = [str(labels[bbox_idx])]+ list(bboxes_yolo[bbox_idx].astype(str))+(['\\n'] if num_bbox!=(bbox_idx+1) else [''])\n",
    "                annot = ' '.join(annot)\n",
    "                annot = annot.strip(' ')\n",
    "                f.write(annot)\n",
    "    return ''\n",
    "\n",
    "df1 = df_train[df_train.fold != VALIDATION_FOLD]\n",
    "for row_idx in tqdm(range(len(df1))):\n",
    "    row = df1.iloc[row_idx]\n",
    "    shutil.copyfile(row.image_path, f'{IMAGE_DIR}/train/{row.image_id}.jpg')\n",
    "    writeLabels(row.bboxes, f'{LABEL_DIR}/train/{row.image_id}.txt')\n",
    "\n",
    "df2 = df_train[df_train.fold == VALIDATION_FOLD]\n",
    "for row_idx in tqdm(range(len(df2))):\n",
    "    row = df2.iloc[row_idx]\n",
    "    shutil.copyfile(row.image_path, f'{IMAGE_DIR}/valid/{row.image_id}.jpg')\n",
    "    writeLabels(row.bboxes, f'{LABEL_DIR}/valid/{row.image_id}.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.029108,
     "end_time": "2022-02-05T14:05:33.517333",
     "exception": false,
     "start_time": "2022-02-05T14:05:33.488225",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Training the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T14:05:33.58307Z",
     "iopub.status.busy": "2022-02-05T14:05:33.582337Z",
     "iopub.status.idle": "2022-02-05T14:05:33.585201Z",
     "shell.execute_reply": "2022-02-05T14:05:33.585609Z",
     "shell.execute_reply.started": "2022-02-05T13:22:23.022248Z"
    },
    "papermill": {
     "duration": 0.038583,
     "end_time": "2022-02-05T14:05:33.585745",
     "exception": false,
     "start_time": "2022-02-05T14:05:33.547162",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "%cd yolov5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.029218,
     "end_time": "2022-02-05T14:05:33.644215",
     "exception": false,
     "start_time": "2022-02-05T14:05:33.614997",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### For demo purposes, only 1 epoch of yolov5s trained @ 1280p to make this notebook run quickly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T14:05:33.710181Z",
     "iopub.status.busy": "2022-02-05T14:05:33.709387Z",
     "iopub.status.idle": "2022-02-05T21:05:51.841326Z",
     "shell.execute_reply": "2022-02-05T21:05:51.84049Z",
     "shell.execute_reply.started": "2022-02-05T13:22:23.03351Z"
    },
    "papermill": {
     "duration": 25218.167778,
     "end_time": "2022-02-05T21:05:51.841496",
     "exception": false,
     "start_time": "2022-02-05T14:05:33.673718",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "!python -m wandb disabled\n",
    "\n",
    "!python train.py \\\n",
    "    --img 3000 \\\n",
    "    --batch 2 \\\n",
    "    --epochs 11 \\\n",
    "    --data data/cots.yaml \\\n",
    "    --cfg yolov5s.yaml \\\n",
    "    --weights yolov5s6.pt \\\n",
    "    --name cots_with_albs \\\n",
    "    --hyp data/hyps/hyps_custom.yaml \\\n",
    "    --save-period 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T21:06:05.478767Z",
     "iopub.status.busy": "2022-02-05T21:06:05.477849Z",
     "iopub.status.idle": "2022-02-05T21:06:05.480161Z",
     "shell.execute_reply": "2022-02-05T21:06:05.479621Z",
     "shell.execute_reply.started": "2022-02-05T13:58:25.354393Z"
    },
    "papermill": {
     "duration": 6.868331,
     "end_time": "2022-02-05T21:06:05.480282",
     "exception": false,
     "start_time": "2022-02-05T21:05:58.611951",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# sdjbfjksbdfkjb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 7.403634,
     "end_time": "2022-02-05T21:06:19.411734",
     "exception": false,
     "start_time": "2022-02-05T21:06:12.0081",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "<!-- ucvgkcvigvuyvoluyvbhlo -->"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 6.669054,
     "end_time": "2022-02-05T21:06:32.861016",
     "exception": false,
     "start_time": "2022-02-05T21:06:26.191962",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# View your Albumentations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T21:06:46.085745Z",
     "iopub.status.busy": "2022-02-05T21:06:46.08486Z",
     "iopub.status.idle": "2022-02-05T21:06:46.086551Z",
     "shell.execute_reply": "2022-02-05T21:06:46.087086Z",
     "shell.execute_reply.started": "2022-02-05T13:58:25.361439Z"
    },
    "papermill": {
     "duration": 6.518623,
     "end_time": "2022-02-05T21:06:46.087233",
     "exception": false,
     "start_time": "2022-02-05T21:06:39.56861",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "TRAIN_RUN_DIR = \"/kaggle/working/yolov5/runs/train/cots_with_albs\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T21:06:59.992709Z",
     "iopub.status.busy": "2022-02-05T21:06:59.991951Z",
     "iopub.status.idle": "2022-02-05T21:07:00.735777Z",
     "shell.execute_reply": "2022-02-05T21:07:00.735328Z",
     "shell.execute_reply.started": "2022-02-05T13:58:25.376236Z"
    },
    "papermill": {
     "duration": 7.478181,
     "end_time": "2022-02-05T21:07:00.735943",
     "exception": false,
     "start_time": "2022-02-05T21:06:53.257762",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "!ls {TRAIN_RUN_DIR}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T21:07:14.0832Z",
     "iopub.status.busy": "2022-02-05T21:07:14.082298Z",
     "iopub.status.idle": "2022-02-05T21:07:15.020023Z",
     "shell.execute_reply": "2022-02-05T21:07:15.020426Z",
     "shell.execute_reply.started": "2022-02-05T13:58:26.204188Z"
    },
    "papermill": {
     "duration": 7.663319,
     "end_time": "2022-02-05T21:07:15.020574",
     "exception": false,
     "start_time": "2022-02-05T21:07:07.357255",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(30,15))\n",
    "plt.axis('off')\n",
    "plt.imshow(plt.imread(f'{TRAIN_RUN_DIR}/train_batch0.jpg'));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T21:07:28.769839Z",
     "iopub.status.busy": "2022-02-05T21:07:28.769306Z",
     "iopub.status.idle": "2022-02-05T21:07:29.626283Z",
     "shell.execute_reply": "2022-02-05T21:07:29.626685Z",
     "shell.execute_reply.started": "2022-02-05T13:58:26.903001Z"
    },
    "papermill": {
     "duration": 7.794203,
     "end_time": "2022-02-05T21:07:29.626844",
     "exception": false,
     "start_time": "2022-02-05T21:07:21.832641",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(30,15))\n",
    "plt.axis('off')\n",
    "plt.imshow(plt.imread(f'{TRAIN_RUN_DIR}/train_batch1.jpg'));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T21:07:43.283511Z",
     "iopub.status.busy": "2022-02-05T21:07:43.278305Z",
     "iopub.status.idle": "2022-02-05T21:07:44.122224Z",
     "shell.execute_reply": "2022-02-05T21:07:44.122628Z",
     "shell.execute_reply.started": "2022-02-05T13:58:27.416153Z"
    },
    "papermill": {
     "duration": 7.646169,
     "end_time": "2022-02-05T21:07:44.122771",
     "exception": false,
     "start_time": "2022-02-05T21:07:36.476602",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(30,15))\n",
    "plt.axis('off')\n",
    "plt.imshow(plt.imread(f'{TRAIN_RUN_DIR}/train_batch2.jpg'));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 7.33905,
     "end_time": "2022-02-05T21:07:58.053851",
     "exception": false,
     "start_time": "2022-02-05T21:07:50.714801",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Cleanup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T21:08:11.541158Z",
     "iopub.status.busy": "2022-02-05T21:08:11.540257Z",
     "iopub.status.idle": "2022-02-05T21:08:17.742678Z",
     "shell.execute_reply": "2022-02-05T21:08:17.742185Z",
     "shell.execute_reply.started": "2022-02-05T13:58:27.9156Z"
    },
    "papermill": {
     "duration": 12.802772,
     "end_time": "2022-02-05T21:08:17.742799",
     "exception": false,
     "start_time": "2022-02-05T21:08:04.940027",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "%cd /kaggle/working\n",
    "\n",
    "!cp -r /kaggle/working/yolov5/runs/train/cots_with_albs /kaggle/working\n",
    "\n",
    "!cp /kaggle/working/yolov5/data/cots.yaml /kaggle/working/cots_with_albs/\n",
    "!cp /kaggle/working/yolov5/data/hyps/hyps_custom.yaml /kaggle/working/cots_with_albs/\n",
    "!cp /kaggle/working/yolov5/utils/augmentations.py /kaggle/working/cots_with_albs/\n",
    "\n",
    "!rm -r /kaggle/working/yolov5\n",
    "!rm -r /kaggle/working/images\n",
    "!rm -r /kaggle/working/labels"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}