training/yolov5s6-vid-2-adam-3584-img-cp-0-5.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d6b7b72f",
   "metadata": {
    "papermill": {
     "duration": 0.022812,
     "end_time": "2022-02-12T17:17:56.808249",
     "exception": false,
     "start_time": "2022-02-12T17:17:56.785437",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Yolov5 high resolution training\n",
    "\n",
    "### Major modification\n",
    "* img=3600\n",
    "* mixup=0.5\n",
    "* fliplr: 0.5\n",
    "\n",
    "### Hardware to reproduce\n",
    "* RTX3090"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f5130e4b",
   "metadata": {
    "papermill": {
     "duration": 0.020671,
     "end_time": "2022-02-12T17:17:56.851204",
     "exception": false,
     "start_time": "2022-02-12T17:17:56.830533",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Training Log:\n",
    "> ```\n",
    "version=1\n",
    "img_size:3584,bs2,e11,[yolov5s6] \n",
    "Fold: video2[validation]\n",
    "Labels: only GT\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d33d375f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-12T17:17:56.903613Z",
     "iopub.status.busy": "2022-02-12T17:17:56.902706Z",
     "iopub.status.idle": "2022-02-12T17:17:57.612074Z",
     "shell.execute_reply": "2022-02-12T17:17:57.611527Z",
     "shell.execute_reply.started": "2022-02-11T22:53:12.917075Z"
    },
    "papermill": {
     "duration": 0.739672,
     "end_time": "2022-02-12T17:17:57.612223",
     "exception": false,
     "start_time": "2022-02-12T17:17:56.872551",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sat Feb 12 17:17:57 2022       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 450.119.04   Driver Version: 450.119.04   CUDA Version: 11.0     |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|                               |                      |               MIG M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n",
      "| N/A   33C    P0    27W / 250W |      0MiB / 16280MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                                  |\n",
      "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
      "|        ID   ID                                                   Usage      |\n",
      "|=============================================================================|\n",
      "|  No running processes found                                                 |\n",
      "+-----------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c1e067ab",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2022-02-12T17:17:57.659432Z",
     "iopub.status.busy": "2022-02-12T17:17:57.658716Z",
     "iopub.status.idle": "2022-02-12T17:17:57.660566Z",
     "shell.execute_reply": "2022-02-12T17:17:57.661043Z",
     "shell.execute_reply.started": "2022-02-11T22:53:13.630554Z"
    },
    "papermill": {
     "duration": 0.027462,
     "end_time": "2022-02-12T17:17:57.661165",
     "exception": false,
     "start_time": "2022-02-12T17:17:57.633703",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load\n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "from shutil import copyfile\n",
    "\n",
    "# Input data files are available in the read-only \"../input/\" directory\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2282cf97",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-12T17:17:57.706223Z",
     "iopub.status.busy": "2022-02-12T17:17:57.705432Z",
     "iopub.status.idle": "2022-02-12T17:17:57.708902Z",
     "shell.execute_reply": "2022-02-12T17:17:57.708498Z",
     "shell.execute_reply.started": "2022-02-11T22:53:13.636763Z"
    },
    "papermill": {
     "duration": 0.026683,
     "end_time": "2022-02-12T17:17:57.709030",
     "exception": false,
     "start_time": "2022-02-12T17:17:57.682347",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# train = pd.read_csv('../input/tensorflow-great-barrier-reef/train.csv')\n",
    "# train['pos'] = train.annotations != '[]'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c8eb7f06",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-12T17:17:57.758256Z",
     "iopub.status.busy": "2022-02-12T17:17:57.757477Z",
     "iopub.status.idle": "2022-02-12T17:17:58.262627Z",
     "shell.execute_reply": "2022-02-12T17:17:58.263037Z",
     "shell.execute_reply.started": "2022-02-11T22:53:13.64869Z"
    },
    "papermill": {
     "duration": 0.533389,
     "end_time": "2022-02-12T17:17:58.263190",
     "exception": false,
     "start_time": "2022-02-12T17:17:57.729801",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>video_id</th>\n",
       "      <th>sequence</th>\n",
       "      <th>video_frame</th>\n",
       "      <th>sequence_frame</th>\n",
       "      <th>image_id</th>\n",
       "      <th>annotations</th>\n",
       "      <th>annotations1</th>\n",
       "      <th>image_path</th>\n",
       "      <th>no_of_bbox</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>40258</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0-0</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>video_0/0.jpg</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>40258</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0-1</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>video_0/1.jpg</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>40258</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0-2</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>video_0/2.jpg</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>40258</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0-3</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>video_0/3.jpg</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>40258</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>0-4</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>video_0/4.jpg</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   video_id  sequence  video_frame  sequence_frame image_id annotations  \\\n",
       "0         0     40258            0               0      0-0          []   \n",
       "1         0     40258            1               1      0-1          []   \n",
       "2         0     40258            2               2      0-2          []   \n",
       "3         0     40258            3               3      0-3          []   \n",
       "4         0     40258            4               4      0-4          []   \n",
       "\n",
       "  annotations1     image_path  no_of_bbox  \n",
       "0           []  video_0/0.jpg           0  \n",
       "1           []  video_0/1.jpg           0  \n",
       "2           []  video_0/2.jpg           0  \n",
       "3           []  video_0/3.jpg           0  \n",
       "4           []  video_0/4.jpg           0  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"../input/tensorflow-great-barrier-reef/train.csv\")\n",
    "\n",
    "# Turn annotations from strings into lists of dictionaries\n",
    "df['annotations1'] = df['annotations'].apply(eval)\n",
    "\n",
    "# Create the image path for the row\n",
    "df['image_path'] = \"video_\" + df['video_id'].astype(str) + \"/\" + df['video_frame'].astype(str) + \".jpg\"\n",
    "\n",
    "length = lambda x: len(x) \n",
    "\n",
    "df[\"no_of_bbox\"] = df[\"annotations1\"].apply(length)\n",
    "\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c1a2028b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-12T17:17:58.311674Z",
     "iopub.status.busy": "2022-02-12T17:17:58.310857Z",
     "iopub.status.idle": "2022-02-12T17:17:58.322699Z",
     "shell.execute_reply": "2022-02-12T17:17:58.323118Z",
     "shell.execute_reply.started": "2022-02-11T22:53:14.099492Z"
    },
    "papermill": {
     "duration": 0.038439,
     "end_time": "2022-02-12T17:17:58.323245",
     "exception": false,
     "start_time": "2022-02-12T17:17:58.284806",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:1: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(677, 9)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df = df[df[\"video_id\"]==2][df[\"no_of_bbox\"]>0]\n",
    "val_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "da821c40",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-12T17:17:58.371245Z",
     "iopub.status.busy": "2022-02-12T17:17:58.370426Z",
     "iopub.status.idle": "2022-02-12T17:17:58.378381Z",
     "shell.execute_reply": "2022-02-12T17:17:58.378765Z",
     "shell.execute_reply.started": "2022-02-11T22:53:14.117588Z"
    },
    "papermill": {
     "duration": 0.033802,
     "end_time": "2022-02-12T17:17:58.378875",
     "exception": false,
     "start_time": "2022-02-12T17:17:58.345073",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:1: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(4242, 9)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = df[df[\"video_id\"]!=2][df[\"no_of_bbox\"]>0]\n",
    "train_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e3fce9a4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-12T17:17:58.441343Z",
     "iopub.status.busy": "2022-02-12T17:17:58.434467Z",
     "iopub.status.idle": "2022-02-12T17:18:01.153475Z",
     "shell.execute_reply": "2022-02-12T17:18:01.152801Z",
     "shell.execute_reply.started": "2022-02-11T22:53:14.130997Z"
    },
    "papermill": {
     "duration": 2.752306,
     "end_time": "2022-02-12T17:18:01.153614",
     "exception": false,
     "start_time": "2022-02-12T17:17:58.401308",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "!mkdir -p ./yolo_data/fold2/images/val\n",
    "!mkdir -p ./yolo_data/fold2/images/train\n",
    "\n",
    "!mkdir -p ./yolo_data/fold2/labels/val\n",
    "!mkdir -p ./yolo_data/fold2/labels/train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "cbba1a24",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-12T17:18:01.207723Z",
     "iopub.status.busy": "2022-02-12T17:18:01.207204Z",
     "iopub.status.idle": "2022-02-12T17:18:10.115684Z",
     "shell.execute_reply": "2022-02-12T17:18:10.115110Z",
     "shell.execute_reply.started": "2022-02-11T22:53:16.720468Z"
    },
    "papermill": {
     "duration": 8.939384,
     "end_time": "2022-02-12T17:18:10.115806",
     "exception": false,
     "start_time": "2022-02-12T17:18:01.176422",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "fold = 2\n",
    "\n",
    "annos = []\n",
    "for i, x in val_df.iterrows():\n",
    "#     if x.video_id == fold:\n",
    "#         if x.pos:\n",
    "    mode = 'val'\n",
    "#     else:\n",
    "#         # train\n",
    "#         mode = 'train'\n",
    "#         if not x.pos: continue\n",
    "        # val\n",
    "    copyfile(f'../input/tensorflow-great-barrier-reef/train_images/video_{x.video_id}/{x.video_frame}.jpg',\n",
    "                f'./yolo_data/fold{fold}/images/{mode}/{x.image_id}.jpg')\n",
    "#     if not x.pos:\n",
    "#         continue\n",
    "    r = ''\n",
    "    anno = eval(x.annotations)\n",
    "    for an in anno:\n",
    "#            annos.append(an)\n",
    "        r += '0 {} {} {} {}\\n'.format((an['x'] + an['width'] / 2) / 1280,\n",
    "                                        (an['y'] + an['height'] / 2) / 720,\n",
    "                                        an['width'] / 1280, an['height'] / 720)\n",
    "    with open(f'./yolo_data/fold{fold}/labels/{mode}/{x.image_id}.txt', 'w') as fp:\n",
    "        fp.write(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "41ea2f75",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-12T17:18:10.169856Z",
     "iopub.status.busy": "2022-02-12T17:18:10.169287Z",
     "iopub.status.idle": "2022-02-12T17:18:58.078268Z",
     "shell.execute_reply": "2022-02-12T17:18:58.077352Z",
     "shell.execute_reply.started": "2022-02-11T22:53:24.858746Z"
    },
    "papermill": {
     "duration": 47.939823,
     "end_time": "2022-02-12T17:18:58.078421",
     "exception": false,
     "start_time": "2022-02-12T17:18:10.138598",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "fold = 2\n",
    "\n",
    "annos = []\n",
    "for i, x in train_df.iterrows():\n",
    "#     if x.video_id == fold:\n",
    "#         if x.pos:\n",
    "    mode = 'train'\n",
    "#     else:\n",
    "#         # train\n",
    "#         mode = 'train'\n",
    "#         if not x.pos: continue\n",
    "        # val\n",
    "    copyfile(f'../input/tensorflow-great-barrier-reef/train_images/video_{x.video_id}/{x.video_frame}.jpg',\n",
    "                f'./yolo_data/fold{fold}/images/{mode}/{x.image_id}.jpg')\n",
    "#     if not x.pos:\n",
    "#         continue\n",
    "    r = ''\n",
    "    anno = eval(x.annotations)\n",
    "    for an in anno:\n",
    "#            annos.append(an)\n",
    "        r += '0 {} {} {} {}\\n'.format((an['x'] + an['width'] / 2) / 1280,\n",
    "                                        (an['y'] + an['height'] / 2) / 720,\n",
    "                                        an['width'] / 1280, an['height'] / 720)\n",
    "    with open(f'./yolo_data/fold{fold}/labels/{mode}/{x.image_id}.txt', 'w') as fp:\n",
    "        fp.write(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eba9002c",
   "metadata": {
    "papermill": {
     "duration": 0.023033,
     "end_time": "2022-02-12T17:18:58.125285",
     "exception": false,
     "start_time": "2022-02-12T17:18:58.102252",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "dff0d081",
   "metadata": {
    "_kg_hide-input": true,
    "execution": {
     "iopub.execute_input": "2022-02-12T17:18:58.187094Z",
     "iopub.status.busy": "2022-02-12T17:18:58.185495Z",
     "iopub.status.idle": "2022-02-12T17:18:58.188442Z",
     "shell.execute_reply": "2022-02-12T17:18:58.189185Z",
     "shell.execute_reply.started": "2022-02-11T22:54:11.394299Z"
    },
    "papermill": {
     "duration": 0.039422,
     "end_time": "2022-02-12T17:18:58.189438",
     "exception": false,
     "start_time": "2022-02-12T17:18:58.150016",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# fold = 1\n",
    "\n",
    "# annos = []\n",
    "# for i, x in train.iterrows():\n",
    "#     if x.video_id == fold:\n",
    "#         if x.pos:\n",
    "#             mode = 'val'\n",
    "#     else:\n",
    "#         # train\n",
    "#         mode = 'train'\n",
    "#         if not x.pos: continue\n",
    "#         # val\n",
    "#     copyfile(f'../input/tensorflow-great-barrier-reef/train_images/video_{x.video_id}/{x.video_frame}.jpg',\n",
    "#                 f'./yolo_data/fold{fold}/images/{mode}/{x.image_id}.jpg')\n",
    "#     if not x.pos:\n",
    "#         continue\n",
    "#     r = ''\n",
    "#     anno = eval(x.annotations)\n",
    "#     for an in anno:\n",
    "# #            annos.append(an)\n",
    "#         r += '0 {} {} {} {}\\n'.format((an['x'] + an['width'] / 2) / 1280,\n",
    "#                                         (an['y'] + an['height'] / 2) / 720,\n",
    "#                                         an['width'] / 1280, an['height'] / 720)\n",
    "#     with open(f'./yolo_data/fold{fold}/labels/{mode}/{x.image_id}.txt', 'w') as fp:\n",
    "#         fp.write(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8a2ec62d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-12T17:18:58.255314Z",
     "iopub.status.busy": "2022-02-12T17:18:58.254323Z",
     "iopub.status.idle": "2022-02-12T17:18:58.266621Z",
     "shell.execute_reply": "2022-02-12T17:18:58.267394Z",
     "shell.execute_reply.started": "2022-02-11T22:54:11.400083Z"
    },
    "papermill": {
     "duration": 0.049031,
     "end_time": "2022-02-12T17:18:58.267651",
     "exception": false,
     "start_time": "2022-02-12T17:18:58.218620",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4242"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "len(os.listdir(\"./yolo_data/fold2/labels/train\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "c58b832f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-12T17:18:58.337288Z",
     "iopub.status.busy": "2022-02-12T17:18:58.335405Z",
     "iopub.status.idle": "2022-02-12T17:18:58.338396Z",
     "shell.execute_reply": "2022-02-12T17:18:58.339045Z",
     "shell.execute_reply.started": "2022-02-11T22:54:11.728589Z"
    },
    "papermill": {
     "duration": 0.042158,
     "end_time": "2022-02-12T17:18:58.339274",
     "exception": false,
     "start_time": "2022-02-12T17:18:58.297116",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "hyps = '''\n",
    "# YOLOv5 by Ultralytics, GPL-3.0 license\n",
    "# Hyperparameters for COCO training from scratch\n",
    "# python train.py --batch 40 --cfg yolov5m.yaml --weights '' --data coco.yaml --img 640 --epochs 300\n",
    "# See tutorials for hyperparameter evolution https://github.com/ultralytics/yolov5#tutorials\n",
    "\n",
    "lr0: 0.001  # initial learning rate (SGD=1E-2, Adam=1E-3)\n",
    "lrf: 0.1  # final OneCycleLR learning rate (lr0 * lrf)\n",
    "momentum: 0.937  # SGD momentum/Adam beta1\n",
    "weight_decay: 0.0005  # optimizer weight decay 5e-4\n",
    "warmup_epochs: 3.0  # warmup epochs (fractions ok)\n",
    "warmup_momentum: 0.8  # warmup initial momentum\n",
    "warmup_bias_lr: 0.1  # warmup initial bias lr\n",
    "box: 0.05  # box loss gain\n",
    "cls: 0.5  # cls loss gain\n",
    "cls_pw: 1.0  # cls BCELoss positive_weight\n",
    "obj: 1.0  # obj loss gain (scale with pixels)\n",
    "obj_pw: 1.0  # obj BCELoss positive_weight\n",
    "iou_t: 0.20  # IoU training threshold\n",
    "anchor_t: 4.0  # anchor-multiple threshold\n",
    "# anchors: 3  # anchors per output layer (0 to ignore)\n",
    "fl_gamma: 0.0  # focal loss gamma (efficientDet default gamma=1.5)\n",
    "hsv_h: 0.015  # image HSV-Hue augmentation (fraction)\n",
    "hsv_s: 0.7  # image HSV-Saturation augmentation (fraction)\n",
    "hsv_v: 0.4  # image HSV-Value augmentation (fraction)\n",
    "degrees: 0.0  # image rotation (+/- deg)\n",
    "translate: 0.1  # image translation (+/- fraction)\n",
    "scale: 0.5  # image scale (+/- gain)\n",
    "shear: 0.0  # image shear (+/- deg)\n",
    "perspective: 0.0  # image perspective (+/- fraction), range 0-0.001\n",
    "flipud: 0.5  # image flip up-down (probability)\n",
    "fliplr: 0.5  # image flip left-right (probability)\n",
    "mosaic: 1.0  # image mosaic (probability)\n",
    "mixup: 0.5  # image mixup (probability)\n",
    "copy_paste: 0.5  # segment copy-paste (probability)\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e48013a1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-12T17:18:58.408678Z",
     "iopub.status.busy": "2022-02-12T17:18:58.406136Z",
     "iopub.status.idle": "2022-02-12T17:18:58.409656Z",
     "shell.execute_reply": "2022-02-12T17:18:58.410249Z",
     "shell.execute_reply.started": "2022-02-11T22:54:11.73652Z"
    },
    "papermill": {
     "duration": 0.041629,
     "end_time": "2022-02-12T17:18:58.410463",
     "exception": false,
     "start_time": "2022-02-12T17:18:58.368834",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "data = '''\n",
    "# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..]\n",
    "\n",
    "path: ../yolo_data/fold2/  # dataset root dir\n",
    "train: images/train  # train images (relative to 'path') 128 images\n",
    "val: images/val  # val images (relative to 'path') 128 images\n",
    "test:  # test images (optional)\n",
    "\n",
    "# Classes\n",
    "nc: 1  # number of classes\n",
    "names: ['reef']  # class names\n",
    "\n",
    "\n",
    "# Download script/URL (optional)\n",
    "# download: https://ultralytics.com/assets/coco128.zip\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "30bf87bb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-12T17:18:58.479793Z",
     "iopub.status.busy": "2022-02-12T17:18:58.478355Z",
     "iopub.status.idle": "2022-02-12T17:19:01.393268Z",
     "shell.execute_reply": "2022-02-12T17:19:01.392736Z",
     "shell.execute_reply.started": "2022-02-11T22:54:11.748717Z"
    },
    "papermill": {
     "duration": 2.953146,
     "end_time": "2022-02-12T17:19:01.393407",
     "exception": false,
     "start_time": "2022-02-12T17:18:58.440261",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'yolov5-w-f2-mod'...\n",
      "remote: Enumerating objects: 132, done.\u001b[K\n",
      "remote: Counting objects: 100% (132/132), done.\u001b[K\n",
      "remote: Compressing objects: 100% (105/105), done.\u001b[K\n",
      "remote: Total 132 (delta 36), reused 120 (delta 26), pack-reused 0\u001b[K\n",
      "Receiving objects: 100% (132/132), 1.39 MiB | 3.28 MiB/s, done.\n",
      "Resolving deltas: 100% (36/36), done.\n"
     ]
    }
   ],
   "source": [
    "# !git clone https://github.com/ultralytics/yolov5.git\n",
    "!git clone https://github.com/soumya997/yolov5-w-f2-mod.git\n",
    "!mv ./yolov5-w-f2-mod ./yolov5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "9e408ce2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-12T17:19:01.449730Z",
     "iopub.status.busy": "2022-02-12T17:19:01.449217Z",
     "iopub.status.idle": "2022-02-12T17:19:01.452913Z",
     "shell.execute_reply": "2022-02-12T17:19:01.452468Z",
     "shell.execute_reply.started": "2022-02-11T22:54:14.990137Z"
    },
    "papermill": {
     "duration": 0.033816,
     "end_time": "2022-02-12T17:19:01.453042",
     "exception": false,
     "start_time": "2022-02-12T17:19:01.419226",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "with open('./yolov5/data/reef_f1_naive.yaml', 'w') as fp:\n",
    "    fp.write(data)\n",
    "with open('./yolov5/data/hyps/hyp.heavy.2.yaml', 'w') as fp:\n",
    "    fp.write(hyps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "47cc924f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-12T17:19:01.509006Z",
     "iopub.status.busy": "2022-02-12T17:19:01.508333Z",
     "iopub.status.idle": "2022-02-12T17:19:01.511261Z",
     "shell.execute_reply": "2022-02-12T17:19:01.511823Z",
     "shell.execute_reply.started": "2022-02-11T22:54:14.997751Z"
    },
    "papermill": {
     "duration": 0.033921,
     "end_time": "2022-02-12T17:19:01.511989",
     "exception": false,
     "start_time": "2022-02-12T17:19:01.478068",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/kaggle/working/yolov5\n"
     ]
    }
   ],
   "source": [
    "%cd yolov5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c57492c8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-12T17:19:01.568430Z",
     "iopub.status.busy": "2022-02-12T17:19:01.567724Z",
     "iopub.status.idle": "2022-02-12T17:19:02.252519Z",
     "shell.execute_reply": "2022-02-12T17:19:02.251866Z",
     "shell.execute_reply.started": "2022-02-11T22:54:15.00922Z"
    },
    "papermill": {
     "duration": 0.714033,
     "end_time": "2022-02-12T17:19:02.252654",
     "exception": false,
     "start_time": "2022-02-12T17:19:01.538621",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Argoverse.yaml\t      VOC.yaml\t     hyps\t\t xView.yaml\n",
      "GlobalWheat2020.yaml  VisDrone.yaml  images\n",
      "Objects365.yaml       coco.yaml      reef_f1_naive.yaml\n",
      "SKU-110K.yaml\t      coco128.yaml   scripts\n"
     ]
    }
   ],
   "source": [
    "!ls data/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "5a30c121",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-12T17:19:02.312717Z",
     "iopub.status.busy": "2022-02-12T17:19:02.312003Z",
     "iopub.status.idle": "2022-02-11T23:08:16.23287Z",
     "shell.execute_reply": "2022-02-11T23:08:16.232013Z",
     "shell.execute_reply.started": "2022-02-11T22:54:15.673194Z"
    },
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": false,
     "start_time": "2022-02-12T17:19:02.279245",
     "status": "running"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "W&B disabled.\n",
      "Downloading https://github.com/ultralytics/yolov5/releases/download/v6.0/yolov5s6.pt to yolov5s6.pt...\n",
      "100%|██████████████████████████████████████| 24.5M/24.5M [00:00<00:00, 33.3MB/s]\n",
      "\n",
      "\u001b[34m\u001b[1mtrain: \u001b[0mScanning '../yolo_data/fold2/labels/train' images and labels...4242 found\u001b[0m\n",
      "\u001b[34m\u001b[1mval: \u001b[0mScanning '../yolo_data/fold2/labels/val' images and labels...677 found, 0 m\u001b[0m\n",
      "      0/10     12.6G   0.05817   0.07797         0         3      3584: 100%|███\n",
      "               Class     Images     Labels          P          R         F2     \n",
      "      1/10     12.6G   0.04662    0.0663         0         9      3584: 100%|███\n",
      "               Class     Images     Labels          P          R         F2     \n",
      "      2/10     12.6G   0.04394   0.06278         0         5      3584: 100%|███\n",
      "               Class     Images     Labels          P          R         F2     \n",
      "      3/10     12.6G   0.04116   0.06103         0         8      3584: 100%|███\n",
      "               Class     Images     Labels          P          R         F2     \n",
      "      4/10     12.6G   0.03971   0.05793         0         7      3584: 100%|███\n",
      "               Class     Images     Labels          P          R         F2     \n",
      "      5/10     12.6G   0.03804   0.05645         0         0      3584: 100%|███\n",
      "               Class     Images     Labels          P          R         F2     \n",
      "      6/10     12.6G   0.03627    0.0547         0         2      3584: 100%|███\n",
      "               Class     Images     Labels          P          R         F2     \n",
      "      7/10     12.6G   0.03547   0.05392         0         2      3584: 100%|███\n",
      "               Class     Images     Labels          P          R         F2     \n",
      "      8/10     12.6G   0.03415    0.0529         0         5      3584: 100%|███\n",
      "               Class     Images     Labels          P          R         F2     \n",
      "      9/10     12.6G   0.03322   0.05239         0         2      3584: 100%|███\n",
      "               Class     Images     Labels          P          R         F2     \n",
      "     10/10     12.6G   0.03253   0.05022         0         9      3584:  98%|███"
     ]
    }
   ],
   "source": [
    "!python -m wandb disabled\n",
    "\n",
    "!python train.py \\\n",
    "    --img 3584 \\\n",
    "    --batch 2 \\\n",
    "    --epochs 11 \\\n",
    "    --optimizer Adam \\\n",
    "    --data data/reef_f1_naive.yaml \\\n",
    "    --weights yolov5s6.pt \\\n",
    "    --name base_vid_2val \\\n",
    "    --hyp data/hyps/hyp.heavy.2.yaml \\\n",
    "    --save-period 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c248639a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-11T23:08:16.237833Z",
     "iopub.status.busy": "2022-02-11T23:08:16.237097Z",
     "iopub.status.idle": "2022-02-11T23:08:31.789432Z",
     "shell.execute_reply": "2022-02-11T23:08:31.788534Z",
     "shell.execute_reply.started": "2022-02-11T23:08:16.237795Z"
    },
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# !python train.py "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac80d642",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-11T23:08:31.791013Z",
     "iopub.status.busy": "2022-02-11T23:08:31.790753Z",
     "iopub.status.idle": "2022-02-11T23:08:31.801148Z",
     "shell.execute_reply": "2022-02-11T23:08:31.80049Z",
     "shell.execute_reply.started": "2022-02-11T23:08:31.790979Z"
    },
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# !python -m wandb disabled\n",
    "\n",
    "# !python train.py \\\n",
    "#     --img 3000 \\\n",
    "#     --batch 2 \\\n",
    "#     --epochs 11 \\\n",
    "#     --data data/reef_f1_naive.yaml \\\n",
    "#     --weights yolov5s6.pt \\\n",
    "#     --name cots_with_albs \\\n",
    "#     --hyp data/hyps/hyp.heavy.2.yaml \\\n",
    "#     --save-period 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c61634d0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-11T23:08:31.804359Z",
     "iopub.status.busy": "2022-02-11T23:08:31.802946Z",
     "iopub.status.idle": "2022-02-11T23:08:32.506668Z",
     "shell.execute_reply": "2022-02-11T23:08:32.505767Z",
     "shell.execute_reply.started": "2022-02-11T23:08:31.80431Z"
    },
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "!ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4dec8a8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-11T23:08:32.510362Z",
     "iopub.status.busy": "2022-02-11T23:08:32.510118Z",
     "iopub.status.idle": "2022-02-11T23:08:36.461965Z",
     "shell.execute_reply": "2022-02-11T23:08:36.461115Z",
     "shell.execute_reply.started": "2022-02-11T23:08:32.510321Z"
    },
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# %cd /kaggle/working\n",
    "\n",
    "# !cp -r /kaggle/working/yolov5/runs/train/base_vid_2val /kaggle/working\n",
    "\n",
    "# !cp /kaggle/working/yolov5/data/reef_f1_naive.yaml /kaggle/working/base_vid_2val/\n",
    "# !cp /kaggle/working/yolov5/data/hyps/hyp.heavy.2.yaml /kaggle/working/base_vid_2val/\n",
    "# # !cp /kaggle/working/yolov5/utils/augmentations.py /kaggle/working/base_vid_2val/\n",
    "\n",
    "# !rm -r /kaggle/working/yolov5\n",
    "# !rm -r /kaggle/working/images\n",
    "# !rm -r /kaggle/working/labels"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "papermill": {
   "default_parameters": {},
   "duration": null,
   "end_time": null,
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2022-02-12T17:17:48.388434",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}