make_datasets.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "03d45097",
   "metadata": {},
   "source": [
    "# 1. 制作图片和标签"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "023cdc67",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 创建图片、标签和数据划分的目录\n",
    "!mkdir -p ../input/mmseg_data/images\n",
    "!mkdir -p ../input/mmseg_data/labels\n",
    "!mkdir -p ../input/mmseg_data/splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0e4c6c4",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-07-31T17:00:53.445424Z",
     "iopub.status.busy": "2023-07-31T17:00:53.444956Z",
     "iopub.status.idle": "2023-07-31T17:00:53.456619Z",
     "shell.execute_reply": "2023-07-31T17:00:53.455819Z"
    },
    "papermill": {
     "duration": 0.020119,
     "end_time": "2023-07-31T17:00:53.458617",
     "exception": false,
     "start_time": "2023-07-31T17:00:53.438498",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "import cv2\n",
    "import shutil\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import shutil\n",
    "import os\n",
    "from glob import glob\n",
    "from tqdm import tqdm\n",
    "\n",
    "input_dir = \"../input/google-research-identify-contrails-reduce-global-warming/train\"\n",
    "output_dir = \"../input/mmseg_data\"\n",
    "os.makedirs(output_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "714af670",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-07-31T17:00:53.468642Z",
     "iopub.status.busy": "2023-07-31T17:00:53.467980Z",
     "iopub.status.idle": "2023-07-31T17:00:53.476757Z",
     "shell.execute_reply": "2023-07-31T17:00:53.476050Z"
    },
    "papermill": {
     "duration": 0.015929,
     "end_time": "2023-07-31T17:00:53.478913",
     "exception": false,
     "start_time": "2023-07-31T17:00:53.462984",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 定义一个函数来读取数据记录， 符合 false color 的格式\n",
    "def read_record(directory):\n",
    "    record_data = {}\n",
    "    for x in [\n",
    "        \"band_11\", \n",
    "        \"band_14\", \n",
    "        \"band_15\"\n",
    "    ]:\n",
    "        record_data[x] = np.load(os.path.join(directory, x + \".npy\"))\n",
    "\n",
    "    return record_data\n",
    "\n",
    "# 数据归一化函数，将数据映射到 [0, 1] 范围\n",
    "def normalize_range(data, bounds):\n",
    "    \"\"\"Maps data to the range [0, 1].\"\"\"\n",
    "    return (data - bounds[0]) / (bounds[1] - bounds[0])\n",
    "\n",
    "\n",
    "# 定义函数获得假色图像\n",
    "def get_false_color(record_data):\n",
    "    _T11_BOUNDS = (243, 303)\n",
    "    _CLOUD_TOP_TDIFF_BOUNDS = (-4, 5)\n",
    "    _TDIFF_BOUNDS = (-4, 2)\n",
    "\n",
    "    N_TIMES_BEFORE = 4\n",
    "\n",
    "    r = normalize_range(record_data[\"band_15\"] - record_data[\"band_14\"], _TDIFF_BOUNDS)\n",
    "    g = normalize_range(record_data[\"band_14\"] - record_data[\"band_11\"], _CLOUD_TOP_TDIFF_BOUNDS)\n",
    "    b = normalize_range(record_data[\"band_14\"], _T11_BOUNDS)\n",
    "    false_color = np.clip(np.stack([r, g, b], axis=2), 0, 1)\n",
    "    img = false_color[..., N_TIMES_BEFORE]\n",
    "\n",
    "    return img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ef39b28",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-07-31T17:00:53.488647Z",
     "iopub.status.busy": "2023-07-31T17:00:53.488259Z",
     "iopub.status.idle": "2023-07-31T17:00:53.867194Z",
     "shell.execute_reply": "2023-07-31T17:00:53.865911Z"
    },
    "papermill": {
     "duration": 0.386776,
     "end_time": "2023-07-31T17:00:53.869896",
     "exception": false,
     "start_time": "2023-07-31T17:00:53.483120",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 获取训练图像的路径\n",
    "train_images = glob(f\"{input_dir}/*\")\n",
    "print(f\"train_images: {len(train_images)}\")\n",
    "train_images[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd74d037",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-07-31T17:00:53.915389Z",
     "iopub.status.busy": "2023-07-31T17:00:53.914763Z",
     "iopub.status.idle": "2023-07-31T17:00:53.921365Z",
     "shell.execute_reply": "2023-07-31T17:00:53.920564Z"
    },
    "papermill": {
     "duration": 0.014684,
     "end_time": "2023-07-31T17:00:53.923537",
     "exception": false,
     "start_time": "2023-07-31T17:00:53.908853",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 遍历每张训练图像，制作图像和标签\n",
    "for path in tqdm(train_images):\n",
    "    id = path.split(\"/\")[-1]\n",
    "    os.makedirs(f\"{output_dir}/{id}\", exist_ok=True)\n",
    "    record_data = read_record(path)\n",
    "    img = get_false_color(record_data)\n",
    "    np.save(f\"{output_dir}/images/{id}.npy\", img)\n",
    "\n",
    "    msk = np.load(f\"{path}/human_pixel_masks.npy\")\n",
    "    cv2.imwrite(f\"{output_dir}/labels/{id}.png\", msk)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2668e237",
   "metadata": {},
   "source": [
    "# 2. 制作splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "88ddea7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 导入相关的库\n",
    "from glob import glob\n",
    "from sklearn.model_selection import GroupKFold, KFold\n",
    "import cv2\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "\n",
    "# 获取所有图像的文件路径\n",
    "all_image_files = glob(\"../input/mmseg_data/labels/*.png\")\n",
    "image_ids = [filepath.split(\"/\")[-1].split(\".\")[0] for filepath in all_image_files]\n",
    "print(f\"Total number of images: {len(image_ids)}\")\n",
    "print(f\"First 5 image ids: {image_ids[:5]}\")\n",
    "\n",
    "# 使用5折交叉验证来划分数据集\n",
    "split = list(KFold(5, random_state=42, shuffle=True).split(image_ids))\n",
    "\n",
    "# 遍历每一个折 (fold)，制作训练和验证数据划分\n",
    "for fold, (train_idx, valid_idx) in enumerate(split):\n",
    "    # # 保存训练集txt\n",
    "    # with open(f\"../input/mmseg_data/splits/fold_{fold}.txt\", \"w\") as f:\n",
    "    #     for idx in train_idx:\n",
    "    #         f.write(image_ids[idx] + \"\\n\")\n",
    "    \n",
    "    # 保存有前景的训练集txt\n",
    "    fore_cnt = 0\n",
    "    with open(f\"../input/mmseg_data/splits/fold_{fold}_fore.txt\", \"w\") as f:\n",
    "        for idx in train_idx:\n",
    "            img = cv2.imread(f\"../input/mmseg_data/labels/{image_ids[idx]}.png\")\n",
    "            if np.sum(img) > 0:\n",
    "                fore_cnt += 1\n",
    "                f.write(image_ids[idx] + \"\\n\")\n",
    "\n",
    "    # 保存验证集txt\n",
    "    with open(f\"../input/mmseg_data/splits/holdout_{fold}.txt\", \"w\") as f:\n",
    "        for idx in valid_idx:\n",
    "            f.write(image_ids[idx] + \"\\n\")\n",
    "\n",
    "\n",
    "    print(f\"fold{fold} 训练集数量：{len(train_idx)}\")\n",
    "    print(f\"fold{fold} 有前景的训练集数量：{fore_cnt}\")\n",
    "    print(f\"fold{fold} 验证集数量：{len(valid_idx)}\\n\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 326.675413,
   "end_time": "2023-07-31T17:06:09.336550",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-07-31T17:00:42.661137",
   "version": "2.4.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}