Numpy_Extraction_for_Month_Start_Month_End.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Numpy_Extraction_for_Month_Start_Month_End.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/DariusTheGeek/Radiant-Earth-Spot-the-Crop-XL-Challenge/blob/main/Numpy_Extraction_for_Month_Start_Month_End.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vemh8H3vZ9VO",
        "outputId": "fef28028-28ff-43a8-b9c5-30d3513dd6a7"
      },
      "source": [
        "!pip -qq install rasterio tifffile"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[K     |████████████████████████████████| 19.3 MB 89 kB/s \n",
            "\u001b[?25h"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3mKYb4JmMBEZ"
      },
      "source": [
        "import os\n",
        "import glob\n",
        "import shutil\n",
        "import gc\n",
        "from joblib import Parallel, delayed\n",
        "from tqdm import tqdm_notebook\n",
        "import h5py\n",
        "\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import datetime as dt\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "\n",
        "import rasterio\n",
        "import tifffile as tiff\n",
        "\n",
        "%matplotlib inline\n",
        "pd.set_option('display.max_colwidth', None)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MiS1dNGpRHBe"
      },
      "source": [
        "%%time\n",
        "# os.mkdir('radiant')\n",
        "shutil.unpack_archive('/content/drive/MyDrive/CompeData/Radiant/Radiant_Data.zip', '/content/radiant')\n",
        "gc.collect()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O0_JwAo0bMdV"
      },
      "source": [
        "train = pd.concat([pd.read_csv(f'/content/radiant/train{i}.csv', parse_dates=['datetime']) for i in range(1, 5)]).reset_index(drop = True)\n",
        "test = pd.concat([pd.read_csv(f'/content/radiant/test{i}.csv', parse_dates=['datetime']) for i in range(1, 5)]).reset_index(drop = True)\n",
        "train.file_path = train.file_path.apply(lambda x: '/'.join(['/content', 'radiant'] + x.split('/')[2:]))\n",
        "test.file_path = test.file_path.apply(lambda x: '/'.join(['/content', 'radiant'] + x.split('/')[2:]))\n",
        "train.datetime, test.datetime = pd.to_datetime(train.datetime.dt.date), pd.to_datetime(test.datetime.dt.date)\n",
        "train['month'], test['month'] = train.datetime.dt.month, test.datetime.dt.month\n",
        "train.head()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ph__jwFtDmR6"
      },
      "source": [
        "train.month.unique()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ofOR8cvvB7YW"
      },
      "source": [
        "train.tile_id.unique()[50:60]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "idcCxN2tAbQz"
      },
      "source": [
        "bands = ['B01','B02','B03','B04','B05','B06','B07','B08','B8A','B09','B11','B12','CLM']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CwkqspiVAgSi"
      },
      "source": [
        "date_cols = []\n",
        "for i in range(4, 12):\n",
        "  for x in range(1, 3):\n",
        "    date_cols.append(str(i) + '_' + str(x))\n",
        "date_cols"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JJhxiJcXAbNN"
      },
      "source": [
        "def process_tile_train(tile):\n",
        "  tile_df = train[(train.tile_id == tile)].reset_index(drop = True)\n",
        "\n",
        "  y = np.expand_dims(rasterio.open(tile_df[tile_df.asset == 'labels'].file_path.values[0]).read(1).flatten(), axis = 1)\n",
        "  fields = np.expand_dims(rasterio.open(tile_df[tile_df.asset == 'field_ids'].file_path.values[0]).read(1).flatten(), axis = 1)\n",
        "\n",
        "  tile_df = train[(train.tile_id == tile) & (train.satellite_platform == 's2')].reset_index(drop = True)\n",
        "\n",
        "  dates = []\n",
        "  for month in range(4, 12):\n",
        "    dates.append(tile_df[tile_df.month == month].datetime.sort_values().tolist()[0])\n",
        "    dates.append(tile_df[tile_df.month == month].datetime.sort_values().tolist()[-1])\n",
        "\n",
        "  X_tile = np.empty((256 * 256, 0))\n",
        "\n",
        "  colls = []\n",
        "  for date, datec in zip(dates, date_cols):\n",
        "    for band in bands:\n",
        "      tif_file = tile_df[(tile_df.asset == band) & (tile_df.datetime == date)].file_path.values[0]\n",
        "      X_tile = np.append(X_tile, (np.expand_dims(rasterio.open(tif_file).read(1).flatten(), axis = 1)), axis = 1)\n",
        "      colls.append(datec + '_' + band)\n",
        "  df = pd.DataFrame(X_tile, columns = colls)\n",
        "  df['y'], df['fields'] = y, fields\n",
        "  return df"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7hQOhSMPb-qI"
      },
      "source": [
        "tiles = train.tile_id.unique()\n",
        "chunks = [tiles[x:x+265] for x in range(0, len(tiles), 265)]\n",
        "[len(x) for x in chunks]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iULhCd36KrGG"
      },
      "source": [
        "for i in range(len(chunks)):\n",
        "  pd.DataFrame(np.vstack(Parallel(n_jobs=-1, verbose=1, backend=\"multiprocessing\")(map(delayed(process_tile_train), [x for x in chunks[i]])))).to_csv(f'/content/drive/MyDrive/CompeData/Radiant/Start_end/train{i}.csv', index = False)\n",
        "  gc.collect()\n",
        "  print(i)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BZD9TPDRLAuQ"
      },
      "source": [
        "def process_tile_test(tile):\n",
        "  tile_df = train[(train.tile_id == tile)].reset_index(drop = True)\n",
        "\n",
        "  # y = np.expand_dims(rasterio.open(tile_df[tile_df.asset == 'labels'].file_path.values[0]).read(1).flatten(), axis = 1)\n",
        "  fields = np.expand_dims(rasterio.open(tile_df[tile_df.asset == 'field_ids'].file_path.values[0]).read(1).flatten(), axis = 1)\n",
        "\n",
        "  tile_df = train[(train.tile_id == tile) & (train.satellite_platform == 's2')].reset_index(drop = True)\n",
        "\n",
        "  dates = []\n",
        "  for month in range(4, 12):\n",
        "    dates.append(tile_df[tile_df.month == month].datetime.sort_values().tolist()[0])\n",
        "    dates.append(tile_df[tile_df.month == month].datetime.sort_values().tolist()[-1])\n",
        "\n",
        "  X_tile = np.empty((256 * 256, 0))\n",
        "\n",
        "  colls = []\n",
        "  for date, datec in zip(dates, date_cols):\n",
        "    for band in bands:\n",
        "      tif_file = tile_df[(tile_df.asset == band) & (tile_df.datetime == date)].file_path.values[0]\n",
        "      X_tile = np.append(X_tile, (np.expand_dims(rasterio.open(tif_file).read(1).flatten(), axis = 1)), axis = 1)\n",
        "      colls.append(datec + '_' + band)\n",
        "  df = pd.DataFrame(X_tile, columns = colls)\n",
        "  df['fields'] = fields\n",
        "  return df"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wbk6YD0akhTo"
      },
      "source": [
        "tiles = train.tile_id.unique()\n",
        "chunks = [tiles[x:x+265] for x in range(0, len(tiles), 265)]\n",
        "[len(x) for x in chunks]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uQS4UBXKkhRF"
      },
      "source": [
        "for i in range(len(chunks)):\n",
        "  pd.DataFrame(np.vstack(Parallel(n_jobs=-1, verbose=1, backend=\"multiprocessing\")(map(delayed(process_tile_train), [x for x in chunks[i]])))).to_csv(f'/content/drive/MyDrive/CompeData/Radiant/Start_end/test{i}.csv', index = False)\n",
        "  gc.collect()\n",
        "  print(i)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ilCskgRkkhMg"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}