notebooks/descartes_spectrogram_run_withpop.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import shutil\n",
    "import sys\n",
    "\n",
    "import descarteslabs as dl\n",
    "from descarteslabs.catalog import Image, properties\n",
    "import geopandas as gpd\n",
    "import rasterio as rs\n",
    "from rasterio.merge import merge\n",
    "from tensorflow.keras.models import load_model\n",
    "from tensorflow import keras\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "\n",
    "module_path = os.path.abspath(os.path.join('..'))\n",
    "if module_path not in sys.path:\n",
    "    sys.path.append(module_path)\n",
    "    \n",
    "from scripts import deploy_nn_v2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# User inputs\n",
    "model_version = '0.0.11'\n",
    "model_name = 'spectrogram_v0.0.11_2021-07-13'\n",
    "model_file = '../models/' + model_name + '.h5'\n",
    "\n",
    "#patch_model_version = 'weak_labels_1.1'\n",
    "patch_model_version = 'weak_labels_3.0'\n",
    "#patch_model_name = 'v1.1_weak_labels_28x28x24'\n",
    "patch_model_name = 'weak_labels_3.0_batchnorm'\n",
    "patch_model_file = '../models/' + patch_model_name + '.h5'\n",
    "patch_model = load_model(patch_model_file, custom_objects={'LeakyReLU': keras.layers.LeakyReLU,\n",
    "                                                           'ELU': keras.layers.ELU,\n",
    "                                                           'ReLU': keras.layers.ReLU})\n",
    "patch_stride = 8\n",
    "patch_input_shape = patch_model.input_shape[2]\n",
    "\n",
    "# Note on dates: The date range should be longer than the spectrogram length.\n",
    "# Starting on successive mosaic periods (typically: monthly), as many\n",
    "# spectrograms are created as fit in the date range.\n",
    "start_date = '2019-01-01'\n",
    "end_date = '2021-06-01'\n",
    "\n",
    "mosaic_period = 3\n",
    "mosaic_method = 'min'\n",
    "spectrogram_interval = 2\n",
    "\n",
    "roi = 'brazil'\n",
    "roi_file = f'../data/boundaries/{roi}.geojson'\n",
    "dlkeys_file = f'../data/boundaries/dlkeys/{roi}_pop_0.5_dlkeys.txt'\n",
    "product_id = f'earthrise:{roi}_v{model_version}_{start_date}_{end_date}' \n",
    "patch_product_id = f'earthrise:{roi}_patch_{patch_model_version}_{start_date}_{end_date}_stride_{patch_stride}' \n",
    "product_name = product_id.split(':')[-1]  # Arbitrary string - optionally set this to something more human readable.\n",
    "\n",
    "run_local = False # If False, the model prediction tasks are async queued and sent to DL for processing.\n",
    "\n",
    "pop_thresh = 10 # Threshold to cut population raster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# If running locally, get results faster by setting smalle tilesize (100?)\n",
    "# If running on Descartes, use tilesize 900\n",
    "\n",
    "if run_local:\n",
    "    tilesize = 100\n",
    "else:\n",
    "    tilesize = 900\n",
    "\n",
    "# Generally, leave padding at 0\n",
    "padding = patch_input_shape - patch_stride\n",
    "\n",
    "args = [\n",
    "    '--roi_file',\n",
    "    roi_file,\n",
    "    '--dlkeys_file',\n",
    "    dlkeys_file,\n",
    "    '--product_id',\n",
    "    product_id,\n",
    "    '--patch_product_id',\n",
    "    patch_product_id,\n",
    "    '--product_name',\n",
    "    product_name,\n",
    "    '--model_file',\n",
    "    model_file,\n",
    "    '--model_name',\n",
    "    model_name,\n",
    "    '--patch_model_name',\n",
    "    patch_model_name,\n",
    "    '--patch_model_file',\n",
    "    patch_model_file,\n",
    "    '--patch_stride',\n",
    "    str(patch_stride),\n",
    "    '--mosaic_period',\n",
    "    str(mosaic_period),\n",
    "    '--mosaic_method',\n",
    "    mosaic_method,\n",
    "    '--spectrogram_interval',\n",
    "    str(spectrogram_interval),\n",
    "    '--start_date',\n",
    "    start_date,\n",
    "    '--end_date',\n",
    "    end_date,\n",
    "    '--pad',\n",
    "    str(padding),\n",
    "    '--tilesize',\n",
    "    str((tilesize // patch_input_shape) * patch_input_shape - padding),\n",
    "    '--pop_thresh',\n",
    "    str(pop_thresh)\n",
    "]\n",
    "if run_local:\n",
    "    args.append('--run_local')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Launch Descartes job. Monitor at https://monitor.descarteslabs.com/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Because of the way DL uploads modules when queuing async tasks, we need to launch from the scripts/ folder\n",
    "%cd ../scripts\n",
    "%pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check if patch feature collection exists. If it does, delete the FC\n",
    "fc_ids = [fc.id for fc in dl.vectors.FeatureCollection.list() if patch_product_id in fc.id]\n",
    "if len(fc_ids) > 0:\n",
    "    fc_id = fc_ids[0]\n",
    "    print(\"Existing product found.\\nDeleting\", fc_id)\n",
    "    dl.vectors.FeatureCollection(fc_id).delete()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "deploy_nn_v2.main(args)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Download Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Alternatively, input a known product_id to download for an earlier model run: \n",
    "#product_id = 'earthrise:Bali_spectrogramV0.0.7_2020-06-01_2021-04-01'\n",
    "#\n",
    "#roi = 'bali'\n",
    "#product_id = f'earthrise:{roi}_v{model_version}_{start_date}_{end_date}'\n",
    "#patch_product_id = f'earthrise:{roi}_patch_v{patch_model_version}_{start_date}_{end_date}' "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Download Patch Classifier Feature Collection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Downloading\", patch_product_id)\n",
    "fc_id = [fc.id for fc in dl.vectors.FeatureCollection.list() if patch_product_id in fc.id][0]\n",
    "fc = dl.vectors.FeatureCollection(fc_id)\n",
    "region = gpd.read_file(roi_file)['geometry']\n",
    "    \n",
    "features = []\n",
    "for elem in tqdm(fc.filter(region).features()):\n",
    "    features.append(elem.geojson)\n",
    "results = gpd.GeoDataFrame.from_features(features)\n",
    "results.to_file(f\"../data/model_outputs/patch_classifier/{patch_product_id.split(':')[-1]}.geojson\", driver='GeoJSON')\n",
    "print(len(features), 'features found')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Download pixel classifier raster"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After the job is complete (only signaled by looking at the console), use this script to download the files. This process downloads each file individually because Descartes may throw a 502 error when trying to download too many tiles."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "search = Image.search().filter(properties.product_id == product_id)\n",
    "search.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select one of these available bands\n",
    "product = dl.catalog.Product.get(product_id)\n",
    "for b in product.bands():\n",
    "    print(b.id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "band = 'median'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "basepath = os.path.join('../data/model_outputs/heatmaps', model_version, product_id.split(':')[-1] + f'mosaic-{band}')\n",
    "print(\"Saving to\", basepath)\n",
    "if not os.path.exists(basepath):\n",
    "    os.makedirs(basepath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "image_list = [image.id for image in search]\n",
    "raster_client = dl.Raster()\n",
    "for image in tqdm(image_list):\n",
    "    try:\n",
    "        raster_client.raster(inputs = image,\n",
    "                             bands = [band],\n",
    "                             outfile_basename = os.path.join(basepath, image),\n",
    "                             srs='WGS84')\n",
    "    except dl.client.exceptions.BadRequestError as e:\n",
    "        print(f'Warning: {repr(e)}\\nContinuing...')\n",
    "    except dl.client.exceptions.ServerError as e:\n",
    "        print(f'Warning: {repr(e)}\\nContinuing...')\n",
    "    except:\n",
    "        print(f'Some other error!')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Combine tiles into single raster\n",
    "Not recommended. Only run these cells if necessary. Mosaicing dramatically increases file size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "files_to_mosaic = []\n",
    "for file in os.listdir(basepath):\n",
    "    src = rs.open(os.path.join(basepath, file))\n",
    "    files_to_mosaic.append(src)\n",
    "mosaic, out_trans = merge(files_to_mosaic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_metadata = src.meta.copy()\n",
    "\n",
    "output_metadata.update({\"height\": mosaic.shape[1],\n",
    "                        \"width\": mosaic.shape[2],\n",
    "                        \"transform\": out_trans\n",
    "                 }\n",
    "                )\n",
    "output_metadata\n",
    "with rs.open(basepath + '.tif', 'w', **output_metadata) as f:\n",
    "    f.write(mosaic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Delete individual files and folder\n",
    "shutil.rmtree(basepath)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Batched Run\n",
    "Deploy model on to run using a folder of boundary files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# User inputs\n",
    "\n",
    "model_version = '0.0.11'\n",
    "model_name = 'spectrogram_v0.0.11_2021-07-13'\n",
    "model_file = '../models/' + model_name + '.h5'\n",
    "\n",
    "patch_model_version = 'weak_labels_1.1'\n",
    "patch_model_name = 'v1.1_weak_labels_28x28x24'\n",
    "patch_model_file = '../models/' + patch_model_name + '.h5'\n",
    "patch_model = load_model(patch_model_file, custom_objects={'LeakyReLU': keras.layers.LeakyReLU,\n",
    "                                                           'ELU': keras.layers.ELU,\n",
    "                                                           'ReLU': keras.layers.ReLU})\n",
    "patch_stride = 8\n",
    "patch_input_shape = patch_model.input_shape[2]\n",
    "\n",
    "# Note on dates: The date range should be longer than the spectrogram length.\n",
    "# Starting on successive mosaic periods (typically: monthly), as many\n",
    "# spectrograms are created as fit in the date range.\n",
    "start_date = '2019-01-01'\n",
    "end_date = '2021-06-01'\n",
    "\n",
    "mosaic_period = 3\n",
    "mosaic_method = 'min'\n",
    "spectrogram_interval = 2\n",
    "\n",
    "run_local = False # If False, the model prediction tasks are async queued and sent to DL for processing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "boundary_folder = '../data/boundaries/indonesian_islands'\n",
    "region_list = [file.split('.')[0] for file in os.listdir(boundary_folder) if 'DS' not in file]\n",
    "region_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bulk Deploy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for roi in region_list:\n",
    "    roi_file = os.path.join(boundary_folder, roi + '.geojson')\n",
    "    product_id = f'earthrise:{roi}_v{model_version}_{start_date}_{end_date}' \n",
    "    patch_product_id = f'earthrise:{roi}_patch_v{patch_model_version}_{start_date}_{end_date}' \n",
    "    product_name = product_id.split(':')[-1]  # Arbitrary string - optionally set this to something more human readable.\n",
    "    tilesize = 900\n",
    "\n",
    "    args = [\n",
    "        '--roi_file',\n",
    "        roi_file,\n",
    "        '--product_id',\n",
    "        product_id,\n",
    "        '--patch_product_id',\n",
    "        patch_product_id,\n",
    "        '--product_name',\n",
    "        product_name,\n",
    "        '--model_file',\n",
    "        model_file,\n",
    "        '--model_name',\n",
    "        model_name,\n",
    "        '--patch_model_name',\n",
    "        patch_model_name,\n",
    "        '--patch_model_file',\n",
    "        patch_model_file,\n",
    "        '--mosaic_period',\n",
    "        str(mosaic_period),\n",
    "        '--mosaic_method',\n",
    "        mosaic_method,\n",
    "        '--spectrogram_interval',\n",
    "        str(spectrogram_interval),\n",
    "        '--start_date',\n",
    "        start_date,\n",
    "        '--end_date',\n",
    "        end_date,\n",
    "        '--pad',\n",
    "        str(padding),\n",
    "        '--tilesize',\n",
    "        str((tilesize // patch_input_shape) * patch_input_shape - padding)\n",
    "    ]\n",
    "    if run_local:\n",
    "        args.append('--run_local')\n",
    "\n",
    "    # Because of the way DL uploads modules when queuing async tasks, we need to launch from the scripts/ folder\n",
    "    %cd ../scripts\n",
    "    %pwd\n",
    "\n",
    "    # Check if patch feature collection exists. If it does, delete the FC\n",
    "    fc_ids = [fc.id for fc in dl.vectors.FeatureCollection.list() if patch_product_id in fc.id]\n",
    "    if len(fc_ids) > 0:\n",
    "        fc_id = fc_ids[0]\n",
    "        print(\"Existing product found.\\nDeleting\", fc_id)\n",
    "        dl.vectors.FeatureCollection(fc_id).delete()\n",
    "    print(\"Deploying\", roi)\n",
    "    deploy_nn_v1.main(args)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bulk Download"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Patch classifier product download\n",
    "for roi in region_list:\n",
    "    roi_file = f'../data/boundaries/{roi}.geojson'\n",
    "    patch_product_id = f'earthrise:{roi}_patch_v{patch_model_version}_{start_date}_{end_date}' \n",
    "    print(\"Downloading\", patch_product_id)\n",
    "    fc_id = [fc.id for fc in dl.vectors.FeatureCollection.list() if patch_product_id in fc.id][0]\n",
    "    fc = dl.vectors.FeatureCollection(fc_id)\n",
    "    region = gpd.read_file(roi_file)['geometry']\n",
    "\n",
    "    features = []\n",
    "    for elem in tqdm(fc.filter(region).features()):\n",
    "        features.append(elem.geojson)\n",
    "    results = gpd.GeoDataFrame.from_features(features)\n",
    "    results.to_file(f\"../data/model_outputs/patch_classifier/{patch_product_id.split(':')[-1]}.geojson\", driver='GeoJSON')\n",
    "    print(len(features), 'features found')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pixel classifier product download\n",
    "for roi in region_list:\n",
    "    product_id = f'earthrise:{roi}_v{model_version}_{start_date}_{end_date}'\n",
    "    patch_product_id = f'earthrise:{roi}_patch_v{patch_model_version}_{start_date}_{end_date}' \n",
    "    \n",
    "    search = Image.search().filter(properties.product_id == product_id)\n",
    "    print(search.summary())\n",
    "    \n",
    "    # Select one of these available bands\n",
    "    product = dl.catalog.Product.get(product_id)\n",
    "    for b in product.bands():\n",
    "        print(b.id)\n",
    "    band = 'median'\n",
    "    \n",
    "    basepath = os.path.join('../data/model_outputs/heatmaps', model_version, product_id.split(':')[-1] + f'mosaic-{band}')\n",
    "    if not os.path.exists(basepath):\n",
    "        os.makedirs(basepath)\n",
    "        \n",
    "    image_list = [image.id for image in search]\n",
    "    raster_client = dl.Raster()\n",
    "    for image in tqdm(image_list):\n",
    "        try:\n",
    "            raster_client.raster(inputs = image,\n",
    "                                 bands = [band],\n",
    "                                 save=True,\n",
    "                                 outfile_basename = os.path.join(basepath, image),\n",
    "                                 srs='WGS84')\n",
    "        except dl.client.exceptions.BadRequestError as e:\n",
    "            print(f'Warning: {repr(e)}\\nContinuing...')\n",
    "        except dl.client.exceptions.ServerError as e:\n",
    "            print(f'Warning: {repr(e)}\\nContinuing...')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "8bc54daa053f2a84a21c2c1a9277c63a1734a487fc6225195213fddd5fc141a9"
  },
  "kernelspec": {
   "display_name": "mldev",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}