buildgti.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f08336c-0608-4dc6-a266-32a8762e406e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# buildGTI https://gdal.org/drivers/raster/gti.html#raster-gti\n",
    "# gdaltindex can be used to generate a tile index suitable for use by the GTI driver. There are two possibilities:\n",
    "\n",
    "# either use directly a vector tile index generated by gdaltindex as the input of the GTI driver\n",
    "\n",
    "# or generate a small XML .gti wrapper file, for easier use with non-file-based formats such as databases, or for vector formats that do not support setting layer metadata items.\n",
    "\n",
    "# Formats that support layer metadata are for example GeoPackage (-f GPKG), FlatGeoBuf (-f FlatGeoBuf) or PostGIS (-f PG)\n",
    "\n",
    "# Setting -tr and -ot is recommended to avoid the GTI driver to have to deduce them by opening the first tile in the index. If the tiles have nodata or mask band, -nodata and -mask should also be set.\n",
    "\n",
    "# In a GTI context, the extent of all tiles referenced in the tile index must be expressed in a single SRS. Consequently, if input tiles may have different SRS, either -t_srs or -skip_different_projection should be specified.\n",
    "\n",
    "\n",
    "#more info: https://gdal.org/programs/gdaltindex.html#options-specific-to-use-by-the-gdal-gti-driver and https://github.com/OSGeo/gdal/pull/8983"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "953c9279-e56a-41b3-991a-23f1a776eac9",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting package metadata (current_repodata.json): done\n",
      "Solving environment: done\n",
      "\n",
      "\n",
      "==> WARNING: A newer version of conda exists. <==\n",
      "  current version: 23.1.0\n",
      "  latest version: 23.11.0\n",
      "\n",
      "Please update conda by running\n",
      "\n",
      "    $ conda update -n base -c conda-forge conda\n",
      "\n",
      "Or to minimize the number of packages updated during conda update use\n",
      "\n",
      "     conda install conda=23.11.0\n",
      "\n",
      "\n",
      "\n",
      "# All requested packages already installed.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#!conda update -n base -c conda-forge conda\n",
    "!conda install -n base --yes --override-channels -c conda-forge mamba 'python_abi=*=*cp*'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a3a737cf-cf58-40a2-b5af-4dbe7b0f399d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting package metadata (current_repodata.json): done\n",
      "Solving environment: done\n",
      "\n",
      "\n",
      "==> WARNING: A newer version of conda exists. <==\n",
      "  current version: 23.1.0\n",
      "  latest version: 23.11.0\n",
      "\n",
      "Please update conda by running\n",
      "\n",
      "    $ conda update -n base -c conda-forge conda\n",
      "\n",
      "Or to minimize the number of packages updated during conda update use\n",
      "\n",
      "     conda install conda=23.11.0\n",
      "\n",
      "\n",
      "\n",
      "## Package Plan ##\n",
      "\n",
      "  environment location: /srv/conda/envs/gdal_master_env\n",
      "\n",
      "\n",
      "\n",
      "Preparing transaction: done\n",
      "Verifying transaction: done\n",
      "Executing transaction: done\n",
      "#\n",
      "# To activate this environment, use\n",
      "#\n",
      "#     $ conda activate gdal_master_env\n",
      "#\n",
      "# To deactivate an active environment, use\n",
      "#\n",
      "#     $ conda deactivate\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!conda create --yes --name gdal_master_env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bd24dd8d-7a67-4694-927c-450713094e39",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "CommandNotFoundError: Your shell has not been properly configured to use 'conda activate'.\n",
      "To initialize your shell, run\n",
      "\n",
      "    $ conda init <SHELL_NAME>\n",
      "\n",
      "Currently supported shells are:\n",
      "  - bash\n",
      "  - fish\n",
      "  - tcsh\n",
      "  - xonsh\n",
      "  - zsh\n",
      "  - powershell\n",
      "\n",
      "See 'conda init --help' for more information and options.\n",
      "\n",
      "IMPORTANT: You may need to close and restart your shell after running 'conda init'.\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!conda activate gdal_master_env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "98679f27-5959-4632-b053-8c243a4163fa",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "                  __    __    __    __\n",
      "                 /  \\  /  \\  /  \\  /  \\\n",
      "                /    \\/    \\/    \\/    \\\n",
      "███████████████/  /██/  /██/  /██/  /████████████████████████\n",
      "              /  / \\   / \\   / \\   / \\  \\____\n",
      "             /  /   \\_/   \\_/   \\_/   \\    o \\__,\n",
      "            / _/                       \\_____/  `\n",
      "            |/\n",
      "        ███╗   ███╗ █████╗ ███╗   ███╗██████╗  █████╗\n",
      "        ████╗ ████║██╔══██╗████╗ ████║██╔══██╗██╔══██╗\n",
      "        ██╔████╔██║███████║██╔████╔██║██████╔╝███████║\n",
      "        ██║╚██╔╝██║██╔══██║██║╚██╔╝██║██╔══██╗██╔══██║\n",
      "        ██║ ╚═╝ ██║██║  ██║██║ ╚═╝ ██║██████╔╝██║  ██║\n",
      "        ╚═╝     ╚═╝╚═╝  ╚═╝╚═╝     ╚═╝╚═════╝ ╚═╝  ╚═╝\n",
      "\n",
      "        mamba (1.4.1) supported by @QuantStack\n",
      "\n",
      "        GitHub:  https://github.com/mamba-org/mamba\n",
      "        Twitter: https://twitter.com/QuantStack\n",
      "\n",
      "█████████████████████████████████████████████████████████████\n",
      "\n",
      "\n",
      "Looking for: ['gdal']\n",
      "\n",
      "conda-forge/linux-64                                        Using cache\n",
      "conda-forge/noarch                                          Using cache\n",
      "\u001b[?25l\u001b[2K\u001b[0G[+] 0.0s\n",
      "\u001b[2K\u001b[1A\u001b[2K\u001b[0G[+] 0.1s\n",
      "gdal-master/linux-64 \u001b[33m━━━━━━━━━━╸\u001b[0m\u001b[90m━━━━━━━━━━━━\u001b[0m   0.0 B /  ??.?MB @  ??.?MB/s  0.1s\n",
      "gdal-master/noarch   \u001b[33m━━━━━━━━━╸\u001b[0m\u001b[90m━━━━━━━━━━━━━\u001b[0m   0.0 B /  ??.?MB @  ??.?MB/s  0.1s\u001b[2K\u001b[1A\u001b[2K\u001b[1A\u001b[2K\u001b[0G[+] 0.2s\n",
      "gdal-master/linux-64 \u001b[33m━━━━━━━━━━━╸\u001b[0m\u001b[90m━━━━━━━━━━━\u001b[0m   0.0 B /  ??.?MB @  ??.?MB/s  0.2s\n",
      "gdal-master/noarch   \u001b[33m━━━━━━━━━━╸\u001b[0m\u001b[90m━━━━━━━━━━━━\u001b[0m   0.0 B /  ??.?MB @  ??.?MB/s  0.2s\u001b[2K\u001b[1A\u001b[2K\u001b[1A\u001b[2K\u001b[0Ggdal-master/noarch                                 130.0 B @ 472.0 B/s  0.3s\n",
      "[+] 0.3s\n",
      "gdal-master/linux-64 \u001b[33m━━━━━━━━━━━━╸\u001b[0m\u001b[90m━━━━━━━━━━\u001b[0m   0.0 B /  ??.?MB @  ??.?MB/s  0.3s\u001b[2K\u001b[1A\u001b[2K\u001b[0G[+] 0.4s\n",
      "gdal-master/linux-64 \u001b[33m━━━━━━━━━━━━━╸\u001b[0m\u001b[90m━━━━━━━━━\u001b[0m   0.0 B /  ??.?MB @  ??.?MB/s  0.4s\u001b[2K\u001b[1A\u001b[2K\u001b[0G[+] 0.5s\n",
      "gdal-master/linux-64 \u001b[33m━━━━━━━━━━━━━━╸\u001b[0m\u001b[90m━━━━━━━━\u001b[0m   0.0 B /  ??.?MB @  ??.?MB/s  0.5s\u001b[2K\u001b[1A\u001b[2K\u001b[0G[+] 0.6s\n",
      "gdal-master/linux-64 \u001b[33m━━━━━━━━━━━━━━━╸\u001b[0m\u001b[90m━━━━━━━\u001b[0m   2.9kB /  ??.?MB @   5.0kB/s  0.6s\u001b[2K\u001b[1A\u001b[2K\u001b[0G[+] 0.7s\n",
      "gdal-master/linux-64 \u001b[90m╸\u001b[0m\u001b[33m━━━━━━━━━━━━━━━╸\u001b[0m\u001b[90m━━━━━━\u001b[0m   9.6kB /  ??.?MB @  14.5kB/s  0.7s\u001b[2K\u001b[1A\u001b[2K\u001b[0G[+] 0.8s\n",
      "gdal-master/linux-64 \u001b[90m━╸\u001b[0m\u001b[33m━━━━━━━━━━━━━━━╸\u001b[0m\u001b[90m━━━━━\u001b[0m  11.8kB /  ??.?MB @  15.6kB/s  0.8s\u001b[2K\u001b[1A\u001b[2K\u001b[0Ggdal-master/linux-64                                25.7kB @  30.4kB/s  0.8s\n",
      "\u001b[?25h\n",
      "Pinned packages:\n",
      "  - python 3.11.*\n",
      "\n",
      "\n",
      "Transaction\n",
      "\n",
      "  Prefix: /srv/conda/envs/notebook\n",
      "\n",
      "  All requested packages already installed\n",
      "\n",
      "\u001b[?25l\u001b[2K\u001b[0G\u001b[?25h"
     ]
    }
   ],
   "source": [
    "!mamba install --yes -c gdal-master gdal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1273707d-ff65-46b7-bbf9-3618965ea6ac",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GDAL 3.7.0, released 2023/05/02\n"
     ]
    }
   ],
   "source": [
    "!gdalinfo --version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "621f812c-61ad-483b-a49b-7f5b7ffc7819",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GDAL 3.7.0, released 2023/05/02\n"
     ]
    }
   ],
   "source": [
    "!gdalinfo --version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eea0d00f-47b1-404d-b1af-12893762d44c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import adlfs\n",
    "import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58b3138c-4b11-4308-ba77-cbb06ad4e8fd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sas_token = \"se=2024-02-08T20%3A33Z&sp=racwdl&sv=2022-11-02&sr=c&skoid=598b2582-4d1d-4c4e-9bb2-3ad571b791b5&sktid=f6b6dd5b-f02f-441a-99a0-162ac5060bd2&skt=2024-02-01T20%3A34%3A11Z&ske=2024-02-08T20%3A33%3A00Z&sks=b&skv=2022-11-02&sig=sghHRHkJrGpyT2MFAqUzno5TNsg75gBRKqrqcOZSaTE%3D\"\n",
    "fs = adlfs.AzureBlobFileSystem(account_name=\"snowmelt\", credential=sas_token, anon=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a0bee0f-b62e-4660-bab7-96d20ce645be",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#tile_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1a6242d-7ef3-46da-9d85-3434274af434",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "years = [2015,2016,2017,2018,2019,2020,2021,2022,2023]\n",
    "years = ['allyears_median','allyears_std','allyears_corr_strength','2015_median','2016_median','2017_median','2018_median','2019_median','2020_median','2021_median','2022_median']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d3f13c6-b7cb-4eaf-b46a-d1af6d0dbc79",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa2cb6b3-08cb-4fa7-9e78-1a876c5e668f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for year in years:\n",
    "    f = open(f'gti/file_list/files_{year}.txt','w')\n",
    "    for fn in fs.glob(f'snowmelt/eric/global/*/*{year}*'):\n",
    "        f.write(f'https://snowmelt.blob.core.windows.net/{fn}\\n')\n",
    "    f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17c9802f-ffc4-4bad-ac34-31f4754cdfd8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!gdalinfo --version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47bb8004-4793-4fb9-b3b2-1ae1f0925829",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# add -gti_filename,-tr, -ot, t_srs, [cehck above]\n",
    "\n",
    "!gdaltindex -t_srs EPSG:4326 tile_index.gti.gpkg --optfile $f'gti/file_list/files_{year}.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7a85cc6-1c50-413c-bfd8-963f2e975d7d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!gdalwarp $f'tile_index.gti.gpkg' $f'/tmp/tif_{year}.tif' -co BIGTIFF=YES -of COG #store in tmp?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac77b63e-72f1-4456-a030-7c82d61a32ad",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import rioxarray as rxr\n",
    "ind = rxr.open_rasterio('tile_index.gti.gpkg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d4d3c14-fcca-49d3-882d-da9bb8940b3e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ind"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67457a13-dc1d-4fa9-8f67-5b705f4885e1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "08fabc51-0232-4841-8d14-475069c9e813",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "84ca4148-2ff8-48f6-91c2-227fc35b7b27",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "098fcd47-1383-4be3-88a8-2ce8c797ac96",
   "metadata": {},
   "outputs": [],
   "source": [
    "for year in years:\n",
    "    # call gti"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "45f80751-854a-4d56-855b-357e9338441c",
   "metadata": {},
   "outputs": [],
   "source": [
    "for year in years[5:]:\n",
    "    if (year == 'allyears_std') | (year == 'allyears_corr_strength') | (year == 'allyears_polyfit'):\n",
    "        nodata = \"nan\"\n",
    "    else:\n",
    "        nodata = '-32768'\n",
    "    \n",
    "    print(f'for year {year}, running gdalwarp...')\n",
    "    !gdalwarp $f'gti/gti_{year}.gti' $f'/tmp/tif/4326/tif_{year}.tif' -co BIGTIFF=YES -of COG #store in tmp?\n",
    "    \n",
    "    print(f'for year {year}, uploading mosaic')\n",
    "    fs.upload(f'/tmp/tif/4326/tif_{year}.tif',f'/snowmelt/eric/mosaics/global_4326/runoff_onset_mosaic_{year}_{resolution}m_4326.tif',overwrite=True)\n",
    "    \n",
    "    !rm $f'/tmp/tif/4326/tif_{year}.tif'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69e8dbb9-6bc6-44c6-8b63-30b90a321280",
   "metadata": {},
   "outputs": [],
   "source": [
    "# warp gti to tif"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}