2.Datasets/Downloading_data_with_Google_Earth_Engine.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Downloading data with Google Earth Engine.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WtVBUg3eZ__h"
      },
      "source": [
        "# Downloading data with Google Earth Engine"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Q0Q1WdCZw4r"
      },
      "source": [
        "import numpy as np\r\n",
        "import math\r\n",
        "import scipy\r\n",
        "import pandas as pd\r\n",
        "import PIL\r\n",
        "import gdal\r\n",
        "import matplotlib.pyplot as plt\r\n",
        "plt.style.use('ggplot')\r\n",
        "import sys, os\r\n",
        "from pathlib import Path\r\n",
        "import time\r\n",
        "import random\r\n",
        "import collections, functools, operator\r\n",
        "import csv\r\n",
        "from osgeo import gdal,osr\r\n",
        "from gdalconst import *\r\n",
        "import subprocess\r\n",
        "from osgeo.gdalconst import GA_Update\r\n",
        "import datetime\r\n",
        "\r\n",
        "import ee\r\n",
        "ee.Authenticate()\r\n",
        "ee.Initialize()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r0B9dC_paAKy"
      },
      "source": [
        "def ee_get_intersections(collection1, collection2, n1, n2):\r\n",
        "    dates1 = collection1.aggregate_array('system:time_start').getInfo()\r\n",
        "    dates1 = [datetime.datetime.fromtimestamp(x // 1000).strftime('%Y-%m-%d') for x in dates1]\r\n",
        "    dates2 = collection2.aggregate_array('system:time_start').getInfo()\r\n",
        "    dates2 = [datetime.datetime.fromtimestamp(x // 1000).strftime('%Y-%m-%d') for x in dates2]\r\n",
        "    datesIntersections = []\r\n",
        "    print(dates1)\r\n",
        "    print(dates2)\r\n",
        "    for date in dates1:\r\n",
        "        if date in dates2:\r\n",
        "            datesIntersections.append(date)\r\n",
        "    return datesIntersections\r\n",
        "\r\n",
        "\r\n",
        "def image_type(img, type):\r\n",
        "    types = {   \"float\":img.toFloat(), \r\n",
        "                \"byte\":img.toByte(), \r\n",
        "                \"int\":img.toInt(),\r\n",
        "                \"double\":img.toDouble(),\r\n",
        "                \"long\": img.toLong(),\r\n",
        "                \"short\": img.toShort(),\r\n",
        "                \"int8\": img.toInt8(),\r\n",
        "                \"int16\": img.toInt16(),\r\n",
        "                \"int32\": img.toInt32(),\r\n",
        "                \"int64\": img.toInt64(),\r\n",
        "                \"uint8\": img.toUint8(),\r\n",
        "                \"uint16\": img.toUint16(),\r\n",
        "                \"uint32\": img.toUint32()}\r\n",
        "  \r\n",
        "    return types[type]\r\n",
        "\r\n",
        "def ee_imageToTask(img, scale, colID=None, crs=None, id=None):\r\n",
        "    if id == None:\r\n",
        "        try:\r\n",
        "            id = img.id().getInfo()\r\n",
        "        except:\r\n",
        "            id = colID+'_image_'+str(i)\r\n",
        "\r\n",
        "    img = image_type(img, \"int32\")\r\n",
        "    \r\n",
        "    return ee.batch.Export.image.toDrive(image=img,\r\n",
        "                            description=id,\r\n",
        "                            folder=folder,\r\n",
        "                            fileNamePrefix=id,\r\n",
        "                            scale=scale, # 30 meters per pixel, set this value according to satellite imagery documentation\r\n",
        "                            maxPixels=1e13,\r\n",
        "                            crs=crs) # set as high as possible\r\n",
        "\r\n",
        "\r\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lnQlT5WM8A0w"
      },
      "source": [
        "## Downloads Hyperion and Landsat day-matching scenes over an area of interest"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "98Z2HL1k7diI"
      },
      "source": [
        "folder = 'GEECollection Projection 32611'\r\n",
        "\r\n",
        "\r\n",
        "collection1 = (ee.ImageCollection('LANDSAT/LE07/C01/T1_SR').filter(ee.Filter.date('2014-01-01', '2014-12-31'))\r\n",
        "                                                            .filter(ee.Filter.eq('WRS_PATH', 144))\r\n",
        "                                                            .filter(ee.Filter.eq('WRS_ROW', 52)))\r\n",
        "collection2 = (ee.ImageCollection('EO1/HYPERION').filter(ee.Filter.date('2001-05-01', '2017-03-13'))\r\n",
        "                                                            .filter(ee.Filter.eq('WRS_PATH', 144))\r\n",
        "                                                            .filter(ee.Filter.eq('WRS_ROW', 52)))\r\n",
        "\r\n",
        "\r\n",
        "n1 = collection1.size().getInfo()\r\n",
        "print(n1)\r\n",
        "colList1 = collection1.toList(n1)\r\n",
        "colID1 = collection1.getInfo()['id'].replace('/','_')\r\n",
        "img = ee.Image(colList1.get(1))\r\n",
        "print(datetime.datetime.fromtimestamp((img.date().getInfo()['value'] // 1000)).strftime('%Y-%m-%d'))\r\n",
        "n2 = collection2.size().getInfo()\r\n",
        "print(n2)\r\n",
        "colList2 = collection2.toList(n2)\r\n",
        "colID2 = collection2.getInfo()['id'].replace('/','_')\r\n",
        "\r\n",
        "\r\n",
        "datesIntersections = ee_get_intersections(collection1, collection2, n1, n2)\r\n",
        "\r\n",
        "print(datesIntersections)\r\n",
        "\r\n",
        "tasks1 = []\r\n",
        "for i in range(n1):\r\n",
        "    img = ee.Image(colList1.get(i))\r\n",
        "    date =  datetime.datetime.fromtimestamp((img.date().getInfo()['value']) // 1000).strftime('%Y-%m-%d')\r\n",
        "    if date in datesIntersections:\r\n",
        "        tasks1.append(ee_imageToTask(img, 30, colID1))\r\n",
        "\r\n",
        "tasks2 = []\r\n",
        "for i in range(n2):\r\n",
        "    img = ee.Image(colList2.get(i))\r\n",
        "    date =  datetime.datetime.fromtimestamp((img.date().getInfo()['value']) // 1000).strftime('%Y-%m-%d')\r\n",
        "    if date in datesIntersections:\r\n",
        "        tasks2.append(ee_imageToTask(img, 30, colID2))\r\n",
        "\r\n",
        "\r\n",
        "print(tasks1)\r\n",
        "print(tasks2)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RWh1kPEs7lih"
      },
      "source": [
        "## Downloads California area data (Hyperion and USDA NASS Crop Data Layer)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Yd6k8yWY7bLc"
      },
      "source": [
        "AOI = ee.Geometry.Polygon([[[-120.36302942846854, 36.950654941997534],\r\n",
        "                    [-120.36302942846854, 35.889823731552326],\r\n",
        "                    [-119.74230188940604, 35.889823731552326],\r\n",
        "                    [-119.74230188940604, 36.950654941997534]]], None, False)\r\n",
        "collection = (ee.ImageCollection('USDA/NASS/CDL').filter(ee.Filter.date('2016-01-01', '2016-01-02'))\r\n",
        "                                                .first())\r\n",
        "cropLandcover = collection.select('cropland')\r\n",
        "cropLandcover = cropLandcover.clip(AOI)\r\n",
        "task = ee_imageToTask(cropLandcover, 30, crs='EPSG:32611')\r\n",
        "task.start()\r\n",
        "\r\n",
        "\r\n",
        "\r\n",
        "AOI = ee.Geometry.Point(-120.04305262182791,36.47617892554505)\r\n",
        "collection = (ee.ImageCollection('EO1/HYPERION').filter(ee.Filter.date('2016-02-22', '2016-02-23')) # Target Path 42, Target Row 35\r\n",
        "                                                .filterBounds(AOI)\r\n",
        "                                                .first())\r\n",
        "\r\n",
        "\r\n",
        "bands=['B008', 'B009', 'B010', 'B011', 'B012', 'B013', 'B014', 'B015', 'B016', 'B017',\r\n",
        "        'B018', 'B019', 'B020', 'B021', 'B022', 'B023', 'B024', 'B025', 'B026', 'B027',\r\n",
        "        'B028', 'B029', 'B030', 'B031', 'B032', 'B033', 'B034', 'B035', 'B036', 'B037',\r\n",
        "        'B038', 'B039', 'B040', 'B041', 'B042', 'B043', 'B044', 'B045', 'B046', 'B047',\r\n",
        "        'B048', 'B049', 'B050', 'B051', 'B052', 'B053', 'B054', 'B055', 'B056', 'B057',\r\n",
        "        \r\n",
        "        'B079', 'B080', 'B081', 'B082', 'B083', 'B084', 'B085', 'B086', 'B087', \r\n",
        "        'B088', 'B089', 'B090', 'B091', 'B092', 'B093', 'B094', 'B095', 'B096', 'B097', \r\n",
        "        'B098', 'B099', 'B100', 'B101', 'B102', 'B103', 'B104', 'B105', 'B106', 'B107', \r\n",
        "        'B108', 'B109', 'B110', 'B111', 'B112', 'B113', 'B114', 'B115', 'B116', 'B117', \r\n",
        "        'B118', 'B119', 'B120',\r\n",
        "        \r\n",
        "        'B131', 'B132', 'B133', 'B134', 'B135', 'B136', 'B137', 'B138', 'B139', 'B140', \r\n",
        "        'B141', 'B142', 'B143', 'B144', 'B145', 'B146', 'B147', 'B148', 'B149', 'B150', \r\n",
        "        'B151', 'B152', 'B153', 'B154', 'B155', 'B156', 'B157', 'B158', 'B159', 'B160', \r\n",
        "        'B161', 'B162', 'B163', 'B164', 'B165',\r\n",
        "        \r\n",
        "        'B181', 'B182', 'B183', 'B184', 'B185', 'B186', 'B187', 'B188', 'B189', 'B190', \r\n",
        "        'B191', 'B192', 'B193', 'B194', 'B195', 'B196', 'B197', 'B198', 'B199', 'B200', \r\n",
        "        'B201', 'B202', 'B203', 'B204', 'B205', 'B206', 'B207', 'B208', 'B209', 'B210', \r\n",
        "        'B211', 'B212', 'B213', 'B214', 'B215', 'B216', 'B217', 'B218', 'B219', 'B220', \r\n",
        "        'B221', 'B222', 'B223']\r\n",
        "\r\n",
        "tasks = []\r\n",
        "for band in bands:\r\n",
        "    cropLandcover = collection.select([band])\r\n",
        "    id = cropLandcover.id().getInfo()[:-2] + band\r\n",
        "    tasks.append(ee_imageToTask(cropLandcover, 30, crs='EPSG:32611', id=id))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ehHn362OaNAK"
      },
      "source": [
        "for task in tasks:\r\n",
        "    task.start()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_ODqBnwE7zLJ"
      },
      "source": [
        "## Downloads Landsat data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nxNN6pHlaQrD"
      },
      "source": [
        "folder = 'GEECollection'\r\n",
        "\r\n",
        "#Function to mask clouds based on the pixel_qa band of Landsat SR data.\r\n",
        "#param {ee.Image} image Input Landsat SR image\r\n",
        "#return {ee.Image} Cloudmasked Landsat image\r\n",
        "def cloudMaskL457(image):\r\n",
        "    # If the cloud bit (5) is set and the cloud confidence (7) is high\r\n",
        "    # or the cloud shadow bit is set (3), then it's a bad pixel.\r\n",
        "    # mask2 removes edge pixels that don't occur in all bands\r\n",
        "    qa = image.select('pixel_qa')\r\n",
        "    cloud = qa.bitwiseAnd(1 << 5).And(qa.bitwiseAnd(1 << 7)).Or(qa.bitwiseAnd(1 << 3))\r\n",
        "    mask2 = image.mask().reduce(ee.Reducer.min())\r\n",
        "    return image.updateMask(cloud.Not()).updateMask(mask2)\r\n",
        "\r\n",
        "def gapfill(image):\r\n",
        "    return image.focal_mean()\r\n",
        "\r\n",
        "\r\n",
        "coll_notmasked = (ee.ImageCollection('LANDSAT/LE07/C01/T1_SR').filter(ee.Filter.eq('WRS_PATH', 144))\r\n",
        "                                                    .filter(ee.Filter.eq('WRS_ROW', 52))\r\n",
        "                                                    .select(['B1', 'B2', 'B3', 'B4', 'B5', 'B7', 'pixel_qa']))\r\n",
        "\r\n",
        "coll_masked = (ee.ImageCollection('LANDSAT/LE07/C01/T1_SR').filter(ee.Filter.eq('WRS_PATH', 144))\r\n",
        "                                                    .filter(ee.Filter.eq('WRS_ROW', 52))\r\n",
        "                                                    .select(['B1', 'B2', 'B3', 'B4', 'B5', 'B7', 'pixel_qa'])\r\n",
        "                                                    .map(cloudMaskL457))\r\n",
        "\r\n",
        "\r\n",
        "datesOI = ['2017-02-11', '2016-11-07', '2016-10-06', '2016-02-25', '2016-01-24', '2015-11-21', '2015-08-17', '2015-04-11', '2015-02-22', '2015-02-06',\r\n",
        "           '2014-10-17', '2014-10-01', '2014-08-30', '2014-06-27', '2014-03-07', '2014-02-19', '2013-10-30', '2013-09-28', '2012-11-12', '2012-10-27']\r\n",
        "\r\n",
        "def ee_dates_of_interest_from_collection(coll, datesOI):\r\n",
        "    nImages = coll.size().getInfo()\r\n",
        "    collList = coll.toList(nImages)\r\n",
        "    collID = coll.getInfo()['id'].replace('/','_')\r\n",
        "    tasks = []\r\n",
        "    for i in range(nImages):\r\n",
        "        img = ee.Image(collList.get(i))\r\n",
        "        date =  datetime.datetime.fromtimestamp((img.date().getInfo()['value']) // 1000).strftime('%Y-%m-%d')\r\n",
        "        if date in datesOI:\r\n",
        "            tasks.append(ee_imageToTask(img, 30, collID))\r\n",
        "    return tasks\r\n",
        "def ee_execute_tasks(tasks):\r\n",
        "    for task in tasks:\r\n",
        "        task.start()\r\n",
        "\r\n",
        "def ee_check_tasks_status(tasks):\r\n",
        "    for task in tasks:\r\n",
        "        print(ee.data.getTaskStatus(task.id)) # To check task state\r\n",
        "\r\n",
        "\r\n",
        "tasks = ee_dates_of_interest_from_collection(coll_masked, datesOI)\r\n",
        "ee_execute_tasks(tasks)\r\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ln6lWXbAaceh"
      },
      "source": [
        "ee_check_tasks_status(tasks)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YgVj7KKraYja"
      },
      "source": [
        "x = gdal.Open(os.getcwd() + '/drive/MyDrive/GEECollection/LE07_144052_20190116.tif', GA_Update)\r\n",
        "x_arr = x.ReadAsArray()\r\n",
        "print(x_arr.shape)\r\n",
        "\r\n",
        "for i in range(1, 11+1):\r\n",
        "    print(x.GetRasterBand(i).GetDescription())"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}