05 Test-Train Split and train - Fixed, overnight run-Copy1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import os\n",
    "from collections import Counter\n",
    "from datetime import datetime\n",
    "import pickle\n",
    "import copy\n",
    "import datetime\n",
    "\n",
    "import geopandas as gpd\n",
    "import shapely.geometry\n",
    "import rasterio\n",
    "import json\n",
    "import geopandas as gpd\n",
    "import geopandas_osm.osm\n",
    "from descartes import PolygonPatch\n",
    "import h5py \n",
    "from scipy.misc import imresize\n",
    "import shapely.geometry\n",
    "import tensorflow as tf\n",
    "import cv2\n",
    "\n",
    "#%matplotlib inline\n",
    "#%config InlineBackend.figure_format = 'retina'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tile_no</th>\n",
       "      <th>flooded</th>\n",
       "      <th>post_resized</th>\n",
       "      <th>pre_resized</th>\n",
       "      <th>post_full</th>\n",
       "      <th>pre_full</th>\n",
       "      <th>course_mask_full</th>\n",
       "      <th>course_mask_resized</th>\n",
       "      <th>fine_mask_filename</th>\n",
       "      <th>footprint</th>\n",
       "      <th>dry_or_wet</th>\n",
       "      <th>mask_poly</th>\n",
       "      <th>tile_transform</th>\n",
       "      <th>geometry</th>\n",
       "      <th>DBScan</th>\n",
       "      <th>DBScan_gauss</th>\n",
       "      <th>bad_image</th>\n",
       "      <th>problem</th>\n",
       "      <th>verified</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.80323</td>\n",
       "      <td>0_post_resize_img</td>\n",
       "      <td>0_pre_resize_img</td>\n",
       "      <td>0_post_full_img</td>\n",
       "      <td>0_pre_full_img</td>\n",
       "      <td>0_mask</td>\n",
       "      <td>0_resize_mask</td>\n",
       "      <td>0_256_fine_mask</td>\n",
       "      <td>3002220.tif</td>\n",
       "      <td>wet</td>\n",
       "      <td>(POLYGON ((-95.57181511210993 29.4410615808823...</td>\n",
       "      <td>[222822.4, 0.0, 0.0, -222822.4, 21295616.0, 65...</td>\n",
       "      <td>POLYGON ((-95.56985294117646 29.44106158088235...</td>\n",
       "      <td>0_256_DBSCAN</td>\n",
       "      <td>0_256_fine_mask_blur</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.8787</td>\n",
       "      <td>1_post_resize_img</td>\n",
       "      <td>1_pre_resize_img</td>\n",
       "      <td>1_post_full_img</td>\n",
       "      <td>1_pre_full_img</td>\n",
       "      <td>1_mask</td>\n",
       "      <td>1_resize_mask</td>\n",
       "      <td>1_256_fine_mask</td>\n",
       "      <td>3002220.tif</td>\n",
       "      <td>wet</td>\n",
       "      <td>POLYGON ((-95.56764981800494 29.44136752727207...</td>\n",
       "      <td>[222822.4, 0.0, 0.0, -222822.4, 21295104.0, 65...</td>\n",
       "      <td>POLYGON ((-95.56755514705883 29.44106158088235...</td>\n",
       "      <td>1_256_DBSCAN</td>\n",
       "      <td>1_256_fine_mask_blur</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0.864578</td>\n",
       "      <td>2_post_resize_img</td>\n",
       "      <td>2_pre_resize_img</td>\n",
       "      <td>2_post_full_img</td>\n",
       "      <td>2_pre_full_img</td>\n",
       "      <td>2_mask</td>\n",
       "      <td>2_resize_mask</td>\n",
       "      <td>2_256_fine_mask</td>\n",
       "      <td>3002220.tif</td>\n",
       "      <td>wet</td>\n",
       "      <td>(POLYGON ((-95.5654521647227 29.44106158088235...</td>\n",
       "      <td>[222822.4, 0.0, 0.0, -222822.4, 21294592.0, 65...</td>\n",
       "      <td>POLYGON ((-95.56525735294117 29.44106158088235...</td>\n",
       "      <td>2_256_DBSCAN</td>\n",
       "      <td>2_256_fine_mask_blur</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0.576237</td>\n",
       "      <td>3_post_resize_img</td>\n",
       "      <td>3_pre_resize_img</td>\n",
       "      <td>3_post_full_img</td>\n",
       "      <td>3_pre_full_img</td>\n",
       "      <td>3_mask</td>\n",
       "      <td>3_resize_mask</td>\n",
       "      <td>3_256_fine_mask</td>\n",
       "      <td>3002220.tif</td>\n",
       "      <td>wet</td>\n",
       "      <td>(POLYGON ((-95.56505909959365 29.443359375, -9...</td>\n",
       "      <td>[222822.4, 0.0, 0.0, -222822.4, 21294080.0, 65...</td>\n",
       "      <td>POLYGON ((-95.56295955882354 29.44106158088235...</td>\n",
       "      <td>3_256_DBSCAN</td>\n",
       "      <td>3_256_fine_mask_blur</td>\n",
       "      <td>True</td>\n",
       "      <td>c</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0.59251</td>\n",
       "      <td>4_post_resize_img</td>\n",
       "      <td>4_pre_resize_img</td>\n",
       "      <td>4_post_full_img</td>\n",
       "      <td>4_pre_full_img</td>\n",
       "      <td>4_mask</td>\n",
       "      <td>4_resize_mask</td>\n",
       "      <td>4_256_fine_mask</td>\n",
       "      <td>3002220.tif</td>\n",
       "      <td>wet</td>\n",
       "      <td>(POLYGON ((-95.56178077548428 29.4410615808823...</td>\n",
       "      <td>[222822.4, 0.0, 0.0, -222822.4, 21293568.0, 65...</td>\n",
       "      <td>POLYGON ((-95.56066176470588 29.44106158088235...</td>\n",
       "      <td>4_256_DBSCAN</td>\n",
       "      <td>4_256_fine_mask_blur</td>\n",
       "      <td>True</td>\n",
       "      <td>c</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>0.664898</td>\n",
       "      <td>5_post_resize_img</td>\n",
       "      <td>5_pre_resize_img</td>\n",
       "      <td>5_post_full_img</td>\n",
       "      <td>5_pre_full_img</td>\n",
       "      <td>5_mask</td>\n",
       "      <td>5_resize_mask</td>\n",
       "      <td>5_256_fine_mask</td>\n",
       "      <td>3002220.tif</td>\n",
       "      <td>wet</td>\n",
       "      <td>(POLYGON ((-95.56066176470588 29.4432499281638...</td>\n",
       "      <td>[222822.4, 0.0, 0.0, -222822.4, 21293056.0, 65...</td>\n",
       "      <td>POLYGON ((-95.55836397058823 29.44106158088235...</td>\n",
       "      <td>5_256_DBSCAN</td>\n",
       "      <td>5_256_fine_mask_blur</td>\n",
       "      <td>True</td>\n",
       "      <td>c</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>0.901104</td>\n",
       "      <td>6_post_resize_img</td>\n",
       "      <td>6_pre_resize_img</td>\n",
       "      <td>6_post_full_img</td>\n",
       "      <td>6_pre_full_img</td>\n",
       "      <td>6_mask</td>\n",
       "      <td>6_resize_mask</td>\n",
       "      <td>6_256_fine_mask</td>\n",
       "      <td>3002220.tif</td>\n",
       "      <td>wet</td>\n",
       "      <td>POLYGON ((-95.55606617647059 29.44209793185462...</td>\n",
       "      <td>[222822.4, 0.0, 0.0, -222822.4, 21292544.0, 65...</td>\n",
       "      <td>POLYGON ((-95.55606617647059 29.44106158088235...</td>\n",
       "      <td>6_256_DBSCAN</td>\n",
       "      <td>6_256_fine_mask_blur</td>\n",
       "      <td>True</td>\n",
       "      <td>c</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>0.53014</td>\n",
       "      <td>7_post_resize_img</td>\n",
       "      <td>7_pre_resize_img</td>\n",
       "      <td>7_post_full_img</td>\n",
       "      <td>7_pre_full_img</td>\n",
       "      <td>7_mask</td>\n",
       "      <td>7_resize_mask</td>\n",
       "      <td>7_256_fine_mask</td>\n",
       "      <td>3002220.tif</td>\n",
       "      <td>wet</td>\n",
       "      <td>(POLYGON ((-95.55376838235294 29.4431568755186...</td>\n",
       "      <td>[222822.4, 0.0, 0.0, -222822.4, 21292032.0, 65...</td>\n",
       "      <td>POLYGON ((-95.55376838235294 29.44106158088235...</td>\n",
       "      <td>7_256_DBSCAN</td>\n",
       "      <td>7_256_fine_mask_blur</td>\n",
       "      <td>True</td>\n",
       "      <td>c</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>0.448421</td>\n",
       "      <td>8_post_resize_img</td>\n",
       "      <td>8_pre_resize_img</td>\n",
       "      <td>8_post_full_img</td>\n",
       "      <td>8_pre_full_img</td>\n",
       "      <td>8_mask</td>\n",
       "      <td>8_resize_mask</td>\n",
       "      <td>8_256_fine_mask</td>\n",
       "      <td>3002220.tif</td>\n",
       "      <td>wet</td>\n",
       "      <td>(POLYGON ((-95.55376838235294 29.4431004961048...</td>\n",
       "      <td>[222822.4, 0.0, 0.0, -222822.4, 21291520.0, 65...</td>\n",
       "      <td>POLYGON ((-95.55147058823529 29.44106158088235...</td>\n",
       "      <td>8_256_DBSCAN</td>\n",
       "      <td>8_256_fine_mask_blur</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>0.258362</td>\n",
       "      <td>9_post_resize_img</td>\n",
       "      <td>9_pre_resize_img</td>\n",
       "      <td>9_post_full_img</td>\n",
       "      <td>9_pre_full_img</td>\n",
       "      <td>9_mask</td>\n",
       "      <td>9_resize_mask</td>\n",
       "      <td>9_256_fine_mask</td>\n",
       "      <td>3002220.tif</td>\n",
       "      <td>wet</td>\n",
       "      <td>(POLYGON ((-95.55013713181758 29.4416478845277...</td>\n",
       "      <td>[222822.4, 0.0, 0.0, -222822.4, 21291008.0, 65...</td>\n",
       "      <td>POLYGON ((-95.54917279411765 29.44106158088235...</td>\n",
       "      <td>9_256_DBSCAN</td>\n",
       "      <td>9_256_fine_mask_blur</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>10</td>\n",
       "      <td>0.0128899</td>\n",
       "      <td>10_post_resize_img</td>\n",
       "      <td>10_pre_resize_img</td>\n",
       "      <td>10_post_full_img</td>\n",
       "      <td>10_pre_full_img</td>\n",
       "      <td>10_mask</td>\n",
       "      <td>10_resize_mask</td>\n",
       "      <td>10_256_fine_mask</td>\n",
       "      <td>3002220.tif</td>\n",
       "      <td>wet</td>\n",
       "      <td>POLYGON ((-95.54917279411765 29.44318860710831...</td>\n",
       "      <td>[222822.4, 0.0, 0.0, -222822.4, 21290496.0, 65...</td>\n",
       "      <td>POLYGON ((-95.546875 29.44106158088235, -95.54...</td>\n",
       "      <td>10_256_DBSCAN</td>\n",
       "      <td>10_256_fine_mask_blur</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>11</td>\n",
       "      <td>0.0711288</td>\n",
       "      <td>11_post_resize_img</td>\n",
       "      <td>11_pre_resize_img</td>\n",
       "      <td>11_post_full_img</td>\n",
       "      <td>11_pre_full_img</td>\n",
       "      <td>11_mask</td>\n",
       "      <td>11_resize_mask</td>\n",
       "      <td>11_256_fine_mask</td>\n",
       "      <td>3002220.tif</td>\n",
       "      <td>wet</td>\n",
       "      <td>(POLYGON ((-95.54232876244808 29.4410615808823...</td>\n",
       "      <td>[222822.4, 0.0, 0.0, -222822.4, 21289472.0, 65...</td>\n",
       "      <td>POLYGON ((-95.54227941176471 29.44106158088235...</td>\n",
       "      <td>11_256_DBSCAN</td>\n",
       "      <td>11_256_fine_mask_blur</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>12</td>\n",
       "      <td>0.0893288</td>\n",
       "      <td>12_post_resize_img</td>\n",
       "      <td>12_pre_resize_img</td>\n",
       "      <td>12_post_full_img</td>\n",
       "      <td>12_pre_full_img</td>\n",
       "      <td>12_mask</td>\n",
       "      <td>12_resize_mask</td>\n",
       "      <td>12_256_fine_mask</td>\n",
       "      <td>3002220.tif</td>\n",
       "      <td>wet</td>\n",
       "      <td>(POLYGON ((-95.54227941176471 29.4424747808023...</td>\n",
       "      <td>[222822.4, 0.0, 0.0, -222822.4, 21288960.0, 65...</td>\n",
       "      <td>POLYGON ((-95.53998161764706 29.44106158088235...</td>\n",
       "      <td>12_256_DBSCAN</td>\n",
       "      <td>12_256_fine_mask_blur</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>13</td>\n",
       "      <td>0.0591354</td>\n",
       "      <td>13_post_resize_img</td>\n",
       "      <td>13_pre_resize_img</td>\n",
       "      <td>13_post_full_img</td>\n",
       "      <td>13_pre_full_img</td>\n",
       "      <td>13_mask</td>\n",
       "      <td>13_resize_mask</td>\n",
       "      <td>13_256_fine_mask</td>\n",
       "      <td>3002220.tif</td>\n",
       "      <td>wet</td>\n",
       "      <td>(POLYGON ((-95.53810357449233 29.4421034503057...</td>\n",
       "      <td>[222822.4, 0.0, 0.0, -222822.4, 21288448.0, 65...</td>\n",
       "      <td>POLYGON ((-95.53768382352941 29.44106158088235...</td>\n",
       "      <td>13_256_DBSCAN</td>\n",
       "      <td>13_256_fine_mask_blur</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>14</td>\n",
       "      <td>0.0279617</td>\n",
       "      <td>14_post_resize_img</td>\n",
       "      <td>14_pre_resize_img</td>\n",
       "      <td>14_post_full_img</td>\n",
       "      <td>14_pre_full_img</td>\n",
       "      <td>14_mask</td>\n",
       "      <td>14_resize_mask</td>\n",
       "      <td>14_256_fine_mask</td>\n",
       "      <td>3020201.tif</td>\n",
       "      <td>dry</td>\n",
       "      <td>POLYGON ((-95.49067773698981 29.92036211127573...</td>\n",
       "      <td>[222822.4, 0.0, 0.0, -222822.4, 21277568.0, 66...</td>\n",
       "      <td>POLYGON ((-95.48885569852941 29.92015165441176...</td>\n",
       "      <td>14_256_DBSCAN</td>\n",
       "      <td>14_256_fine_mask_blur</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>15</td>\n",
       "      <td>0.0227242</td>\n",
       "      <td>15_post_resize_img</td>\n",
       "      <td>15_pre_resize_img</td>\n",
       "      <td>15_post_full_img</td>\n",
       "      <td>15_pre_full_img</td>\n",
       "      <td>15_mask</td>\n",
       "      <td>15_resize_mask</td>\n",
       "      <td>15_256_fine_mask</td>\n",
       "      <td>3020201.tif</td>\n",
       "      <td>dry</td>\n",
       "      <td>POLYGON ((-95.48974122587842 29.91913950177231...</td>\n",
       "      <td>[222822.4, 0.0, 0.0, -222822.4, 21277568.0, 66...</td>\n",
       "      <td>POLYGON ((-95.48885569852941 29.91785386029412...</td>\n",
       "      <td>15_256_DBSCAN</td>\n",
       "      <td>15_256_fine_mask_blur</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>16</td>\n",
       "      <td>0.0239563</td>\n",
       "      <td>16_post_resize_img</td>\n",
       "      <td>16_pre_resize_img</td>\n",
       "      <td>16_post_full_img</td>\n",
       "      <td>16_pre_full_img</td>\n",
       "      <td>16_mask</td>\n",
       "      <td>16_resize_mask</td>\n",
       "      <td>16_256_fine_mask</td>\n",
       "      <td>3020201.tif</td>\n",
       "      <td>wet</td>\n",
       "      <td>POLYGON ((-95.4894728167227 29.91615656045376,...</td>\n",
       "      <td>[222822.4, 0.0, 0.0, -222822.4, 21277568.0, 66...</td>\n",
       "      <td>POLYGON ((-95.48885569852941 29.91555606617647...</td>\n",
       "      <td>16_256_DBSCAN</td>\n",
       "      <td>16_256_fine_mask_blur</td>\n",
       "      <td>True</td>\n",
       "      <td>u</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>17</td>\n",
       "      <td>0.0269051</td>\n",
       "      <td>17_post_resize_img</td>\n",
       "      <td>17_pre_resize_img</td>\n",
       "      <td>17_post_full_img</td>\n",
       "      <td>17_pre_full_img</td>\n",
       "      <td>17_mask</td>\n",
       "      <td>17_resize_mask</td>\n",
       "      <td>17_256_fine_mask</td>\n",
       "      <td>3020201.tif</td>\n",
       "      <td>wet</td>\n",
       "      <td>POLYGON ((-95.4892905093979 29.90883091281986,...</td>\n",
       "      <td>[222822.4, 0.0, 0.0, -222822.4, 21277568.0, 66...</td>\n",
       "      <td>POLYGON ((-95.48885569852941 29.90866268382353...</td>\n",
       "      <td>17_256_DBSCAN</td>\n",
       "      <td>17_256_fine_mask_blur</td>\n",
       "      <td>True</td>\n",
       "      <td>f</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>18</td>\n",
       "      <td>0.00780869</td>\n",
       "      <td>18_post_resize_img</td>\n",
       "      <td>18_pre_resize_img</td>\n",
       "      <td>18_post_full_img</td>\n",
       "      <td>18_pre_full_img</td>\n",
       "      <td>18_mask</td>\n",
       "      <td>18_resize_mask</td>\n",
       "      <td>18_256_fine_mask</td>\n",
       "      <td>3020201.tif</td>\n",
       "      <td>dry</td>\n",
       "      <td>POLYGON ((-95.49115349264706 29.89205154933227...</td>\n",
       "      <td>[222822.4, 0.0, 0.0, -222822.4, 21277568.0, 66...</td>\n",
       "      <td>POLYGON ((-95.48885569852941 29.89028033088235...</td>\n",
       "      <td>18_256_DBSCAN</td>\n",
       "      <td>18_256_fine_mask_blur</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>19</td>\n",
       "      <td>0.0371819</td>\n",
       "      <td>19_post_resize_img</td>\n",
       "      <td>19_pre_resize_img</td>\n",
       "      <td>19_post_full_img</td>\n",
       "      <td>19_pre_full_img</td>\n",
       "      <td>19_mask</td>\n",
       "      <td>19_resize_mask</td>\n",
       "      <td>19_256_fine_mask</td>\n",
       "      <td>3020201.tif</td>\n",
       "      <td>wet</td>\n",
       "      <td>POLYGON ((-95.48690993951742 29.94320979637846...</td>\n",
       "      <td>[222822.4, 0.0, 0.0, -222822.4, 21277056.0, 66...</td>\n",
       "      <td>POLYGON ((-95.48655790441177 29.94312959558824...</td>\n",
       "      <td>19_256_DBSCAN</td>\n",
       "      <td>19_256_fine_mask_blur</td>\n",
       "      <td>True</td>\n",
       "      <td>f</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   tile_no     flooded        post_resized        pre_resized  \\\n",
       "0        0     0.80323   0_post_resize_img   0_pre_resize_img   \n",
       "1        1      0.8787   1_post_resize_img   1_pre_resize_img   \n",
       "2        2    0.864578   2_post_resize_img   2_pre_resize_img   \n",
       "3        3    0.576237   3_post_resize_img   3_pre_resize_img   \n",
       "4        4     0.59251   4_post_resize_img   4_pre_resize_img   \n",
       "5        5    0.664898   5_post_resize_img   5_pre_resize_img   \n",
       "6        6    0.901104   6_post_resize_img   6_pre_resize_img   \n",
       "7        7     0.53014   7_post_resize_img   7_pre_resize_img   \n",
       "8        8    0.448421   8_post_resize_img   8_pre_resize_img   \n",
       "9        9    0.258362   9_post_resize_img   9_pre_resize_img   \n",
       "10      10   0.0128899  10_post_resize_img  10_pre_resize_img   \n",
       "11      11   0.0711288  11_post_resize_img  11_pre_resize_img   \n",
       "12      12   0.0893288  12_post_resize_img  12_pre_resize_img   \n",
       "13      13   0.0591354  13_post_resize_img  13_pre_resize_img   \n",
       "14      14   0.0279617  14_post_resize_img  14_pre_resize_img   \n",
       "15      15   0.0227242  15_post_resize_img  15_pre_resize_img   \n",
       "16      16   0.0239563  16_post_resize_img  16_pre_resize_img   \n",
       "17      17   0.0269051  17_post_resize_img  17_pre_resize_img   \n",
       "18      18  0.00780869  18_post_resize_img  18_pre_resize_img   \n",
       "19      19   0.0371819  19_post_resize_img  19_pre_resize_img   \n",
       "\n",
       "           post_full         pre_full course_mask_full course_mask_resized  \\\n",
       "0    0_post_full_img   0_pre_full_img           0_mask       0_resize_mask   \n",
       "1    1_post_full_img   1_pre_full_img           1_mask       1_resize_mask   \n",
       "2    2_post_full_img   2_pre_full_img           2_mask       2_resize_mask   \n",
       "3    3_post_full_img   3_pre_full_img           3_mask       3_resize_mask   \n",
       "4    4_post_full_img   4_pre_full_img           4_mask       4_resize_mask   \n",
       "5    5_post_full_img   5_pre_full_img           5_mask       5_resize_mask   \n",
       "6    6_post_full_img   6_pre_full_img           6_mask       6_resize_mask   \n",
       "7    7_post_full_img   7_pre_full_img           7_mask       7_resize_mask   \n",
       "8    8_post_full_img   8_pre_full_img           8_mask       8_resize_mask   \n",
       "9    9_post_full_img   9_pre_full_img           9_mask       9_resize_mask   \n",
       "10  10_post_full_img  10_pre_full_img          10_mask      10_resize_mask   \n",
       "11  11_post_full_img  11_pre_full_img          11_mask      11_resize_mask   \n",
       "12  12_post_full_img  12_pre_full_img          12_mask      12_resize_mask   \n",
       "13  13_post_full_img  13_pre_full_img          13_mask      13_resize_mask   \n",
       "14  14_post_full_img  14_pre_full_img          14_mask      14_resize_mask   \n",
       "15  15_post_full_img  15_pre_full_img          15_mask      15_resize_mask   \n",
       "16  16_post_full_img  16_pre_full_img          16_mask      16_resize_mask   \n",
       "17  17_post_full_img  17_pre_full_img          17_mask      17_resize_mask   \n",
       "18  18_post_full_img  18_pre_full_img          18_mask      18_resize_mask   \n",
       "19  19_post_full_img  19_pre_full_img          19_mask      19_resize_mask   \n",
       "\n",
       "   fine_mask_filename    footprint dry_or_wet  \\\n",
       "0     0_256_fine_mask  3002220.tif        wet   \n",
       "1     1_256_fine_mask  3002220.tif        wet   \n",
       "2     2_256_fine_mask  3002220.tif        wet   \n",
       "3     3_256_fine_mask  3002220.tif        wet   \n",
       "4     4_256_fine_mask  3002220.tif        wet   \n",
       "5     5_256_fine_mask  3002220.tif        wet   \n",
       "6     6_256_fine_mask  3002220.tif        wet   \n",
       "7     7_256_fine_mask  3002220.tif        wet   \n",
       "8     8_256_fine_mask  3002220.tif        wet   \n",
       "9     9_256_fine_mask  3002220.tif        wet   \n",
       "10   10_256_fine_mask  3002220.tif        wet   \n",
       "11   11_256_fine_mask  3002220.tif        wet   \n",
       "12   12_256_fine_mask  3002220.tif        wet   \n",
       "13   13_256_fine_mask  3002220.tif        wet   \n",
       "14   14_256_fine_mask  3020201.tif        dry   \n",
       "15   15_256_fine_mask  3020201.tif        dry   \n",
       "16   16_256_fine_mask  3020201.tif        wet   \n",
       "17   17_256_fine_mask  3020201.tif        wet   \n",
       "18   18_256_fine_mask  3020201.tif        dry   \n",
       "19   19_256_fine_mask  3020201.tif        wet   \n",
       "\n",
       "                                            mask_poly  \\\n",
       "0   (POLYGON ((-95.57181511210993 29.4410615808823...   \n",
       "1   POLYGON ((-95.56764981800494 29.44136752727207...   \n",
       "2   (POLYGON ((-95.5654521647227 29.44106158088235...   \n",
       "3   (POLYGON ((-95.56505909959365 29.443359375, -9...   \n",
       "4   (POLYGON ((-95.56178077548428 29.4410615808823...   \n",
       "5   (POLYGON ((-95.56066176470588 29.4432499281638...   \n",
       "6   POLYGON ((-95.55606617647059 29.44209793185462...   \n",
       "7   (POLYGON ((-95.55376838235294 29.4431568755186...   \n",
       "8   (POLYGON ((-95.55376838235294 29.4431004961048...   \n",
       "9   (POLYGON ((-95.55013713181758 29.4416478845277...   \n",
       "10  POLYGON ((-95.54917279411765 29.44318860710831...   \n",
       "11  (POLYGON ((-95.54232876244808 29.4410615808823...   \n",
       "12  (POLYGON ((-95.54227941176471 29.4424747808023...   \n",
       "13  (POLYGON ((-95.53810357449233 29.4421034503057...   \n",
       "14  POLYGON ((-95.49067773698981 29.92036211127573...   \n",
       "15  POLYGON ((-95.48974122587842 29.91913950177231...   \n",
       "16  POLYGON ((-95.4894728167227 29.91615656045376,...   \n",
       "17  POLYGON ((-95.4892905093979 29.90883091281986,...   \n",
       "18  POLYGON ((-95.49115349264706 29.89205154933227...   \n",
       "19  POLYGON ((-95.48690993951742 29.94320979637846...   \n",
       "\n",
       "                                       tile_transform  \\\n",
       "0   [222822.4, 0.0, 0.0, -222822.4, 21295616.0, 65...   \n",
       "1   [222822.4, 0.0, 0.0, -222822.4, 21295104.0, 65...   \n",
       "2   [222822.4, 0.0, 0.0, -222822.4, 21294592.0, 65...   \n",
       "3   [222822.4, 0.0, 0.0, -222822.4, 21294080.0, 65...   \n",
       "4   [222822.4, 0.0, 0.0, -222822.4, 21293568.0, 65...   \n",
       "5   [222822.4, 0.0, 0.0, -222822.4, 21293056.0, 65...   \n",
       "6   [222822.4, 0.0, 0.0, -222822.4, 21292544.0, 65...   \n",
       "7   [222822.4, 0.0, 0.0, -222822.4, 21292032.0, 65...   \n",
       "8   [222822.4, 0.0, 0.0, -222822.4, 21291520.0, 65...   \n",
       "9   [222822.4, 0.0, 0.0, -222822.4, 21291008.0, 65...   \n",
       "10  [222822.4, 0.0, 0.0, -222822.4, 21290496.0, 65...   \n",
       "11  [222822.4, 0.0, 0.0, -222822.4, 21289472.0, 65...   \n",
       "12  [222822.4, 0.0, 0.0, -222822.4, 21288960.0, 65...   \n",
       "13  [222822.4, 0.0, 0.0, -222822.4, 21288448.0, 65...   \n",
       "14  [222822.4, 0.0, 0.0, -222822.4, 21277568.0, 66...   \n",
       "15  [222822.4, 0.0, 0.0, -222822.4, 21277568.0, 66...   \n",
       "16  [222822.4, 0.0, 0.0, -222822.4, 21277568.0, 66...   \n",
       "17  [222822.4, 0.0, 0.0, -222822.4, 21277568.0, 66...   \n",
       "18  [222822.4, 0.0, 0.0, -222822.4, 21277568.0, 66...   \n",
       "19  [222822.4, 0.0, 0.0, -222822.4, 21277056.0, 66...   \n",
       "\n",
       "                                             geometry         DBScan  \\\n",
       "0   POLYGON ((-95.56985294117646 29.44106158088235...   0_256_DBSCAN   \n",
       "1   POLYGON ((-95.56755514705883 29.44106158088235...   1_256_DBSCAN   \n",
       "2   POLYGON ((-95.56525735294117 29.44106158088235...   2_256_DBSCAN   \n",
       "3   POLYGON ((-95.56295955882354 29.44106158088235...   3_256_DBSCAN   \n",
       "4   POLYGON ((-95.56066176470588 29.44106158088235...   4_256_DBSCAN   \n",
       "5   POLYGON ((-95.55836397058823 29.44106158088235...   5_256_DBSCAN   \n",
       "6   POLYGON ((-95.55606617647059 29.44106158088235...   6_256_DBSCAN   \n",
       "7   POLYGON ((-95.55376838235294 29.44106158088235...   7_256_DBSCAN   \n",
       "8   POLYGON ((-95.55147058823529 29.44106158088235...   8_256_DBSCAN   \n",
       "9   POLYGON ((-95.54917279411765 29.44106158088235...   9_256_DBSCAN   \n",
       "10  POLYGON ((-95.546875 29.44106158088235, -95.54...  10_256_DBSCAN   \n",
       "11  POLYGON ((-95.54227941176471 29.44106158088235...  11_256_DBSCAN   \n",
       "12  POLYGON ((-95.53998161764706 29.44106158088235...  12_256_DBSCAN   \n",
       "13  POLYGON ((-95.53768382352941 29.44106158088235...  13_256_DBSCAN   \n",
       "14  POLYGON ((-95.48885569852941 29.92015165441176...  14_256_DBSCAN   \n",
       "15  POLYGON ((-95.48885569852941 29.91785386029412...  15_256_DBSCAN   \n",
       "16  POLYGON ((-95.48885569852941 29.91555606617647...  16_256_DBSCAN   \n",
       "17  POLYGON ((-95.48885569852941 29.90866268382353...  17_256_DBSCAN   \n",
       "18  POLYGON ((-95.48885569852941 29.89028033088235...  18_256_DBSCAN   \n",
       "19  POLYGON ((-95.48655790441177 29.94312959558824...  19_256_DBSCAN   \n",
       "\n",
       "             DBScan_gauss bad_image problem verified  \n",
       "0    0_256_fine_mask_blur      None    None    False  \n",
       "1    1_256_fine_mask_blur      None    None     True  \n",
       "2    2_256_fine_mask_blur      None    None     True  \n",
       "3    3_256_fine_mask_blur      True       c     True  \n",
       "4    4_256_fine_mask_blur      True       c     True  \n",
       "5    5_256_fine_mask_blur      True       c     True  \n",
       "6    6_256_fine_mask_blur      True       c     True  \n",
       "7    7_256_fine_mask_blur      True       c     True  \n",
       "8    8_256_fine_mask_blur      None    None     True  \n",
       "9    9_256_fine_mask_blur      None    None     True  \n",
       "10  10_256_fine_mask_blur      None    None     True  \n",
       "11  11_256_fine_mask_blur      None    None     True  \n",
       "12  12_256_fine_mask_blur      None    None     True  \n",
       "13  13_256_fine_mask_blur      None    None     True  \n",
       "14  14_256_fine_mask_blur      None    None     True  \n",
       "15  15_256_fine_mask_blur      None    None     True  \n",
       "16  16_256_fine_mask_blur      True       u     True  \n",
       "17  17_256_fine_mask_blur      True       f     True  \n",
       "18  18_256_fine_mask_blur      None    None     True  \n",
       "19  19_256_fine_mask_blur      True       f     True  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_dir = \"/home/ubuntu/data/TX_paired/\"\n",
    "\n",
    "geo_df = pickle.load( open( output_dir+\"GeoDataFrame_fine_turked.pickled\", \"rb\" ))\n",
    "geo_df.rename(columns = {'post-storm_full':'post_resized','pre-storm_full':'pre_resized','post-storm_resized':'post_full','pre-storm_resized':'pre_full'}, inplace=True)\n",
    "geo_df.set_index(\"tile_no\")\n",
    "geo_df.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "INPUT_SIZE = 256\n",
    "\n",
    "#helper functions:\n",
    "def scale_bands(img, lower_pct = 1, upper_pct = 99):\n",
    "    \"\"\"Rescale the bands of a multichannel image for display\"\"\"\n",
    "    img_scaled = np.zeros(img.shape, np.uint8)\n",
    "    for i in range(img.shape[2]):\n",
    "        band = img[:, :, i]\n",
    "        lower, upper = np.percentile(band, [lower_pct, upper_pct])\n",
    "        band = (band - lower) / (upper - lower) * 255\n",
    "        img_scaled[:, :, i] = np.clip(band, 0, 255).astype(np.uint8)\n",
    "    return img_scaled\n",
    "\n",
    "def resize(image, new_shape):\n",
    "    #img_resized = np.zeros(new_shape+(img.shape[2],)).astype('float32')\n",
    "    #for i in range(img.shape[2]):\n",
    "    #    img_resized[:, :, i] = imresize(img[:, :, i], new_shape, interp='bicubic')\n",
    "    #img_resized = cv2.resize(img,dsize=(new_shape,new_shape))\n",
    "    \n",
    "    r = new_shape / (1.0*image.shape[1])\n",
    "    dim = (new_shape, int(image.shape[0] * r))\n",
    "    \n",
    "    if image.dtype == \"int64\": image = image.astype('float32')   #crashes if given 0 and 1 integers for some reason\n",
    "    # perform the actual resizing of the image and show it\n",
    "    img_resized = cv2.resize(image, dim, interpolation = cv2.INTER_AREA)\n",
    "    \n",
    "    #make training masks the proper dimensionality if only 2-D\n",
    "    if len(image.shape) == 2: img_resized = np.expand_dims(img_resized,axis=2)\n",
    "\n",
    "    return img_resized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#identify a training/testing set"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## make test/train split\n",
    "#### assumes image and mask files are of identical number and none are missing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tile_no</th>\n",
       "      <th>flooded</th>\n",
       "      <th>post_resized</th>\n",
       "      <th>pre_resized</th>\n",
       "      <th>post_full</th>\n",
       "      <th>pre_full</th>\n",
       "      <th>course_mask_full</th>\n",
       "      <th>course_mask_resized</th>\n",
       "      <th>fine_mask_filename</th>\n",
       "      <th>footprint</th>\n",
       "      <th>dry_or_wet</th>\n",
       "      <th>mask_poly</th>\n",
       "      <th>tile_transform</th>\n",
       "      <th>geometry</th>\n",
       "      <th>DBScan</th>\n",
       "      <th>DBScan_gauss</th>\n",
       "      <th>bad_image</th>\n",
       "      <th>problem</th>\n",
       "      <th>verified</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.80323</td>\n",
       "      <td>0_post_resize_img</td>\n",
       "      <td>0_pre_resize_img</td>\n",
       "      <td>0_post_full_img</td>\n",
       "      <td>0_pre_full_img</td>\n",
       "      <td>0_mask</td>\n",
       "      <td>0_resize_mask</td>\n",
       "      <td>0_256_fine_mask</td>\n",
       "      <td>3002220.tif</td>\n",
       "      <td>wet</td>\n",
       "      <td>(POLYGON ((-95.57181511210993 29.4410615808823...</td>\n",
       "      <td>[222822.4, 0.0, 0.0, -222822.4, 21295616.0, 65...</td>\n",
       "      <td>POLYGON ((-95.56985294117646 29.44106158088235...</td>\n",
       "      <td>0_256_DBSCAN</td>\n",
       "      <td>0_256_fine_mask_blur</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  tile_no  flooded       post_resized       pre_resized        post_full  \\\n",
       "0       0  0.80323  0_post_resize_img  0_pre_resize_img  0_post_full_img   \n",
       "\n",
       "         pre_full course_mask_full course_mask_resized fine_mask_filename  \\\n",
       "0  0_pre_full_img           0_mask       0_resize_mask    0_256_fine_mask   \n",
       "\n",
       "     footprint dry_or_wet                                          mask_poly  \\\n",
       "0  3002220.tif        wet  (POLYGON ((-95.57181511210993 29.4410615808823...   \n",
       "\n",
       "                                      tile_transform  \\\n",
       "0  [222822.4, 0.0, 0.0, -222822.4, 21295616.0, 65...   \n",
       "\n",
       "                                            geometry        DBScan  \\\n",
       "0  POLYGON ((-95.56985294117646 29.44106158088235...  0_256_DBSCAN   \n",
       "\n",
       "           DBScan_gauss bad_image problem verified  \n",
       "0  0_256_fine_mask_blur      None    None    False  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "geo_df.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1278"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#only use 'good' and verified files\n",
    "good_geo_df = geo_df[geo_df.bad_image != True]\n",
    "good_geo_df = good_geo_df[good_geo_df.verified == True]\n",
    "#good_geo_df = good_geo_df[good_geo_df.tile_no <200]  #for testing with a smaller (faster) dataset\n",
    "len(good_geo_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#note, these are the 256 resized images\n",
    "image_files = sorted([output_dir+file+'.npy' for file in good_geo_df[\"post_resized\"].values])  \n",
    "before_image_files = sorted([output_dir+file+'.npy' for file in good_geo_df[\"pre_resized\"].values])  \n",
    "mask_files = sorted([output_dir+file+'.npy' for file in good_geo_df[\"DBScan_gauss\"].values])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1278, 1278)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "len(image_files),len(mask_files)\n",
    "#min([len(image_files),len(mask_files)])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def TT_tile_no_split(good_geo_df,split=0.8):\n",
    "    test_no = np.random.choice(good_geo_df.tile_no.values,int((1-split)*len(good_geo_df)))\n",
    "    train_no = [num for num in good_geo_df.tile_no.values if num not in test_no]\n",
    "    return train_no, test_no"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_no, test_no = TT_tile_no_split(good_geo_df,split=0.8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1045, 255)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_no),len(test_no)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1393, 528, 1170, 192, 1739, 279, 1573, 901, 1146, 639, 1618, 1536,\n",
       "       1342, 964, 1524, 1608, 1486, 1243, 1526, 1449, 1709, 878, 1315, 78,\n",
       "       755, 1790, 889, 1764, 269, 1089, 41, 1715, 266, 269, 1615, 1679,\n",
       "       1585, 11, 1715, 1063, 1297, 1214, 1036, 1035, 1591, 1691, 672, 885,\n",
       "       1022, 850, 593, 1553, 1101, 881, 1516, 1789, 249, 1710, 1700, 1124,\n",
       "       789, 1391, 14, 1238, 1533, 1122, 1677, 1649, 1611, 670, 192, 1689,\n",
       "       1552, 956, 915, 1231, 1006, 1002, 1251, 702, 1288, 920, 1217, 1082,\n",
       "       1624, 430, 1302, 1456, 1774, 568, 192, 345, 438, 363, 1069, 1329,\n",
       "       195, 522, 1659, 1365, 1746, 438, 1439, 1350, 1554, 1740, 457, 1516,\n",
       "       726, 1726, 57, 1302, 1448, 1339, 535, 499, 1270, 1725, 1009, 1485,\n",
       "       1046, 97, 1213, 1411, 416, 615, 95, 1664, 628, 1126, 1579, 1325,\n",
       "       1372, 1107, 1233, 152, 1518, 997, 993, 615, 1450, 1174, 1635, 1024,\n",
       "       1250, 18, 566, 1665, 975, 1101, 79, 691, 1331, 1019, 999, 34, 1119,\n",
       "       788, 1776, 796, 627, 215, 1142, 1452, 1214, 647, 185, 795, 1628,\n",
       "       926, 185, 879, 1156, 435, 1265, 1333, 1377, 443, 923, 1343, 1309,\n",
       "       1513, 1785, 15, 1386, 1587, 67, 1350, 1197, 587, 100, 1327, 125,\n",
       "       1426, 1535, 961, 1167, 1795, 1729, 1510, 78, 1451, 901, 1711, 1734,\n",
       "       592, 1567, 1494, 1365, 1781, 1685, 1020, 536, 1664, 646, 766, 936,\n",
       "       688, 1181, 586, 959, 1336, 107, 1591, 1207, 1027, 920, 228, 972,\n",
       "       588, 1153, 1449, 190, 1701, 189, 1734, 31, 529, 1267, 1112, 934,\n",
       "       773, 1784, 670, 388, 760, 182, 1230, 964, 931, 1443, 504, 1752,\n",
       "       1079, 1099], dtype=object)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_no"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#write test and train tile_no lists to disk\n",
    "f = open('/home/ubuntu/Notebooks/test_train_filelists/Unn_train_tile_numbers_'+str(datetime.datetime.now())+'.txt', 'w')\n",
    "for item in train_no:f.write(\"%s\\n\" % item)\n",
    "f.close()\n",
    "\n",
    "f = open('/home/ubuntu/Notebooks/test_train_filelists/Unn_test_tile_numbers_'+str(datetime.datetime.now())+'.txt', 'w')\n",
    "for item in test_no:f.write(\"%s\\n\" % item)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### make memory-friendly generators instead of a large X,Y to hold in memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/ubuntu/data/TX_paired/2_post_resize_img.npy'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_dir+good_geo_df.post_resized[2]+'.npy'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def gen_train(good_geo_df,train_no):\n",
    "    indexes = copy.deepcopy(train_no)\n",
    "    while True:\n",
    "        np.random.shuffle(indexes)\n",
    "\n",
    "        for index in indexes:\n",
    "            Xpost = np.load(output_dir+good_geo_df.post_resized[index]+'.npy')\n",
    "            Xpre = np.load(output_dir+good_geo_df.pre_resized[index]+'.npy')\n",
    "            Xpre = Xpre.astype('float32')\n",
    "            Xpost = Xpost.astype('float32')\n",
    "            pre_mean = 92.36813   #taken from a central common image\n",
    "            post_mean = 92.21524   #much closer than expected... are these representative?\n",
    "            \n",
    "            Xdiff = Xpost/post_mean - Xpre/pre_mean\n",
    "            \n",
    "            Xpost = (Xpost-post_mean)/post_mean  #divide by their respective means (per footprint would be even better)\n",
    "            Xpre =  (Xpre-pre_mean)/pre_mean\n",
    "            \n",
    "            R,G,B = Xpost[:,:,0],Xpost[:,:,1],Xpost[:,:,2]\n",
    "            Xratios_post = np.stack([R/G-1,R/B-1,G/B-1,R/(G+B)-0.5,G/(R+B)-0.5,B/(R+G)-0.5],axis=2)\n",
    "            \n",
    "            R,G,B = Xpre[:,:,0],Xpre[:,:,1],Xpre[:,:,2]\n",
    "            Xratios_pre = np.stack([R/G-1,R/B-1,G/B-1,R/(G+B)-0.5,G/(R+B)-0.5,B/(R+G)-0.5],axis=2)\n",
    "            \n",
    "            #X = np.concatenate([Xpost,Xdiff,Xpre,Xratios_post,Xratios_pre],axis=2)\n",
    "            X = np.concatenate([Xpost-1,Xdiff-1,Xratios_post],axis=2)\n",
    "              \n",
    "            Y = np.load(output_dir+good_geo_df.DBScan_gauss[index]+'.npy')\n",
    "            Y = Y.astype('float32') #/ 255.\n",
    "            #add extra first dimension for tensorflow compatability\n",
    "            X = np.expand_dims(X,axis=0)\n",
    "            Y = np.expand_dims(Y,axis=0)\n",
    "            Y = np.expand_dims(Y,axis=3)\n",
    "            yield (X,Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def gen_test(good_geo_df,test_no):\n",
    "    indexes = copy.deepcopy(test_no)\n",
    "    while True:\n",
    "        np.random.shuffle(indexes)\n",
    "\n",
    "        for index in indexes:\n",
    "            Xpost = np.load(output_dir+good_geo_df.post_resized[index]+'.npy')\n",
    "            Xpre = np.load(output_dir+good_geo_df.pre_resized[index]+'.npy')\n",
    "            Xpre = Xpre.astype('float32')\n",
    "            Xpost = Xpost.astype('float32')\n",
    "            pre_mean = 92.36813   #taken from a central common image\n",
    "            post_mean = 92.21524   #much closer than expected... are these representative?\n",
    "            \n",
    "            Xdiff = Xpost/post_mean - Xpre/pre_mean\n",
    "            \n",
    "            Xpost = (Xpost-post_mean)/post_mean  #divide by their respective means (per footprint would be even better)\n",
    "            Xpre =  (Xpre-pre_mean)/pre_mean\n",
    "            \n",
    "            R,G,B = Xpost[:,:,0],Xpost[:,:,1],Xpost[:,:,2]\n",
    "            Xratios_post = np.stack([R/G-1,R/B-1,G/B-1,R/(G+B)-0.5,G/(R+B)-0.5,B/(R+G)-0.5],axis=2)\n",
    "            \n",
    "            R,G,B = Xpre[:,:,0],Xpre[:,:,1],Xpre[:,:,2]\n",
    "            Xratios_pre = np.stack([R/G-1,R/B-1,G/B-1,R/(G+B)-0.5,G/(R+B)-0.5,B/(R+G)-0.5],axis=2)\n",
    "            \n",
    "            #X = np.concatenate([Xpost,Xdiff,Xpre,Xratios_post,Xratios_pre],axis=2)\n",
    "            X = np.concatenate([Xpost-1,Xdiff-1,Xratios_post],axis=2)\n",
    "              \n",
    "            Y = np.load(output_dir+good_geo_df.DBScan_gauss[index]+'.npy')\n",
    "            Y = Y.astype('float32') #/ 255.\n",
    "            #add extra first dimension for tensorflow compatability\n",
    "            X = np.expand_dims(X,axis=0)\n",
    "            Y = np.expand_dims(Y,axis=0)\n",
    "            Y = np.expand_dims(Y,axis=3)\n",
    "            yield (X,Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# Source: https://github.com/jocicmarko/ultrasound-nerve-segmentation/blob/master/train.py\n",
    "\n",
    "import keras\n",
    "from keras import backend as K\n",
    "from keras.layers import Input, concatenate, Conv2D, MaxPooling2D, Conv2DTranspose, Dropout, BatchNormalization\n",
    "from keras.models import Model\n",
    "from keras.optimizers import Adam, Nadam, Adamax, RMSprop\n",
    "\n",
    "# Set network size params\n",
    "N_CLASSES = 1\n",
    "N_CHANNEL = 12\n",
    "\n",
    "#dropout rate\n",
    "out_per = 0.30\n",
    "\n",
    "# Define metrics\n",
    "smooth = 1.\n",
    "\n",
    "def dice_coef(y_true, y_pred):\n",
    "    y_true_f = K.flatten(y_true)\n",
    "    y_pred_f = K.flatten(y_pred)\n",
    "    intersection = K.sum(y_true_f * y_pred_f)\n",
    "    return (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n",
    "\n",
    "# Just put a negative sign in front of an accuracy metric to turn it into a loss to be minimized\n",
    "def dice_coef_loss(y_true, y_pred):\n",
    "    return -dice_coef(y_true, y_pred)\n",
    "\n",
    "def jacc_coef(y_true, y_pred):\n",
    "    intersection = K.sum(y_true * y_pred, axis=[0, -1, -2])\n",
    "    sum_ = K.sum(y_true + y_pred, axis=[0, -1, -2])\n",
    "    jac = (intersection + smooth) / (sum_ - intersection + smooth)\n",
    "    return K.mean(jac)\n",
    "\n",
    "def jacc_coef_loss(y_true, y_pred):\n",
    "    return -jacc_coef(y_true, y_pred)\n",
    "\n",
    "def jacc_coef_int(y_true, y_pred):\n",
    "    y_pred_pos = K.round(K.clip(y_pred, 0, 1))\n",
    "    intersection = K.sum(y_true * y_pred_pos, axis=[0, -1, -2])\n",
    "    sum_ = K.sum(y_true + y_pred, axis=[0, -1, -2])\n",
    "    jac = (intersection + smooth) / (sum_ - intersection + smooth)\n",
    "    return K.mean(jac)\n",
    "\n",
    "def get_unet(lr=0.00001):\n",
    "    inputs = Input((INPUT_SIZE, INPUT_SIZE, N_CHANNEL))\n",
    "    conv1 = Conv2D(32, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal')(inputs)\n",
    "    conv1 = Dropout(out_per)(conv1)\n",
    "    conv1 = Conv2D(32, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal')(conv1)\n",
    "    pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)\n",
    "\n",
    "    conv2 = Conv2D(64, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal')(pool1)\n",
    "    conv2 = Dropout(out_per)(conv2)\n",
    "    conv2 = Conv2D(64, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal')(conv2)\n",
    "    pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)\n",
    "\n",
    "    conv3 = Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal')(pool2)\n",
    "    conv3 = Dropout(out_per)(conv3)\n",
    "    conv3 = Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal')(conv3)\n",
    "    pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)\n",
    "\n",
    "    conv4 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal')(pool3)\n",
    "    conv4 = Dropout(out_per)(conv4)\n",
    "    conv4 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal')(conv4)\n",
    "    pool4 = MaxPooling2D(pool_size=(2, 2))(conv4)\n",
    "\n",
    "    conv5 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal')(pool4)\n",
    "    conv5 = Dropout(out_per)(conv5)\n",
    "    conv5 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal')(conv5)\n",
    "\n",
    "    up6 = concatenate([Conv2DTranspose(256, (2, 2), strides=(2, 2), padding='same')(conv5), conv4], axis=3)\n",
    "    conv6 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal')(up6)\n",
    "    conv6 = Dropout(out_per)(conv6)\n",
    "    conv6 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal')(conv6)\n",
    "\n",
    "    up7 = concatenate([Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same')(conv6), conv3], axis=3)\n",
    "    conv7 = Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal')(up7)\n",
    "    conv7 = Dropout(out_per)(conv7)\n",
    "    conv7 = Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal')(conv7)\n",
    "\n",
    "    up8 = concatenate([Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(conv7), conv2], axis=3)\n",
    "    conv8 = Conv2D(64, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal')(up8)\n",
    "    conv8 = Dropout(out_per)(conv8)\n",
    "    conv8 = Conv2D(64, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal')(conv8)\n",
    "\n",
    "    up9 = concatenate([Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(conv8), conv1], axis=3)\n",
    "    conv9 = Conv2D(32, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal')(up9)\n",
    "    conv9 = Dropout(out_per)(conv9)\n",
    "    conv9 = Conv2D(32, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal')(conv9)\n",
    "    conv9 = BatchNormalization(axis = 1)(conv9)\n",
    "    \n",
    "    conv10 = Conv2D(N_CLASSES, (1, 1), activation='sigmoid')(conv9)\n",
    "\n",
    "    model = Model(inputs=[inputs], outputs=[conv10])\n",
    "\n",
    "    # model.compile(optimizer=Adam(lr=lr), loss=jacc_coef_loss, metrics=[jacc_coef_int])\n",
    "    # model.compile(optimizer=Adam(lr=lr), loss='binary_crossentropy', metrics=[jacc_coef_int])\n",
    "    # model.compile(optimizer=Adam(lr=lr), loss='binary_crossentropy', metrics=[dice_coef])\n",
    "    model.compile(optimizer=Adam(lr=lr), loss=dice_coef_loss, metrics=[dice_coef, jacc_coef_int])\n",
    "\n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"training/testing set size:\",len(train_no),len(test_no))\n",
    "\n",
    "from keras.callbacks import LearningRateScheduler, ModelCheckpoint, ReduceLROnPlateau, TensorBoard\n",
    "# https://keras.io/callbacks/#reducelronplateau\n",
    "\n",
    "# This sets the number of training epochs (you should do a lot more than 5)\n",
    "NUM_EPOCHS = 500\n",
    "\n",
    "# Define callback to save model checkpoints\n",
    "if not os.path.exists('checkpoints'):\n",
    "    os.makedirs('checkpoints')\n",
    "model_checkpoint = ModelCheckpoint(os.path.join('checkpoints', 'weights.{epoch:02d}-{val_loss:.5f}.hdf5'), monitor='loss', save_best_only=True)\n",
    "\n",
    "# Define callback to reduce learning rate when learning stagnates\n",
    "# This won't actually kick in with only 5 training epochs, but I'm assuming you'll train for hundreds of epochs when you get serious about training this NN.\n",
    "reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3, epsilon=0.002, cooldown=2)\n",
    "\n",
    "# # Define rate scheduler callack (this is an alternative to ReduceLROnPlateau. There is no reason to use both.)\n",
    "# schedule = lambda epoch_i: 0.01*np.power(0.97, i)\n",
    "# schedule_lr = LearningRateScheduler(schedule)\n",
    "\n",
    "# TensorBoard visuluaziations... this stuff is so freaking cool\n",
    "tensorboard = TensorBoard(log_dir='/tmp/tboard_logs2', histogram_freq=0, write_graph=True, write_images=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Train the model\n",
    "model = get_unet(1e-4)\n",
    "#model.fit(X_train, Y_train, batch_size=32, epochs=NUM_EPOCHS, verbose=1, shuffle=True, callbacks=[model_checkpoint, reduce_lr, tensorboard], validation_data=(X_test, Y_test))\n",
    "hist = model.fit_generator(gen_train(good_geo_df,train_no), steps_per_epoch=len(train_no), epochs=NUM_EPOCHS, verbose=1, shuffle=True, \\\n",
    "                    callbacks=[model_checkpoint, reduce_lr, tensorboard], \\\n",
    "                    validation_data=gen_train(good_geo_df,test_no), validation_steps = len(test_no),\\\n",
    "                    )#workers=6,use_multiprocessing=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "hist.history.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "val_dice = hist.history['val_dice_coef']\n",
    "val_loss = hist.history['val_loss']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fig = plt.figure(1)\n",
    "\n",
    "# plot dice\n",
    "plt.subplot(211)\n",
    "plt.plot(val_dice)\n",
    "plt.title('Validation dice coefficient and loss')\n",
    "plt.ylabel('Dice coef value')\n",
    "\n",
    "# plot loss\n",
    "plt.subplot(212)\n",
    "plt.plot(val_loss, 'r')\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('Loss value')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}