notebooks/models/evaluate_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/py3.6/lib/python3.6/site-packages/cupy/core/fusion.py:659: FutureWarning: cupy.core.fusion is experimental. The interface can change in the future.\n",
      "  util.experimental('cupy.core.fusion')\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "sys.path.append(\"../../src/models\")\n",
    "\n",
    "from evaluate_model import eval_semantic_segmentation, calc_semantic_segmentation_confusion\n",
    "from segmentation import SegmentationModel as Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os import path\n",
    "from PIL import Image\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load trained model\n",
    "# Modify the the paths based on your trained model location if needed.\n",
    "mean = np.load(\"../../data/dataSplit/mean.npy\")\n",
    "model = Model(\"../../models/logs/model_iter_14878\", mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load test-set\n",
    "# Modify the the paths based on your data location if needed.\n",
    "with open( \"../../data/dataSplit/test.txt\") as f:\n",
    "    lines = f.readlines()\n",
    "\n",
    "test_set = []\n",
    "for line in lines:\n",
    "    line = line.rstrip()\n",
    "    test_set.append(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1388/1388 [02:20<00:00,  9.85it/s]\n"
     ]
    }
   ],
   "source": [
    "#  Evaluate model\n",
    "\n",
    "# Modify the the paths based on your raster image and building label location.\n",
    "image_dir = '../../data/processedBuildingLabels/3band'\n",
    "label_dir = '../../data/buildingLabelImages'\n",
    "\n",
    "gt_labels = []\n",
    "pred_labels = []\n",
    "\n",
    "for test_data in tqdm(test_set):\n",
    "    image_path = path.join(image_dir, test_data)\n",
    "    label_path = path.join(label_dir, test_data)\n",
    "    \n",
    "    image = np.array(Image.open(image_path))\n",
    "    label = np.array(Image.open(label_path))\n",
    "    \n",
    "    # Make gt_label, ground-truth building mask\n",
    "    h, w = label.shape\n",
    "    gt_label = np.zeros(shape=[h, w], dtype=np.int32) # 0: background\n",
    "    gt_label[label > 0] = 1 # 1: \"building\"\n",
    "    \n",
    "    # Make pred_label, predicted building mask\n",
    "    score = model.apply_segmentation(image)\n",
    "    pred_label = np.argmax(score, axis=0)\n",
    "    \n",
    "    gt_labels.append(gt_label)\n",
    "    pred_labels.append(pred_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'iou': array([0.95339559, 0.60201456]), 'miou': 0.7777050778994709, 'pixel_accuracy': 0.9564646763810029, 'class_accuracy': array([0.97648055, 0.74886715]), 'mean_class_accuracy': 0.8626738533691427}\n"
     ]
    }
   ],
   "source": [
    "result = eval_semantic_segmentation(pred_labels, gt_labels)\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IoU for class Building =  0.6020145619704008\n",
      "Accuracy for class Building =  0.7488671526505621\n"
     ]
    }
   ],
   "source": [
    "building_class = 1\n",
    "print(\"IoU for class Building = \", result['iou'][building_class])\n",
    "print(\"Accuracy for class Building = \", result['class_accuracy'][building_class])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "confusion:\n",
      "[[220150859   5302539]\n",
      " [  5458996  16278487]]\n",
      "\n",
      "precision:  0.75429625078993\n",
      "recall:  0.7488671526505621\n"
     ]
    }
   ],
   "source": [
    "confusion = calc_semantic_segmentation_confusion(pred_labels, gt_labels)\n",
    "print(\"confusion:\")\n",
    "print(confusion)\n",
    "\n",
    "tn = confusion[0][0]\n",
    "fp = confusion[0][1]\n",
    "tp = confusion[1][1]\n",
    "fn = confusion[1][0]\n",
    "\n",
    "precision = float(tp) / float(tp + fp)\n",
    "recall       = float(tp) / float(tp + fn)\n",
    "\n",
    "print()\n",
    "print(\"precision: \", precision)\n",
    "print(\"recall: \", recall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}