baseline-transfer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import torch\n",
    "import torchvision\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "import time\n",
    "\n",
    "from BaselineDataset import BaselineDataset\n",
    "from Model import EnDeWithPooling, EnDeConvLSTM_ws, SkipLSTMEnDe\n",
    "from torchvision import transforms\n",
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def saveTransformedImages(imageTensor):\n",
    "    to_pil = torchvision.transforms.ToPILImage()\n",
    "    im = to_pil(imageTensor)\n",
    "    mn, mx = np.min(im), np.max(im)\n",
    "    im = (im - mn) / (mx - mn)\n",
    "    print(im)\n",
    "    plt.imshow(im, cmap='gray')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plotTrajectory(xValsGT, yValsGT, xValsPred, yValsPred, xValsPredMulti, yValsPredMulti, seqLen, im_path, numFrames=None):\n",
    "    fig = plt.figure(figsize=(8, 8))\n",
    "    plt.plot(yValsGT, xValsGT, c='r', label='Ground Truth')\n",
    "    plt.plot(yValsPred, xValsPred, c='g', label='Prediction')\n",
    "    plt.plot(yValsPredMulti, xValsPredMulti, c='b', label='Multimodal Prediction', alpha=0.8)\n",
    "    axes = plt.gca()\n",
    "    axes.set_xlim([1, 512])\n",
    "    axes.set_ylim([1, 512])\n",
    "    plt.xlabel('X-Axis')\n",
    "    plt.ylabel('Y-Axis')\n",
    "    plt.legend(loc='upper right')\n",
    "    if numFrames == None:\n",
    "        plt.title('Trajectory')\n",
    "    else:\n",
    "        plot_title = 'Trajectory (' + str(numFrames // 10 - 2) + \"s)\"\n",
    "        plt.title(plot_title)\n",
    "    plt.savefig(im_path)\n",
    "    plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def heatmapAccuracy(outputMap, labelMap, thr=1.5):\n",
    "    pred = np.unravel_index(outputMap.argmax(), outputMap.shape)\n",
    "    gt = np.unravel_index(labelMap.argmax(), labelMap.shape)\n",
    "\n",
    "    dist = math.sqrt((pred[0] - gt[0]) ** 2 + (pred[1] - gt[1]) ** 2)\n",
    "    if dist <= thr:\n",
    "        return 1, dist, (pred[0], pred[1]), (gt[0], gt[1])\n",
    "    return 0, dist, (pred[0], pred[1]), (gt[0], gt[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def largest_indices(ary, n):\n",
    "    \"\"\"Returns the n largest indices from a numpy array.\"\"\"\n",
    "    flat = ary.flatten()\n",
    "    indices = np.argpartition(flat, -n)[-n:]\n",
    "    indices = indices[np.argsort(-flat[indices])]\n",
    "    return np.unravel_index(indices, ary.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def multiAccuracy(outputMap, labelMap, topK=5):\n",
    "    pred = largest_indices(outputMap, topK)\n",
    "    gt = np.unravel_index(labelMap.argmax(), labelMap.shape)\n",
    "    dist_arr = []\n",
    "    for i in range(len(pred[0])):\n",
    "        dist = math.sqrt((pred[0][i] - gt[0]) ** 2 + (pred[1][i] - gt[1]) ** 2)\n",
    "        dist_arr.append(dist)\n",
    "    \n",
    "    min_val = np.min(dist_arr)\n",
    "    min_idx = np.argmin(dist_arr)\n",
    "    within_radius = 0\n",
    "    if min_val <= 4:\n",
    "        within_radius = 1\n",
    "    return 0, min_val, (pred[0][min_idx], pred[1][min_idx]), (gt[0], gt[1]), within_radius"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda:0\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.set_default_tensor_type(torch.cuda.FloatTensor)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cityscapes Future Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint_path = os.path.join(\"/home/fbd/rrc/backup/test-baseline-transfer-0\", \"checkpoint_future.tar\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint = torch.load(checkpoint_path)\n",
    "model = SkipLSTMEnDe(activation=\"relu\", initType=\"default\", numChannels=4, imageHeight=256, imageWidth=256, batchnorm=False, softmax=False)\n",
    "model.load_state_dict(checkpoint[\"model_state_dict\"])\n",
    "model = model.cuda()\n",
    "model.convlstm = model.convlstm.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = \"/home/fbd/rrc/submission/INFER-datasets/cityscapes\"\n",
    "val_dir = os.path.join(data_dir, \"test.csv\")\n",
    "val_dataset = BaselineDataset(data_dir, height=256, width=256, train=False, infoPath=val_dir, augmentation=False, groundTruth=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "upsample_512 = torch.nn.Upsample(scale_factor=2, mode='bilinear')\n",
    "labelTransform = transforms.Compose([\n",
    "    transforms.ToTensor()\n",
    "])\n",
    "targetGTDir = os.path.join(data_dir, 'targetGT')\n",
    "valLoss1, valLoss2, valLoss3, valLoss4, valLoss = [], [], [], [], []\n",
    "futureFrames = 14\n",
    "topK = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fbd/anaconda3/envs/torchenv/lib/python3.5/site-packages/torch/nn/functional.py:1749: UserWarning: Default upsampling behavior when mode=bilinear is changed to align_corners=False since 0.4.0. Please specify align_corners=True if the old behavior is desired. See the documentation of nn.Upsample for details.\n",
      "  \"See the documentation of nn.Upsample for details.\".format(mode))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SeqNum: 1, KittiSeqNum: 0, VehicleNum: 3, numFrames: 30, loss: 4.437170643907619, len(seqLoss): 8\n",
      "SeqNum: 2, KittiSeqNum: 0, VehicleNum: 4, numFrames: 30, loss: 3.4176136863300393, len(seqLoss): 8\n",
      "SeqNum: 3, KittiSeqNum: 18, VehicleNum: 6, numFrames: 30, loss: 9.473552973931827, len(seqLoss): 8\n",
      "SeqNum: 4, KittiSeqNum: 18, VehicleNum: 7, numFrames: 30, loss: 4.715468606885709, len(seqLoss): 8\n",
      "SeqNum: 5, KittiSeqNum: 26, VehicleNum: 21, numFrames: 30, loss: 0.8017766952966369, len(seqLoss): 8\n",
      "SeqNum: 6, KittiSeqNum: 32, VehicleNum: 14, numFrames: 30, loss: 2.423145600891813, len(seqLoss): 8\n",
      "SeqNum: 7, KittiSeqNum: 42, VehicleNum: 15, numFrames: 30, loss: 6.089248944342616, len(seqLoss): 8\n",
      "SeqNum: 8, KittiSeqNum: 50, VehicleNum: 2, numFrames: 30, loss: 3.4582886740603396, len(seqLoss): 8\n",
      "SeqNum: 9, KittiSeqNum: 1, VehicleNum: 25, numFrames: 30, loss: 7.6653954699247935, len(seqLoss): 8\n",
      "SeqNum: 10, KittiSeqNum: 10, VehicleNum: 20, numFrames: 30, loss: 2.2519174652330793, len(seqLoss): 8\n",
      "SeqNum: 11, KittiSeqNum: 11, VehicleNum: 12, numFrames: 30, loss: 2.2568999858950685, len(seqLoss): 8\n",
      "SeqNum: 12, KittiSeqNum: 44, VehicleNum: 21, numFrames: 30, loss: 4.097843554630874, len(seqLoss): 8\n",
      "SeqNum: 13, KittiSeqNum: 52, VehicleNum: 8, numFrames: 30, loss: 8.661876980932563, len(seqLoss): 8\n",
      "SeqNum: 14, KittiSeqNum: 6, VehicleNum: 6, numFrames: 30, loss: 1.4045084971874737, len(seqLoss): 8\n",
      "SeqNum: 15, KittiSeqNum: 24, VehicleNum: 1, numFrames: 30, loss: 5.883883476483184, len(seqLoss): 8\n",
      "SeqNum: 16, KittiSeqNum: 30, VehicleNum: 6, numFrames: 30, loss: 3.13684119535434, len(seqLoss): 8\n",
      "SeqNum: 17, KittiSeqNum: 30, VehicleNum: 7, numFrames: 30, loss: 1.0295084971874737, len(seqLoss): 8\n",
      "SeqNum: 18, KittiSeqNum: 37, VehicleNum: 5, numFrames: 30, loss: 0.8607936896715843, len(seqLoss): 8\n",
      "SeqNum: 19, KittiSeqNum: 48, VehicleNum: 3, numFrames: 30, loss: 2.278346595301795, len(seqLoss): 8\n",
      "SeqNum: 20, KittiSeqNum: 17, VehicleNum: 7, numFrames: 30, loss: 4.228854443848749, len(seqLoss): 8\n",
      "SeqNum: 21, KittiSeqNum: 17, VehicleNum: 8, numFrames: 30, loss: 6.90286260416803, len(seqLoss): 8\n",
      "SeqNum: 22, KittiSeqNum: 17, VehicleNum: 9, numFrames: 30, loss: 0.875, len(seqLoss): 8\n",
      "SeqNum: 23, KittiSeqNum: 25, VehicleNum: 45, numFrames: 30, loss: 6.506883216716823, len(seqLoss): 8\n",
      "SeqNum: 24, KittiSeqNum: 38, VehicleNum: 13, numFrames: 30, loss: 4.859976999142769, len(seqLoss): 8\n",
      "SeqNum: 25, KittiSeqNum: 38, VehicleNum: 14, numFrames: 30, loss: 7.196394433574046, len(seqLoss): 8\n",
      "SeqNum: 26, KittiSeqNum: 49, VehicleNum: 17, numFrames: 30, loss: 1.3006147934109582, len(seqLoss): 8\n"
     ]
    }
   ],
   "source": [
    "debug, prevOut, state = True, None, None\n",
    "xValsGT, yValsGT, xValsPred, yValsPred = [], [], [], []\n",
    "seqLoss, seqVals = [], []\n",
    "seqNum, seqLen = 0, 0\n",
    "\n",
    "start_time = time.time()\n",
    "model.eval()\n",
    "\n",
    "for i in range(len(val_dataset)):\n",
    "    grid, kittiSeqNum, vehicleId, frame1, frame2, endOfSequence, offset, numFrames, augmentation = val_dataset[i]\n",
    "    \n",
    "    if endOfSequence is False:\n",
    "        if int(offset) % 2 == 0:\n",
    "            continue\n",
    "\n",
    "    # The Last Channel is the target frame and first n - 1 are source frames\n",
    "    inp = grid[:-1, :].unsqueeze(0).to(device)\n",
    "    currLabel = grid[-1:, :].unsqueeze(0).to(device)\n",
    "    \n",
    "    if offset >= futureFrames:\n",
    "        new_inp = inp.clone().squeeze(0)\n",
    "        mn, mx = torch.min(prevOut), torch.max(prevOut)\n",
    "        prevOut = (prevOut - mn) / (mx - mn)\n",
    "        new_inp[0] = prevOut\n",
    "        inp = new_inp.unsqueeze(0).cuda()\n",
    "\n",
    "    # Forward the input and obtain the result\n",
    "    out = model.forward(inp, state)\n",
    "    state = (model.h, model.c, model.h1, model.c1, model.h2, model.c2)\n",
    "    currOutputMap = out.clone()\n",
    "    newOutputMap = upsample_512(currOutputMap)\n",
    "    nextTargetImg = Image.open(os.path.join(targetGTDir, str(kittiSeqNum).zfill(4), \n",
    "                                            str(frame2).zfill(6), str(vehicleId).zfill(6) + '.png'))\n",
    "    \n",
    "    nextTargetTensor = labelTransform(nextTargetImg).unsqueeze(0)\n",
    "    \n",
    "    prevOut = currOutputMap.detach().cpu().squeeze(0).squeeze(0)\n",
    "    currOutputMap = currOutputMap.detach().cpu().numpy().squeeze(0).squeeze(0)\n",
    "    currLabel = currLabel.detach().cpu().numpy().squeeze(0).squeeze(0)\n",
    "    _, dist, predCoordinates, gtCoordinates = heatmapAccuracy(currOutputMap, currLabel)\n",
    "    \n",
    "    # Upsampled outputs and inputs\n",
    "    currOutputMap1 = newOutputMap.detach().cpu().numpy().squeeze(0).squeeze(0)\n",
    "    currLabel1 = nextTargetTensor.detach().cpu().numpy().squeeze(0).squeeze(0)\n",
    "    \n",
    "    _, dist1, predCoordinates1, gtCoordinates1 = heatmapAccuracy(currOutputMap1, currLabel1)\n",
    "    _, dist2, predCoordinates2, gtCoordinates2, within_radius = multiAccuracy(currOutputMap1, currLabel1, topK=topK)    \n",
    "    \n",
    "    if offset >= futureFrames:\n",
    "        seqLoss.append(dist2)\n",
    "\n",
    "    seqLen += 1\n",
    "    xValsGT.append(gtCoordinates1[0])\n",
    "    yValsGT.append(gtCoordinates1[1])\n",
    "    xValsPred.append(predCoordinates1[0])\n",
    "    yValsPred.append(predCoordinates1[1])\n",
    "    \n",
    "    if endOfSequence:\n",
    "        seqVals.append(seqLen)\n",
    "        xValsGT, yValsGT, xValsPred, yValsPred = [], [], [], []\n",
    "        seqNum += 1\n",
    "        state = None\n",
    "        valLoss.append(np.mean(seqLoss))\n",
    "        print(\"SeqNum: {}, KittiSeqNum: {}, VehicleNum: {}, numFrames: {}, loss: {}, len(seqLoss): {}\".format(seqNum, kittiSeqNum, vehicleId, numFrames, np.mean(seqLoss), len(seqLoss)))\n",
    "        seqLoss = []\n",
    "\n",
    "end_time = time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Avg Loss: 4.0851795278580845\n",
      "Avg Loss in m: 1.0212948819645211\n",
      "Num Seq: 26\n"
     ]
    }
   ],
   "source": [
    "print(\"Avg Loss: {}\".format(np.mean(valLoss)))\n",
    "print(\"Avg Loss in m: {}\".format(np.mean(valLoss) * 0.25))\n",
    "print(\"Num Seq: {}\".format(len(valLoss)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}