GTSRB_Neuron2ProjectionCoverage_TestGen.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neuron 2-projection on-off activation coverage (2) - German Traffic Sign Recognition Benchmark (GTSRB)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Put these at the top of every notebook, to get automatic reloading and inline plotting\n",
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy.random\n",
    "# Fix the number for repeatability (we have also stored the trained model)\n",
    "numpy.random.seed(42)\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "\n",
    "# Device configuration\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Start by accessing the root folder where internally, subfolders are images with folder name being their classified result.\n",
    "\n",
    "The dataset should be available at the following site \n",
    "http://benchmark.ini.rub.de/?section=gtsrb&subsection=dataset\n",
    "\n",
    "Here we just pick the 26K smaller training set (the online version), as the training and test set can all be directly loaded using PyTorch included functionalities. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchvision.datasets import ImageFolder\n",
    "from torchvision.transforms import ToTensor\n",
    "\n",
    "standard_transform = transforms.Compose([\n",
    "        transforms.ToTensor(),\n",
    "        # Change the image to PIL format, such that resize can be done\n",
    "        transforms.ToPILImage(),\n",
    "        transforms.Resize((32,32)),\n",
    "        # Bring it back to tensor\n",
    "        transforms.ToTensor()\n",
    "    ])\n",
    "\n",
    "# Here the data is not be normalized to [-1,1]\n",
    "\n",
    "# Change the folder based on your specific needs. \n",
    "# This one is a smaller (26640 examples) data set (for online training), so the \n",
    "data = ImageFolder(root='data/GTSRB-Training_fixed/GTSRB/Training',  transform=standard_transform)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['00000', '00001', '00002', '00003', '00004', '00005', '00006', '00007', '00008', '00009', '00010', '00011', '00012', '00013', '00014', '00015', '00016', '00017', '00018', '00019', '00020', '00021', '00022', '00023', '00024', '00025', '00026', '00027', '00028', '00029', '00030', '00031', '00032', '00033', '00034', '00035', '00036', '00037', '00038', '00039', '00040', '00041', '00042']\n"
     ]
    }
   ],
   "source": [
    "print(data.classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 0 \t Speed limit (20km/h)\n",
    "- 1 \t Speed limit (30km/h)\n",
    "- 2 \t Speed limit (50km/h)\n",
    "- 3 \t Speed limit (60km/h)\n",
    "- 4 \t Speed limit (70km/h)\n",
    "- 5 \t Speed limit (80km/h)\n",
    "- 6 \t End of speed limit (80km/h)\n",
    "- 7 \t Speed limit (100km/h)\n",
    "- 8 \t Speed limit (120km/h)\n",
    "- 9 \t No passing\n",
    "- 10 \t No passing for vechiles over 3.5 metric tons\n",
    "- 11 \t Right-of-way at the next intersection\n",
    "- 12 \t Priority road\n",
    "- 13 \t Yield\n",
    "- 14 \t Stop\n",
    "- 15 \t No vechiles\n",
    "- 16 \t Vechiles over 3.5 metric tons prohibited\n",
    "- 17 \t No entry\n",
    "- 18 \t General caution\n",
    "- 19 \t Dangerous curve to the left\n",
    "- 20 \t Dangerous curve to the right\n",
    "- 21 Double curve\n",
    "- 22 \t Bumpy road\n",
    "- 23 \t Slippery road\n",
    "- 24 \t Road narrows on the right\n",
    "- 25 \t Road work\n",
    "- 26 \t Traffic signals\n",
    "- 27 \t Pedestrians\n",
    "- 28 \t Children crossing\n",
    "- 29 \t Bicycles crossing\n",
    "- 30 \t Beware of ice/snow\n",
    "- 31 \t Wild animals crossing\n",
    "- 32 \t End of all speed and passing limits\n",
    "- 33 \t Turn right ahead\n",
    "- 34 \t Turn left ahead\n",
    "- 35 \t Ahead only\n",
    "- 36 \t Go straight or right\n",
    "- 37 \t Go straight or left\n",
    "- 38 \t Keep right\n",
    "- 39 \t Keep left\n",
    "- 40 \t Roundabout mandatory\n",
    "- 41 \t End of no passing\n",
    "- 42 \t End of no passing by vechiles over 3.5 metric tons\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyper-parameters \n",
    "num_classes = 43\n",
    "learning_rate = 0.001\n",
    "sizeOfNeuronsToMonitor = 84\n",
    "batch_size = 64\n",
    "num_epochs = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "\n",
    "loader = DataLoader(data, batch_size=batch_size, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAJIAAACPCAYAAAARM4LLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAFj5JREFUeJztXWuMXddV/tY59965c+f98Nielx3Hrzhx7DaOEyehCW2SuqFgKFXUIFCRihACJBD8oCp/QAIp/IH+qECKRCE/KE6gIEIpTZraCSVpHDsOThw7fsT2eGY89ow9dx6euc9zNz/m+qy1tn1n7tgnd/zYn2R5nbv33WefM+vu9dxrkzEGDg43Cm+pJ+Bwe8AxkkMkcIzkEAkcIzlEAsdIDpHAMZJDJHCM5BAJboiRiGgnER0jopNE9M2oJuVw64Gu1yFJRD6A4wCeAjAEYD+A54wxR6KbnsOtgtgNfHc7gJPGmFMAQES7AewCUJGROjs7zOrVfeUrm4HlNVU5hWr72SgxFRRVy9joBTGjuGpLNjaHdEO8jmeRn1X9pibHQzpb1ONfLgbi3iXVZgJ+B6bEtP2UJN5VYC0ERvQm8UXP06N45PNFSY8RGJ5jJl+8aIxZhgVwI4zUA2BQXA8BeGi+L6xe3YcDB14rXwVWq7y2Ja6Pa8N+xV4F2h4/F1KZyfOq199+59shncdy1bbx4adDenvfmpCuG3hf9Xvtv3aH9MeX0qrtrVG+zk5nVFt+shDShSzTCetd+SVmzmmLUQuCQeI+v4OG+oTql4w3hXQpp8eYykyF9AdnLwygCnzqyjYR/TYRHSCiA2Njlz7t2zksEW5kRRoG0Ceue8ufKRhjXgDwAgBs27ZVrKHziaX5xJ78ddpjSFE0n+7Hv9pYUv+WNvY3hvTw2RnVdteyzpBu7V4d0qOe7vfO9HRID4zpFW/0zFhI53NadMYNP0+zkEv1MT3HeCoV0lTS4nEmz+9Hfq2lPqX6mSLfu1Sn5+HnL2OxuJEVaT+AdUR0FxElAHwNwCs3MJ7DLYzrXpGMMUUi+n0Ar2LuJ/5dY8xHkc3M4ZbCjYg2GGN+COCHEc3F4RbGDTHS4mFwtbUm2ypB6gFSGmsdyQhrhjx7PGnyixFI6xi9HaxLZEf0CJ7QwUwdD+IntB6UCU6F9PDQpB5D6CYxy3T3xfPUJViPq09qqzVex3+2TC6n2vyCeB5B5jN51a9UYqswb72DVH2lv1FluBCJQyRwjOQQCWor2kwBpfyc55gsk1aLGGtalBQXwlQ1WrQFYpn3rOU6kF5kYucc+XqM7s4VIV3o1ibz6DSb+R05NvGzM3q+PaWekJ7SfkAcn2UnpE9atHnieqLAoidd1OIrNivmbPRzoshthvgdZ7J6jACsBuSTev7rOjvE1SiqgVuRHCKBYySHSOAYySES1FRHCoI8Lk+eAQDErSBiPME87ccarG9y1J2onj8m7dqPJSs/DolHLQVMe6RN3ca2u7itoGODvogVTpxlfaalfoPqt+sX/5TvlXtBtX34wbt8Yak3MaHzZfKswwQF3U/qVglPrwUqLUiQHSmt77U28b0mLD1rbFrrU9XArUgOkcAxkkMkqKlo82N1aG4vi46rWFjmxNTpJpLLspyyNQhV/l1I0ebHGkVLs+pX39cS0mu2WR5r4jGa7+4P6WKiUfVb3toa0qfSJ1Tbux5H1jMDx1Wbl87ymCL6H1hJDlJ8leYJCJAQ/Z5UCQBAJPTlclnVVGroElenUA3ciuQQCRwjOUSCGgdtPcC/kuJZtNrkddxqkxaeXOft34F8HDs9V45ZV7lfki3Gxs061TYrJMD+NFt76ZK2/Nb7LOrufupZ1fbsuvtCuuekFnvDr70W0q+cPBrSZ6d0um6+KLzj1ivwhQgj8a7GrbzycXAQN2O9gt42yx1fBdyK5BAJHCM5RALHSA6RoMY6EsA6znyJbHab1J/klO3kf9lmmbuG5f5Yhr93yhpiYIB1h/QlnQx2KcvjD2b5NzgLbT4/3sB6VmmiXY8/sz2kk4l7VVvDVnZFrPD/O6TvS+lk/OTMUEi/d15n3xUDNt17hIl/9LLul07yg2/dsEq1PbqONzm8+iNUBbciOUQCx0gOkaDGoo3AZridFzzflm1pn0oz3jZTWZwFJe0dn7rI4799hkXRvxvtahgeYtM6kdPzKAoR69fxnPqXJVW/ni5uKxodAH370P+F9GvHDqu2OC6GdCpoC+lfeeQZ1W9NkV0DvdlB1bb1ia+EdN/w6ZDefWCP6jfaxB78LStaVdvnevjd/S6qg1uRHCKBYySHSOAYySESLIH5fwU2D1MF2r6WU9a6CQzL9qkR7UJ46QiHCPZN8RhnilZiWImrdBhrb1zRZ71uexvrQb/QpfvdL6zp2Aqtq5mP2ZSfOapN8sNTPMcZseHhhUNjqt8zX9oa0l/a8nOqrbtrJV9s7Q3J3/niZtUvZ/gd1JPOnDu152UsFguuSET0XSIaJaLD4rN2IvoxEZ0o/9823xgOtz+qEW3/CGCn9dk3AfzEGLMOwE/K1w53MBYUbcaY/yGi1dbHuwA8UaZfBPAGgD9Z+HYGCL3AVsKyEl92m1x6K0fuB8UWrANHtLf5nUvc90yJaftOJZEpFo/r1lUiz/nBlUw/1KfnkZROdcvB/oWd94f08Uu6htXwOydDOi28BvtOaRE4tJdFYHvHDtX2RAffsCXOrg0/pv/UKSPe6azeVn782OLrWF2vsr3cGHPl6c4DVmkzhzsON2y1mbm8z4qBM12xbbxSN4dbHNdrtV0gopXGmBEiWol59vXqim2b54nUzufZ5rZCkelzI1Oq1ytH2dP9prVXetJnK0gV8LQqggQei7O2ei3anl7Br+vRPv4NxvVOH8iyjicHzqq28SKLlJ27fl63jfNcXv2IPdY5azvSxWP8g/zO60dVW7aL64b+0mpOsGsgHYAO8hMhPbvvbdV26NX3sFhc74r0CoCvl+mvA/iP6xzH4TZBNeb/PwP4GYANRDRERN8A8DyAp4joBIAny9cOdzCqsdqeq9D0hYjn4nALYwmi/1duaUf/LUVAQpjkA6d5r9n39ujI9wmfTeF0U5NqKwg9S25/I6uQuaya1tyo9adNPWzmN4td5UNTWpeaTrOSdHxAq48fj7DOtKZOb00PAn4nns9/mljJykLIc9vpjy6qtrfe4Hr5O351W0gnmvX7PnvyQEj/y/f+QbX9bHAai4WLtTlEAsdIDpFgCYuRLsKzneVks/OHWbQdPK33as2K8vFFyySHJ0WbqGp21Sz43p6nxYHMgTs2zG1vHjyt+jV7LKYnxy+otoPv816200YnlGUn2UQvCvnreVbVFVkUNaOfYOwsu0QuZDkK0N7covqVPBbT+WY9fqnR3le4MNyK5BAJHCM5RALHSA6RYAl0pKKgJeZJbBNHTpXOcqQ6P62j7kUjrmNWMXTibADPl3qQvpXMZUundeL+7rc4rPDJEd5bNj6hy988fjenZzUUrELpo1wZ94RW8ZARv+u8zGwg/T7knn5T0M85OMKJc68f4TOGVnT0qX79/ZtC+tkdj6u2puF3QvoHGEI1cCuSQyRwjOQQCZYgZ7uS+S+WcqNFVv4Ci7Pxcyxeivku1U+u8o0pLQ52dPCjrqwXhUS79G/p0nn+3uFDWrS98R6b7iNjx0K6rV3fqyREW96q6+mLpDpZcBQApoRZH5Tkn8Z6V4bdIWS1DY/ymAcH+b3tCtarfqvAWQIb++9XbbEnOVPij17Wp2NWgluRHCKBYySHSLAEVtuVpdgWbTKSqpf8oqicMTPLpk4x0J7nmPAor7WSf399Aye69UrJaeVUnxLHiA6dPqNneJ6ts6RhK7AtpW/W1sjnmcRJB2YTdZx/XTL6CFNprMpjwsxVFUflCdy6LSuO2bpcJ44MI60uFISbfjbQ27qmCnad84XhViSHSOAYySESOEZyiAQ1Pq+tBOTKpmvMvrWQ9RmdWFWa4eStrDhCE57WpbY0s5LxZWvv7zKhCyVEcDub02O8/tMPQ/qTUZ2435ZiWz5f4t/g8h6tIz26cS2PP6QT2040cgW3Mxd1Mp8R1WqNEceqzrMZwtY1C+Ax8s0c/Z+1EvjODZ4L6Xf36OT/Eyf0Prdq4FYkh0jgGMkhEtRUtGVnZnH8vf0AAD+pz++IiehpoqjN4sZ4o+jHS3kyocXSPd3c9tAKvZQ3VcjVslZ8rF4lTrD2dOJZYZZFyslxlpXN7VpEdXexayBT0L/VliY2w+1i61JgGVM5iK3aPB0U7hfbx7eu4eKmKSs6ncnwnC+N62KnQ2N6v2A1cCuSQyRwjOQQCRwjOUSCmupI8UQcy7vnqohRUmfnS13FK2mTtq7ALvyeNg6R+KM6tN7dzXTC8vJXqixA5/S+sA1dfNR6Jq91k9xl1sla29jkT3bq3+OFGc5QSGf0+NNZLhlTssIWXpx9Fp6YcbFg7TMjNuv9hmWqqUPoYI+lWLfssc98a2Qdr69L64Inz0xgsahmy3YfEe0loiNE9BER/UH5c1e1zSFENaKtCOCPjTGbADwM4PeIaBNc1TYHgWr2/o8AGCnT00R0FEAPrqNqm59IoKWvfIQn2d5agcAqxD7MIizVzclssQt6z9ieo2wKBzpFGY8Ib4NMh7swoKuTHU2zaPjR+7pkzOwEm8nFej6v47OPb1L9GupY9EzO6PFzQuwZa1+bITbX/Ri/n2RC/95LYvFv6L5Lta1Zw6JuQz336zQ6U2I0yyb+7KTOOW+N62p31WBRyna5BOBnAOyDq9rmIFA1IxFRI4DvA/hDY4zyWM1XtU1XbFu8Eudwa6AqRqK545q/D+CfjDH/Vv74QrlaG+ar2maMecEYs80Ys23ZstZrdXG4DbCgjkREBODvARw1xvy1aLpSte15VFm1rVQsYCY9V4rGWEeyqyPIrbBCXBRir29jPai7WU//g2OsS42s0OGT9i3cl4QK1rr2btVv7G3Wuz4Z03rFhWHWdzIxdkOYfr0J4SsP8TlscSsUVBTnxpmYfgclI0IVcu9/XYfqVycyMpf39Kq2lnp+7nbidxrLaldJboSfs5DXYZbU4o+0rcqP9CiA3wDwIRFdOdrnW5hjoJfLFdwGADxb4fsOdwCqsdr+F1dXB70CV7XNAUDNE9sMcOV4TGPxph2Gl4jzMt8r9ow9Ma3d14U0r8mnP9ER7ZcK/KjdQqL0+VoESkUvQfoYUQKLBwqEGB3QZ4W8PiC82ZYWmmsX7vdCp9XIFeiCHIubkpWF0LScTfz71+hyNZ/byO6LNpnykNMZFaUZ4SqBTo2YKNrV9BaGi7U5RALHSA6RoKaizYvH0dDZW6FRiBhryzbAAd66JrZEdlgFTQsjXIC0NK0fbfisKOApEtQKw4f0nUp8Pkh6SouURONGnmKR87lLMzrHOT2VDunmeh2cLiTYAou16a3SSXH01YwYo77rHtVvxb0szh57QO9J297HKkKdkFhkHasa5HkNGbeqoswErmKbwxLBMZJDJHCM5BAJal+w3bf1nysQoTqy+3Cb38wmfvda7YJ9sof1hYRVaOzNI6xznDjPMb+J8zrxbHCSrxsatD6XEl5qmmU9wrOqsuXHOBGtsU/rMN39bP6Pntc6WEOSdaHGVay4dHTrfg88wElpG/u17tMq/qKqpaDdHJfP8/s4e0krSVPFxbOFW5EcIoFjJIdIUPst28XyMupbkUFTvDYNAPLYTMMeZc86b2RZK29XfrpDe3xXGRYHey9ywPJVX2e/TAfs9g7yGdXmiwSzIM+e81iDnkddhq/Pn9CJbafPsRj0UmtUW2s7J7a1t7Pb4JnN+l092sHvoymtXSDULkSpL4qW5vXeu9nzPK+RCf2clwK75NDCcCuSQyRwjOQQCRwjOUSCmupIxXwWY4NzCfUJUR4PAOIxlvV2pVYSSV6+SAbzk1q2Bxk+69XP6uz/Det5v1pQZN1kfFZH7s0s+w2mdEYxZjIcQc8HQteJ6yj+5dmVIT0xbWWFNvCzdLdpN0dvD+sx69fxcz7Wr/WgFSKxwarKUxm23jMrXBZWU6FY7aAMtyI5RALHSA6RoKaizfdjaGqZEwNeslm16bPRLNEmzisjsR/O860otTiyc3RQV1v7z0lOWdvx2c+E9G/NahM8VmDxeDjQkfugjkVR+jLfu7ldzyOTYJHrL9NiqW8lm+e77tei7Z4Gvk7V87OkYnoMv5XvF7dz0CoEDuxthKk4z6PF17nj6Zz21FcDtyI5RALHSA6RoKaijWJxJFt7rlxYrVKcWfwtj8U2pmI/L8/rfDGntx0PXmQLrHecA5apXFr1uzjNQdsNj+jzOzauZgss3rE5pIOUzu1uGmTrLpUaV21n6zkJrkR6/oHH3neTY/Hle7riiFXbzWqTY/L7KEGLq5x434WibSVXCqxXhluRHCKBYySHSOAYySES1FRHKhULmJ2YKxROMb2VWaoLfszSfUQynEcs68mydYOM8MjO6qj7ygRH5GmG9YqRnE7cPzG0j4fYO6zaLosSMj33sY60doeuMVYY5WLoR07pnezvTnDmweS921Xb+i/ytRF/GmP0lm2pFpmSfZabPBmHXeD57BnVbWKMj5jP5vWet+A61pdqKrYliehdIjpUrtj25+XP7yKifUR0koheIqLr2DHucLugGtbLAfi8MWYLgK0AdhLRwwD+CsDfGGPWAkgD+ManN02Hmx3V7P03AK6skfHyPwPg8wB+rfz5iwD+DMDfLXzLa5ZRsmBv31YHmV2bBoAkm8ydbdrb/OU2kTTWxR5rf8WDql/vnn8N6ZYubdZ/9bGHQ7pvPXvEU23aA173OIu94kP6QLgHj3wQ0stadW2yHlGlzasTCWr29nb13PZaIPuyCyRW1MHppsvs5vCKOumt+GmZ/0TklyuRjAL4MYBPAEwYPnllCHPlAB3uUFTFSMaYwBizFUAvgO0ANi7wlRCyYtvFS4s/msDh1sCi1HNjzASAvQB2AGglCt3TvQCGK3wnrNjW2dF8rS4OtwGqqdi2DEDBGDNBRPUAnsKcor0XwFcB7EaVFdu8WBypiiESGca2ovokr2XSlda3fJ+j7q2d2r3Q0sAlcLwmNplnslof6O5fF9KbNj+i2u7bsiGk21pEBTj755hgPaWU0BXhtm/keRQCvafOUxXc5MZ9a/x5zmvT4X9RGsc6P7cQsF5krAcwFcthVUY1fqSVAF6kuQCMB+BlY8wPiOgIgN1E9BcA3sdceUCHOxTVWG0fYK4ksv35KczpSw4OIGOqMccjuhnRGObqTXYCuLhA9zsFN/u7WGWMsdMPrkJNGSm8KdEBY8y2mt/4JsTt8i5c0NYhEjhGcogES8VILyzRfW9G3BbvYkl0JIfbD060OUSCmjISEe0komPlHKY77qDA2/k0zpqJtrJn/DjmQixDAPYDeM4Yc6QmE7gJUD5FaqUx5iARNQF4D8AvA/hNAOPGmOfLP7A2Y8y8hyjebKjlirQdwEljzCljTB5zMbpdNbz/ksMYM2KMOVimpwHI0zhfLHd7EXPMdUuhlozUA2BQXN/ROUy322mcTtleAlzvaZw3M2rJSMMAZNGiijlMtzNu5DTOmxm1ZKT9ANaVd58kAHwNc6dQ3jGo4jROoMrcrpsNtY7+PwPg25jLvvquMeYva3bzmwBE9BiAnwL4EJyR9i3M6UkvA+hH+TROY8z4NQe5SeE82w6RwCnbDpHAMZJDJHCM5BAJHCM5RALHSA6RwDGSQyRwjOQQCRwjOUSC/wdo595bE4iv8AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 144x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import util\n",
    "\n",
    "# get some random training images\n",
    "dataiter = iter(loader)\n",
    "images, labels = dataiter.next()\n",
    "\n",
    "# print(images.shape)\n",
    "util.displayGTSRB(images[0].numpy())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "\n",
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(3, 40, 5)\n",
    "        self.conv1_bn = nn.BatchNorm2d(40)\n",
    "        self.pool = nn.MaxPool2d(2, 2)\n",
    "        self.conv2 = nn.Conv2d(40, 20, 5)\n",
    "        self.conv2_bn = nn.BatchNorm2d(20)\n",
    "        self.fc1 = nn.Linear(20 * 5 * 5, 240)        \n",
    "        self.fc2 = nn.Linear(240, sizeOfNeuronsToMonitor)\n",
    "        self.fc3 = nn.Linear(sizeOfNeuronsToMonitor, num_classes)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.pool(F.relu(self.conv1_bn(self.conv1(x))))\n",
    "        x = self.pool(F.relu(self.conv2_bn((self.conv2(x)))))\n",
    "        # Flatten it to an array of inputs\n",
    "        x = x.view(-1, 20 * 5 * 5)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        x = self.fc3(x)\n",
    "        return x \n",
    "\n",
    "    def forwardWithIntermediate(self, x):\n",
    "        x = self.pool(F.relu(self.conv1_bn(self.conv1(x))))\n",
    "        x = self.pool(F.relu(self.conv2_bn((self.conv2(x)))))\n",
    "        # Flatten it to an array of inputs\n",
    "        x = x.view(-1, 20 * 5 * 5)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        intermediateValues = F.relu(self.fc2(x))\n",
    "        x = self.fc3(intermediateValues)\n",
    "        return x , intermediateValues\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = Net()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# .. to load pre-trained model:\n",
    "net.load_state_dict(torch.load('models/3_model_GTSRB_CNN_27k_train99%.ckpt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.optim as optim\n",
    "\n",
    "# Loss and optimizer\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(net.parameters(), lr=learning_rate)  \n",
    "\n",
    "furtherTrain = False\n",
    "\n",
    "if furtherTrain: \n",
    "\n",
    "    # Train the model\n",
    "    total_step = len(loader)\n",
    "    for epoch in range(num_epochs):\n",
    "        for i, (images, labels) in enumerate(loader):  \n",
    "            # Move tensors to the configured device\n",
    "            labels = labels.to(device)\n",
    "\n",
    "            # Forward pass\n",
    "            outputs = net(images)\n",
    "            loss = criterion(outputs, labels)\n",
    "\n",
    "            # Backward and optimize\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            if (i+1) % 100 == 0:\n",
    "                print ('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}' \n",
    "                       .format(epoch+1, num_epochs, i+1, total_step, loss.item()))\n",
    "\n",
    "\n",
    "    print('Finished Training')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For testing, we should also use the test data, where they are separated into folders based on their classes, followed by shuffling."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "\n",
    "testdata = ImageFolder(root='data/GTSRB_Online-Test-Images-Sorted/GTSRB/Online-Test-sort', transform=standard_transform)\n",
    "testloader = DataLoader(testdata, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAJIAAACPCAYAAAARM4LLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAEh1JREFUeJztXV2obddV/sZaa+/zc3+SlMYYcqMtGJS+WCHEij5INRB8iQ8ijSAtFOJDBQUfLH2qoBB9sH0rBBq8D2IMKFikIKVEVJCaWH9qU9rGQO0Nt4lpmubk3nP2Xj+jD3vdM78x9ln7rHPOzD7nnjs+SO5aa64919zrjD3H/xiiqggEToritBcQOB8IQgpkQRBSIAuCkAJZEIQUyIIgpEAWBCEFsuBEhCQij4nIN0XkZRH5ZK5FBW4/yHENkiJSAvgWgEcBXAPwAoAnVPWlfMsL3C6oTvDZRwC8rKqvAICIPAvgcQCDhFSWlVbVBAAghd0MRWT/uChL/8l0qF06pGOPrnM/EP4c0phgGFLYdWiX5hDzQTuL0mlZTMxYQd97+XumdZVl+pz/lisejZLfKy1Su3ZwDik8GaTRN177vzdU9V4cgpMQ0gMAvkvn1wD8/MqHVRNcufI+AMBkesGObUz3jy9cvseMdcX2/rHWu/vHbTOz99EL2L25a8fmN9McqPePRS1BFwX9Mad3m7F2luYoK/rzFhvmPi3SOi5d/HEztrWV7t2+235PRfpjX7583/7xbmtJaVLRj07s+i9up/daTtOfd+/GW+a+aZk+N7lgv6dia//483/2O9/BCJyEkEZBRJ4E8CQAVNW7/rjAKeEkf9lXATxI51f6awaq+jSApwFgOt3Qvb3Fr1od62nbtCN1bhsuJ+lXVtEvqZhsmvtu7Ly9f9zMbpixjnavtmvSHI4DllViN3W3Y8akmaf52/TBtrM7Y0Vsaaexr6StL6f13rS7xMZ2egdN2jRx46Zdx5R2ro3NLTM2u3Bx/3hzI+1+s9Z+0Xovrfmuu2DH8AMcFSfR2l4A8JCIvF9EpgA+AuALJ5gvcBvj2DuSqjYi8rsA/gELafgZVf16tpUFbiucSGhR1S8C+GKmtQRuY6xV+lVVdL0K3dRWrigkySbNvDFjXZvu7XjFc6uZ1XXSqpq9m2asbZNcocoymJUdyi6tQwqrLQl9jlV8b2roSMBpG7tGKMlnk6kZ6rok37Tt9/ePd50stdukOS7e9WNmbNamF7Qre/vHO3MrMxazJE8WpMUCwM05joxwkQSyIAgpkAVrN+zccsl4w3NLrpolo/Qsqb8bxHpq595p6rRFN43drjtS+Y1V16+PWZ1bCFuHjSXezcEsUdU+YbaXWEzVWDMHW9+bho7nlk2jS9+tnlnDbtOm1TCH2ptb9liSKeOH33/NjLV69P0ldqRAFgQhBbIgCCmQBafg/FrIHc4PCSVVvlCnfyqp5C1ftstv5iQXOfmJfzFCY9K5hbArQa2cxTKM8G3woTjpXJ1Tta1ozso5nZvk8pEJm0Ds+yjLdN87O++4Z5NaT/4ftwzM2Rxyw8pq0w3rehqD2JECWRCEFMiCtbO2Wxq0OmuqUPhW2TqlnKzeLe3RZeksz6Su+wiCglV3YmeFjxDlYDnH9tQE1RH7sjPARp06E8IsWZs5bgkAZDdZumUzsXrZsPFObBHfq9+2Y0gssSzT/JOpY1e0LHHBd7Mm1P/AKSEIKZAFa3fatvssxjME4RvtiNE+EpvT1jtViWV1nu0NjKm3LqfDonS/MyEWwPHQbr3KGpGPlTas032ONFcOxJO5ZUv1Jml0zvHb0BdoW2b19n1wTDic9X1vdw9HRexIgSwIQgpkQRBSIAvWb9nuZZKlvEySOTqv/dPNFctBfg7OO2u97HOwSi5ODiqrpGqXU692k1xBMoY4NZ4TG9ramjlaCrhrZs6r35Lsw6LUng2Oa+i7ldsXzRjLTEJmk8691Anl1DW71jrezK3FfQxiRwpkQRBSIAtOIbCtPxBn1TXmAEffbEU2sdduDuNU9SYEmp3YUuEclOV0i47tWDFJrE443dqlnyuZMirvFKbzZmadsfXbb6bj3WSxFh+kx+xyz8ZikzEbbUVOYLVx8DW943lj19F2zlk9ArEjBbIgCCmQBUFIgSxYv4zUCysCL8MkmhYXhcVBZIUJSvPuB77Ryk+suhs5yMtIdC6VqzJCchHLQYX4FAJ+sPutkotnMrFe94rMDcVOWsd8x+bisytFayvfKM3P0pm4IMA5mQNUVplKxuHQHUlEnhGR10Xkf+jae0TkSyLy7f7fe1bNETj/GMPa/gLAY+7aJwF8WVUfAvDl/jxwB+NQ1qaq/yQi73OXHwfwy/3xVQD/COAPRz2x51M+zpkd4RNv2bYrGjh2uWaVZRuoksW3pe2/dSaEkvLJSliVWZidUZUzzwkayqHrWjsHB85VriIcmyWmF6nWjHvAfIe8+rW1erekylurvTNRcFCgewerKuEN4bjC9n2qer0//h6A+1bdHDj/OLGwraoqIoPSGVds83UjA+cHxyWk10TkflW9LiL3A3h96Eau2FZWlXZ9MFrh2RdfcGTJZGpSidyzTHFPx9o41puDvJo9G8RVNyk9vHBBb5tbKT16a/tSGvAaETk9584BahyindW4pmRJ395KzlhvYa820zrq1lmhmZWyF0DcfbKC7fk0rBE47hbxBQAf7Y8/CuDvjjlP4JxgjPr/VwD+FcBPi8g1Efk4gKcAPCoi3wbwq/154A7GGK3tiYGhX8m8lsBtjDVbtmXf67/UccCkQPt8bj3w2NeYFpKLOnHF1tk0QGp3VQ4bF+q5XcfezeRp57I5lQuA443e581xQfV5bZ99850kn2mb5KfNzUvmPiGZqahtVVular4c3KelSzQwOXrDxdzHItSoQBYEIQWyYK2sTUBq/1JgG597tkcqPx+7Vh5sp1JnNWZ7g9IHS2eHqDbJuuwcum17sMV6PrMmhAmx2NKZBibkCJ5uumprVKBVyDreuvy90ljwLVtt64Pjrb2TnEUL73Pu9OjMLXakQBYEIQWyIAgpkAVrVv91We3vwZq8wMtPdDx4AiMHla4TU02ztFTwHLVvYUXV4Zx8U7ILhr5G68vf0KD3o9t+ba7Fl6bnSXdw3zXAvo/S9XxrKSqBK/kWvlQwBbN1bv5ujd7/QMAgCCmQBesta4OkdopnS3S6HJQywA49C+S478qzjcQC5jWp7q5nyS5506dTWzKm2kidLGEqx/m2qnR+lPhnzt/jvif+907f07Ml69UfrmDXCZtKfB7hyPXyko7+kUBgGUFIgSw4tSazS7ZT45gdtnqzQ3e5CCjd5wt5EwuYkmZWuFae83m6bza3gWccEDfZoJSm0sWHWxV0GEtf4OBO4Kss/UvFTrnGNw52di9Oh/ulDGnWqxA7UiALgpACWRCEFMiCU5CRhoSG4S5qOhTwv4Lvd65iGxdH54C4DReUxu3PGzcHV1/jlu+da9c+4appLgpBjAzjQHKcsNyiw5ZzPyYrrOruYYMj68xrCwQMgpACWbD2wLZbG+pSyLbp7eGttVwcPV32Rci534iP2e6U24iSVdoFtk0qZksuJRzUBpViqrvaBrYpsb0lx7LRyFfErbMpw6e3s5V6Rb8UO/UqVuZFhMFbBxE7UiALgpACWRCEFMiCNTe1IR7uZAcjBywFo6fjkt0lvig7BeQXrgdZS7IDyzfilGTT1t3NweIUl6RpnYuEzQatMw20JN+UlY0uwICZw7t7jJuoGy6b46IAhx61lEf4rqj/IvKgiDwvIi+JyNdF5Pf661G1LbCPMaytAfAHqvoBAB8C8AkR+QCialuAMCb3/zqA6/3xjoh8A8ADOGbVtiHN0qj/LrLNckFibU715ULm5cRarKfc14wm3HOBbW2T+nJMJq6cDBUP5djuycT2A6mbxL52d3fMWEuF2Ceu9M6UiqSy9b3wrIbeVddY1mbeHAeveXGB1+RZp+9hNwJHErb7EoA/B+AriKptAcJoYVtELgL4GwC/r6pvi+2gOFi1zVRs883nA+cGo/6yIjLBgoj+UlX/tr/8Wl+tDauqtqnq06r6sKo+vBSnHTg3OHRHksVf//MAvqGqf05Dt6q2PYUjVG27Fe2oLs+qpQ2tdMHoxoNufCTODUKlZoq5y8cnGWl6Mck025cu23VwAXSfCkY/hJJkpMqp8Rv0+5xuun60Myq23toIzGKoHI73pJCZQ13V3I5lTU4SsFOY3LVuqa/v0X0kY1jbLwL4bQBfE5H/7K99CgsCeq6v4PYdAL955KcHzg3GaG3/guHglajaFgCw7sC2Ve5/olWfMqyUhmwC630Vsposyq59J/cLmRCrnPpWnlNidSsiFEwenvuZlcSiNkubXLBBpXK86t4RO+6IBfqi762pjOsD+Ejl5+t2iZad+d52ODpCjQpkQRBSIAvWy9qUd1G/gVK1kKWUtIMdqUvFSMl52rn2mvVuKtLJgWETxxomFxJrE+9UHUiH9k5OWZVPRk5WnbsWo8zO6LidWzatJoDPvgNeia6ID+eYdl/NDcew98WOFMiCIKRAFgQhBbJg/a1I+399SRpzj5MruCKaUKWxiatWZlwwTm7pqFxNt5tkjsbl9zd75P3fvsuMyVC7doeuXeGdJ2t2M7NBbx1Vx+2aNLYUaMbtRn2reHoF1sO/ynrtgt6Gm10NInakQBYEIQWyYO2W7WKgF4ktIO6KY5KDt1ZiFc6yPSX1v3Cp2Mq+WGJz6lhPvZMC0eob75gxEDvjUjZSuNdotH9npe84rdzFW3fcX40t+M45zb3nltT/9HCOD/dBgCYm3KcRRl5b4LQQhBTIgiCkQBacgovk8JyppZR4LvdCvH7mPd8kH/hgs3IrVaQtSOVfagLDpgbvq6Hzllq5+9x8K3+4MoZ87l0RVCBejRxk52iFIyVWlPSj7+J7COdG7EiBLAhCCmTBKVRsU/r/0uUF/DY8EL7svdY1xWz7amvTKfdJS2yvcjHVOieVvLE5b2yxZna2tFxiS+KLoZtcM/v6O2OVJjXePYGD0rxabx61KtliIJLh4PPDETtSIAuCkAJZsH7WdmvXXJXyspLvrajstiLYbEYO0jmlHFWuksiE0rLLDRvPXdHrMizFO4hN9TkMjvnOV2yJ5mp0R6kOYhu6rih8qjYEbniWcYgdKZAFQUiBLAhCCmTB+mWkAZV0VaCblX3GNYzxjVrMc0nPrjsbgN9w2rdPLuDUcW6ZvrTaFUXTV1S1HWqhLq5CL7cz7bygRbKPkZeWZFJ+p3ZkKRlgBMZUbNsUkX8Tkf/qK7b9UX/9/SLyFRF5WUT+WkSmh80VOL8Yw9pmAD6sqj8L4IMAHhORDwH4UwCfUdWfAvADAB9/95YZOOsYk/uvAG5FeE36/xTAhwH8Vn/9KoBPA/jcysmErK0+TZi38qUYYup8TVt34YutU0y1L/TJOW9s8V3q5Dmy9I6tD+ULmo6reFY61sksi2PCCx+bbjpprzBzUNp664rKm6A6H8nmxYIRGFsfqewrkbwO4EsA/hfAW6r74YrXsCgHGLhDMYqQVLVV1Q8CuALgEQA/M/YBIvKkiLwoIi/6mkiB84Mjqf+q+haA5wH8AoC7Rfa9jlcAvDrwmVSx7d0OigmcGsZUbLsXQK2qb4nIFoBHsRC0nwfwGwCexdiKbRTYdqS9aaCqra/strGR5KLN7Ut2bCu5OwqqJuuD0oYaywCuRxvJGF3jAuxMBbThnrOlV+tZLiJTw1LPN7qvbe38HUUUcLXatrVmDlvpbUXx/JEYY0e6H8BVWRgzCgDPqerfi8hLAJ4VkT8G8B9YlAcM3KEYo7X9NxYlkf31V7CQlwIByHFacx/7YSL/j0W9yfcCeGNtDz7bOOvv4idV9d7DblorIe0/VORFVX147Q8+gzgv7yKctoEsCEIKZMFpEdLTp/Tcs4hz8S5ORUYKnD8EawtkwVoJSUQeE5Fv9jFMd1yjwPPcjXNtrK23jH8LCxfLNQAvAHhCVV9aywLOAPouUver6ldF5BKAfwfw6wA+BuBNVX2q/4Hdo6qHNlE8S1jnjvQIgJdV9RVVnWPho3t8jc8/dajqdVX9an+8A4C7cV7tb7uKBXHdVlgnIT0A4Lt0fkfHMJ23bpwhbJ8CfDdOHusjUm87VXqdhPQqgAfpfDCG6TzjJN04zzLWSUgvAHiozz6ZAvgIFl0o7xiM6MYJHKEb51nCur3/vwbgswBKAM+o6p+s7eFnACLySwD+GcDXkFLePoWFnPQcgJ9A341TVd88lUUeE2HZDmRBCNuBLAhCCmRBEFIgC4KQAlkQhBTIgiCkQBYEIQWyIAgpkAU/Ar1pPgzcgN00AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 144x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "dataiter = iter(testloader)\n",
    "images, labels = dataiter.next()\n",
    "\n",
    "util.displayGTSRB(images[0].numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([5])\n",
      "tensor([5])\n"
     ]
    }
   ],
   "source": [
    "outputs = net(images)\n",
    "\n",
    "_, predicted = torch.max(outputs, 1)\n",
    "\n",
    "print(predicted)\n",
    "print(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This may take a while ...\n",
      "Accuracy of the network over test images: 96.39589466146869 %\n",
      "\n",
      "\n",
      "Accuracy of     0 : 88 %\n",
      "Accuracy of     1 : 98 %\n",
      "Accuracy of     2 : 98 %\n",
      "Accuracy of     3 : 98 %\n",
      "Accuracy of     4 : 95 %\n",
      "Accuracy of     5 : 92 %\n",
      "Accuracy of     6 : 97 %\n",
      "Accuracy of     7 : 98 %\n",
      "Accuracy of     8 : 96 %\n",
      "Accuracy of     9 : 98 %\n",
      "Accuracy of    10 : 98 %\n",
      "Accuracy of    11 : 95 %\n",
      "Accuracy of    12 : 99 %\n",
      "Accuracy of    13 : 99 %\n",
      "Accuracy of    14 : 96 %\n",
      "Accuracy of    15 : 93 %\n",
      "Accuracy of    16 : 100 %\n",
      "Accuracy of    17 : 99 %\n",
      "Accuracy of    18 : 97 %\n",
      "Accuracy of    19 : 100 %\n",
      "Accuracy of    20 : 99 %\n",
      "Accuracy of    21 : 37 %\n",
      "Accuracy of    22 : 98 %\n",
      "Accuracy of    23 : 87 %\n",
      "Accuracy of    24 : 94 %\n",
      "Accuracy of    25 : 99 %\n",
      "Accuracy of    26 : 91 %\n",
      "Accuracy of    27 : 100 %\n",
      "Accuracy of    28 : 91 %\n",
      "Accuracy of    29 : 62 %\n",
      "Accuracy of    30 : 92 %\n",
      "Accuracy of    31 : 96 %\n",
      "Accuracy of    32 : 100 %\n",
      "Accuracy of    33 : 95 %\n",
      "Accuracy of    34 : 100 %\n",
      "Accuracy of    35 : 99 %\n",
      "Accuracy of    36 : 100 %\n",
      "Accuracy of    37 : 100 %\n",
      "Accuracy of    38 : 92 %\n",
      "Accuracy of    39 : 96 %\n",
      "Accuracy of    40 : 90 %\n",
      "Accuracy of    41 : 100 %\n",
      "Accuracy of    42 : 100 %\n"
     ]
    }
   ],
   "source": [
    "class_correct = list(0. for i in range(num_classes))\n",
    "class_total = list(0. for i in range(num_classes))\n",
    "\n",
    "with torch.no_grad():\n",
    "    print('This may take a while ...')\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for data in testloader:\n",
    "        images, labels = data\n",
    "        outputs = net(images)\n",
    "        _, predicted = torch.max(outputs, 1)\n",
    "        c = (predicted == labels)\n",
    "        label = labels[0]\n",
    "        class_correct[label] += c[0].item()\n",
    "        class_total[label] += 1\n",
    "        total += labels.size(0)\n",
    "        correct += (predicted == labels).sum().item()    \n",
    "    \n",
    "    print('Accuracy of the network over test images: {} %\\n\\n'.format(100 * correct / total))\n",
    "\n",
    "for i in range(num_classes):\n",
    "    print('Accuracy of %5s : %2d %%' % (\n",
    "        i, 100 * class_correct[i] / class_total[i]))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This may take a while ...\n",
      "Accuracy of the network over train images: 99.92867867867868 %\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    \n",
    "    correct = 0\n",
    "    total = 0\n",
    "    print('This may take a while ...')\n",
    "    for data in loader:\n",
    "        images, labels = data\n",
    "        outputs = net(images)\n",
    "        _, predicted = torch.max(outputs, 1)\n",
    "        c = (predicted == labels)\n",
    "        total += labels.size(0)\n",
    "        correct += (predicted == labels).sum().item()\n",
    "    \n",
    "    print('Accuracy of the network over train images: {} %\\n\\n'.format(100 * correct / total))\n",
    "\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Trigger 2-projection neuron on-off activation coverage computation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nndependability.metrics import NeuronKProjection\n",
    "\n",
    "k_Value = 2\n",
    "\n",
    "metric = NeuronKProjection.Neuron_OnOff_KProjection_Metric(k_Value, sizeOfNeuronsToMonitor)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Trigger the function addInputs() to update the k-projection table based on all visited patterns for each batch. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing 2-projection neuron on-off activation coverage ...\n",
      "\n",
      "Current input size fed into the metric: 64\n",
      "2-projection neuron on-off activation coverage:11711/13944=0.839859437751004\n",
      "\n",
      "\n",
      "Computing 2-projection neuron on-off activation coverage ...\n",
      "\n",
      "Current input size fed into the metric: 3264\n",
      "2-projection neuron on-off activation coverage:11711/13944=0.839859437751004\n",
      "\n",
      "\n",
      "Computing 2-projection neuron on-off activation coverage ...\n",
      "\n",
      "Current input size fed into the metric: 6464\n",
      "2-projection neuron on-off activation coverage:11711/13944=0.839859437751004\n",
      "\n",
      "\n",
      "Computing 2-projection neuron on-off activation coverage ...\n",
      "\n",
      " ... early stopping for DEMO.\n"
     ]
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "\n",
    "    total = 0\n",
    "    i = 0\n",
    "    print('Computing 2-projection neuron on-off activation coverage ...')\n",
    "    print('')\n",
    "    for images, labels in loader:\n",
    "        \n",
    "        total = total + (len(labels))\n",
    "        labels = labels.to(device)\n",
    "        outputs, intermediateValues = net.forwardWithIntermediate(images)\n",
    "        \n",
    "        # Add the batch of neuron activation patterns to the k-projection table\n",
    "        metric.addInputs(intermediateValues.numpy())\n",
    "                \n",
    "        if(i % 50) == 0:\n",
    "            print('Current input size fed into the metric: '+str(total))\n",
    "            metric.printMetricQuantity()\n",
    "            print(\"\\n\")\n",
    "            print('Computing 2-projection neuron on-off activation coverage ...')\n",
    "            print('')\n",
    "        i = i+1\n",
    "        if i > 100:\n",
    "            print(' ... early stopping for DEMO.')\n",
    "            break\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2-projection neuron on-off activation coverage:11711/13944=0.839859437751004\n"
     ]
    }
   ],
   "source": [
    "metric.printMetricQuantity()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "(Optional) Now, ask the test case generator to derive us a pattern which maximally increases 2-projection coverage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nndependability.atg.nap import napgen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimal solution found\n",
      "Maximum possibility for improvement = 2233\n",
      "Optimal objective value computed from IP = 1071\n",
      "\n",
      "for neuron 0, set it to 0\n",
      "for neuron 1, set it to 1\n",
      "for neuron 2, set it to 1\n",
      "for neuron 3, set it to 1\n",
      "for neuron 4, set it to 0\n",
      "for neuron 5, set it to 1\n",
      "for neuron 6, set it to 1\n",
      "for neuron 7, set it to 1\n",
      "for neuron 8, set it to 1\n",
      "for neuron 9, set it to 1\n",
      "for neuron 10, set it to 1\n",
      "for neuron 11, set it to 1\n",
      "for neuron 12, set it to 1\n",
      "for neuron 13, set it to 1\n",
      "for neuron 14, set it to 1\n",
      "for neuron 15, set it to 1\n",
      "for neuron 16, set it to 1\n",
      "for neuron 17, set it to 1\n",
      "for neuron 18, set it to 1\n",
      "for neuron 19, set it to 1\n",
      "for neuron 20, set it to 1\n",
      "for neuron 21, set it to 1\n",
      "for neuron 22, set it to 1\n",
      "for neuron 23, set it to 0\n",
      "for neuron 24, set it to 1\n",
      "for neuron 25, set it to 1\n",
      "for neuron 26, set it to 1\n",
      "for neuron 27, set it to 1\n",
      "for neuron 28, set it to 1\n",
      "for neuron 29, set it to 1\n",
      "for neuron 30, set it to 1\n",
      "for neuron 31, set it to 1\n",
      "for neuron 32, set it to 1\n",
      "for neuron 33, set it to 1\n",
      "for neuron 34, set it to 1\n",
      "for neuron 35, set it to 1\n",
      "for neuron 36, set it to 1\n",
      "for neuron 37, set it to 1\n",
      "for neuron 38, set it to 1\n",
      "for neuron 39, set it to 1\n",
      "for neuron 40, set it to 0\n",
      "for neuron 41, set it to 1\n",
      "for neuron 42, set it to 0\n",
      "for neuron 43, set it to 1\n",
      "for neuron 44, set it to 1\n",
      "for neuron 45, set it to 1\n",
      "for neuron 46, set it to 1\n",
      "for neuron 47, set it to 1\n",
      "for neuron 48, set it to 1\n",
      "for neuron 49, set it to 0\n",
      "for neuron 50, set it to 1\n",
      "for neuron 51, set it to 0\n",
      "for neuron 52, set it to 0\n",
      "for neuron 53, set it to 1\n",
      "for neuron 54, set it to 1\n",
      "for neuron 55, set it to 1\n",
      "for neuron 56, set it to 0\n",
      "for neuron 57, set it to 1\n",
      "for neuron 58, set it to 1\n",
      "for neuron 59, set it to 0\n",
      "for neuron 60, set it to 1\n",
      "for neuron 61, set it to 1\n",
      "for neuron 62, set it to 1\n",
      "for neuron 63, set it to 1\n",
      "for neuron 64, set it to 1\n",
      "for neuron 65, set it to 1\n",
      "for neuron 66, set it to 0\n",
      "for neuron 67, set it to 1\n",
      "for neuron 68, set it to 1\n",
      "for neuron 69, set it to 0\n",
      "for neuron 70, set it to 1\n",
      "for neuron 71, set it to 1\n",
      "for neuron 72, set it to 1\n",
      "for neuron 73, set it to 1\n",
      "for neuron 74, set it to 1\n",
      "for neuron 75, set it to 1\n",
      "for neuron 76, set it to 1\n",
      "for neuron 77, set it to 1\n",
      "for neuron 78, set it to 1\n",
      "for neuron 79, set it to 0\n",
      "for neuron 80, set it to 1\n",
      "for neuron 81, set it to 1\n",
      "for neuron 82, set it to 1\n",
      "for neuron 83, set it to 0\n"
     ]
    }
   ],
   "source": [
    "napgen.proposeNAPcandidate(metric)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now pick an image, and try to find a perturbation to satisfy a specific neuron pattern"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAJIAAACPCAYAAAARM4LLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAFoJJREFUeJztXWuMXddV/tY5933n7fFjbI/rOPEjjhs3NHHTNopQSFBaCgFRoaYSKlIlflAkkPhB1V8gQRWEBPxAAoKIcKSKNIi2hDY0TYILSZqHnaft8fsRv8bjGc/rzn3fczY/7vVZa217xnfsk2t7vD8pyrp37bvPnut993qvTcYYODhcL7wbvQCHpQG3kRxigdtIDrHAbSSHWOA2kkMscBvJIRa4jeQQC65rIxHR40R0iIiOEtG341qUw60HulaHJBH5AA4DeAzAGQC7ATxpjBmJb3kOtwoS1/HZHQCOGmOOAwARPQfgCQDzbqRMNm+6u/sAAAZ6A3seL4V8X/NMGNG1gD+X6+5R4yqToxFdrNnPTkd0EAQRne/uVeOy2UxE+77+eoJ6NaJnLl6I6FKlrMYl/RSvcdkqxevKMQ+hYqFR4zfmipMR7SVyalwqQRFdb1Q1z2NesdKI6ExWz0EeC6OgWtHrEN/3xPlTE8aY5bgKrmcjrQFwWrw+A+BzC32gu7sPv/3VPwAA1NBQvFzXAC+qr1vxMtVSRJ8t1CN6+0OPqXFHnv9uRL95Sm/UzVvviui5memI3vHor6tx2z69KaL7evT3N3X2WET/9Nm/j+g9B/epccPLVvMav64l/kMPrI9oU1QsjJ+ejehfvPnvEZ1b/mlrfv5RnLt4XPE2ZHijvnX0YkRv2nqvGpfp4o01deyw4k3W5yL6n777rY/RBj5xZZuIfp+I9hDRnkq5ePUPONySuJ4T6SyAYfF6bes9BWPM0wCeBoDVa+8wy9etAQBMjJXUuIEhPjECM6d461ati+jMJPPqYyfVuItTLNpmpwqKd+Ikn1DrN3yK58sl1bik3xXRZ89oKX1kz+sRffD4gYjesP1hNa5e4K+hf0CLFBgWnVWjRcrJE29FdEmI5tW9KTXu7Ni5iL53hz6VT7zxw4i+c+MvRfSKvBaBZ84ejehTE6OKl+lavN58PSfSbgAbiegOIkoB+BqAF65jPodbGNd8IhljGkT0hwBeAuADeMYYsz+2lTncUrge0QZjzIsAXoxpLQ63MK5rIy0WoSFUw+Yjq2ZcM5NsLfV2rVGs4Y2sinknDkU0dQ+ocWvXDEX0uweOKZ6f2xbRD9/HesXE+Bk17kfvvhHR46dPKt7YqQ8jWprWPTNTatzMFOt/509qtXH2CM958OCbird8eEtEb9tyT0RfGD2qxiV61/OaPnhN8fae4LVszvI6jpzRltlHh/n7WbXqbsUb9Be/LVyIxCEWuI3kEAs6KtrI1JEot0RadqXirb+TRRuq2vz082ySDy5j8XXg2Ek1rta9IqJtn1WtyHP+7NUfRPTspJ5jcoxFXaWg5zANtslDsAf56O5X1TjPZ5fCrmf/WvM8XkfPui2Kt/JOdsQWCyyiDo28q8at3cYe/dNH31Y8P8/O0EqF1YeJqdNq3OrNWyN6290PKN7yQf1v0w7cieQQC9xGcogFbiM5xIKO6kjdPT145EuPAgDe+YWW+4cOcOBz2dA6xRv02MzPDHI0ve+clvsfjfLrZFKHPqZPsC4xd4wDv/WiDseENQ4lmDBQPIiUG6nF6fAzAOLfZ2VqTLG8JIdICpOaV7jA+llvntc/F+jfe/H9n0Z00u9XvAGP9aLDB0S8NdB/y5p1InMi26d41driY6LuRHKIBW4jOcSCjoo2pLMw65tm5+cDHdH+0fc5/+ZYua54vSvZrO9PiISshF5+SoiUXDqveBVhTlcr7PE1gZVdhiuLLwDC4J/vE5cg5gy04JPJcaWqFiHVGU6Wu5jOMiNjJbaJnKP+tfco3ooV7FLYIPKiBgaH1Djp9njrf08q3tCyZVgs3InkEAvcRnKIBR0VbfUgxGihmd/8hW3rFe9Xx74Y0S/seknxXv7JiYjevpmzeSdGtFd33969EV2emlC8sC7EpZGkJZiEePQs0emnWKR4Iq/cFnmhsJBCS7SFDX5tAi3CGyXmUY2T3hINPQ4JTgFOpPT682vWR3R/D3vKuzLaMqsFnNbbV7YCyxM6B70duBPJIRa4jeQQC9xGcogFHdWRCoUSdr3aTA5b/aXtihcOsdl692ZdATPyPieUvfMT1oNOHd6txk2fP8/z1bReoXQhYq0mkdFugkwve9GTVt2cL3QmErqUCbULwQidRpr7AFAvi+KFojb/G6K+LBRzNEq6kEHWAJYnJhXv/BEuSsjeuTGiZ6o6wd9kWM/a8lldqhRO62S/duBOJIdY4DaSQyzoqGgrz0xh338/DwDYO6hLpXPE5nR2+EHF2zTF4mHfh/8c0YVx67iuiWIwyyaX4iDdy8d6bnCFHpfmKlYpAptTXrnei6y3TZLn8CyvdCKbF7QOGNcLM0wXWZwFlpiuz3Gl8NTpI3otolw8nebAb95yZcxUOGDcEOsFgBXkzH+HGwS3kRxigdtIDrGgs8n/jRr8iaZpOTqt5fDKHtZH+rLaJD81xiGSqQlO3AoqunZePcvTrXEyor1MXiTHUUKPM0q50srP/K2kbF2KXxtbzxKJbamETr7zxWsZgqnOzqhxDaEz1Qva/J88fZDnz7P7YnjVajXOr3KI5NSHOiR1fO4cFournkhE9AwRXSCifeK9ASJ6mYiOtP7fv9AcDksf7Yi2fwXwuPXetwG8aozZCODV1muH2xhXFW3GmP8jovXW208A+OUWvRPAzwH86dXm8qmO/kTz2Hz75/+leL/x5a9E9OTJDxRv3wfcTqYyxybzZW0LhRhJduvkrOwAm/x1EXVv1HRrNzXjZXa9fBT/BhNJuzUOi6XQypWu1fl5duaBnxDd4rr4kE/ZnnMh6oKGXn9QYpE1M3oyotOe5UIQXnQ/nVE8vzqNxeJale2VxphLTpzzABZfUeewpHDdVptpHgvzq6GiY1utXp9vmMMtjmu12saIaMgYM0pEQwAuzDdQdWxbucLsuO9+AMBLB8+rcak8i57xkf9RvKkL7IUNF9iMXorznDN9WrTV62wlTk3IRqJW11IhHufL0QaARJbLyPuX6V6TIVicTU7ov7NcYS+972uRmMlzIloux5ZrKqd7aiZlyVRJd74zQmRVZvh7m8noHHkjRHPW057tWsnOY786rvVEegHAN1r0NwD85zXO47BE0I75/28A3gSwmYjOENE3ATwF4DEiOgLg0dZrh9sY7VhtT87D+pWY1+JwC6Ojnu1kpgtD93wBAOC/96ziHTvLUezCGd1trSQSwEwo6s5IH6gJoUskMlru14UnNxAmuO/pOfJ51k08y4Yol1kfCUQ5t7FM8JlpoYNZOkxOeJuTljyo13hsWehqfs4yz4Xn36vpxLmgzgUEgVjv3LQuD+9fuz6iU1pVw7m5zpn/Dg4KbiM5xIKOijY/mUHXimZnth5tjeLcMQ421sa1yRyoui4WN+RZdWciiczzrd+I9HpnWFTkc7req1vcTdKo6FzpsugCl5TJYKGuXWuIMvC8dQ3FgHAVUKAD14VpDsBW67I7nA5ipzLs5vCKWi4FNZ5TdVZpaDFNhr+7woxOEFy15o6IPrxPRxnmgzuRHGKB20gOscBtJIdY0Nm2Nl4ClGsm268YyCrWgYMsi7sKs4oXCp1DSnr7PjVPJIbZroFkmvWn/kHWb/ykNq1lWES2vwGAhogcdAt9zCcdUiDVP0DrMCR0N9/TiqK8Q82Ih9l/i5eU69fzN0TivhG6W1DRNXTT41zvv2qlbnkDa13twJ1IDrHAbSSHWNBZ0QY071ECsKxX17XNHuK70byUnSstX4t8aCsvW4oGsvOoE6LWbIG/uiFM5mpZm+dyjrTwnHuBlTtOkrT+lnlKxwHAF38P0ZXFOaDFpXfZvSHi+zFyDv2svpV8/0i+R/Pqxok2hxsEt5EcYkFnr9kKDUqlpuhYPqybaPqv74noEixrRogAeczbKdv6KNewRZ1YlXrVqLI4q1ql0knhBU+JSGdolUXJJ4VWr265Zs8SzbLbiVcTQWFji0fx+6cF0u9kX3BrXKPOgdlSRa+jf/DO+eecB+5EcogFbiM5xAK3kRxiQWe72oIwHjb1nzX33K94Qz3fi+iPp7XOYZRZL2DVe4WNQIyzot3aJufPBHpcrcrebDsFPp9jb3ZC1K41Lmt/I9Z+WWncldcBaI94KPU9u67NCA+4sXhaixSTWzV04PVnBjYo3sBy17Dd4QbBbSSHWNDZZqQz03jlxR8CAH7rK7+meBvv4saZx9/Yo3iyiboUWcZKKAtlMphVKi29wSTkjd0stCK92QnthpDebFLi7DIZJV5c5qOYj6Ou9dJ0OO842QDenl+uw/etujafP5fr0YXSabKuF2sD7kRyiAVuIznEAreRHGJBZ83/uQmMvrYTAPBKl066/+wm7mSbeks3Yq/IqLhM/rKT7ius3zSqutYsqaLkrHM0ajrCXxNhkWRe9w9LSj1LvC+7qwGAL5LX6talNnXRRidh9O+4JngNoft4lgpmhC4YWG15QhWCEfNbk8jvo1bQNW8zNIDFop2S7WEi2kVEI0S0n4j+qPW+69rmEKEd0dYA8CfGmK0AHgTwLSLaCte1zUGgndr/UQCjLbpARAcArME1dG1LJRPYMDwIADj0yj8q3swd2yK6apnTgcyBFkeyPOIBIBBe6XpZ51v7ohG7J7y8VWtcIJ6dt5qiJtQdbcK0thqeZ0QHtPqczpWeneRy7lnrvraKyBFP5URpt9VsPRAl1XbHOQkS3xVZCXCh8GyXp3QdYa0wb5eiebEoZbvVAvA+AG/DdW1zEGh7IxFRF4D/APDHxhhV5rFQ1zbXse32QFsbiYiSaG6i7xljftB6e6zVrQ0LdW0zxjxtjLnfGHN/yiqdcVg6uKqORM1YwL8AOGCM+RvButS17Sm02bXNT2aQW7EZALC+dkjxRt75cUQXa9pkTouatITMTLTueg1FuKM2p5ucy/to/RTrB4F1XXtWtJ3J5XTtnTdPpqZntfDLdveJcfqgnhNdeevW+nNd/OzeAb5sJxnqME5R1P3ZoSAVTkqJmj2rF8JcledMZXRtX3lGf3ftoB0/0hcB/C6AvUR0qYrxO2huoOdbHdw+BvA7i366w5JBO1bb65i/L6fr2uYAoMOe7SAMMde6h8zv0kbe4ADff1E8p9usyFLpXJrFjd3hNhDebPv6zqowf9M9LHq6+6372oT32i6HNirV7cpiDgB8scZuq2Q7181eYztxTt5FYoSYLl0cV+MaC93BIuZI5bl2MLAyCKpz7NGvdenMgJ6exfuWXazNIRa4jeQQCzpbsk0ekGhZYEZ7ZBNZPoY9X3taG6LBJrr52PWtpDQTyIQv+/rOKTGQhVG6Vx/jygILLaHlXZG8PC9bNn23LDrpHbfnl575orhmtTaru7Po4jgdMPZEI/neZaw+VOf0dVyBKAmvFbR3v75Kd5lrB+5EcogFbiM5xAK3kRxiQUd1pIRPGOxr6gxzRa3frFrH8nz24gnFuyAs+YG16yK6Pq7dW9MXRJTGakkj7zWries7w7o2pRvCZE7krctkMvN4isn+PcoCBbv2Tq5De5DLUxMRXRfrv+yaeHlXnHUdfH6IO9IOb+CMiosnRtS4WZFUl+tdpXi1iqWTtQF3IjnEAreRHGJBR0VbKtuFddseBgD0WAFRUxBXjE7pRIKL+09H9MrheyM60afzvk97ByJ68rz2jhuRz21EPrTtAQ9EchlZN1j7Iggqg8B2expZRh1ad4U0RFNQO99a+xGu3KUOADzhOc8u041EhzZ9LqLX3c2tg7IN/bccPsXNSKfHTimebwVx24E7kRxigdtIDrHAbSSHWNBRHcmEQKSq1HRSvBFm+Mo1mxTPG2EZ7vdxC5YH7tXtAz0R/Z+c1DqBTNanGj/LWBkERtShhRWrt4BoC1iX9f1Wa5mF6vsXanmjOt5IEz/XpYZlRegj26/DGbUKuxAKBXHvmlXXNlu8yPNndB3bwKDVwL0NuBPJIRa4jeQQCzoq2shPITOwBgAw1KuTqWoiWSucPqN4Kf9nEV0XV6F3Da7V84fsKS5b0X9KsInueeIekaSV87xAaxy7uyy/b78hnmsPFiKGrE5vXorr41Jd7FVP9mjRs2Ltloju6dUJgqUSJ8GdP300ohOWmPbFFe3Jho7+z81otaAduBPJIRa4jeQQCzpstTXQKDWthWJoNytnb21vn76npDfFImBqWlh7ni6VTonkuGXL71K8qYt8i3dZSKzeXi02/EB2+pj/BmvZNQ2BnX3NIKtTiSfKr/20dcVXkr8DWcZkfD0um+Df/5x1jWggjgZT5OBrNqX/qZMJnjPTpa3C+ozuTtIO3InkEAvcRnKIBW4jOcSCjupI5eIsPnjzZQDAlq3bFa+nm8uVYfSy0iJ5a+zjvUxf3KjGeWk2mat1rTtUZRl4gtvVDK3TF7g0qqyDTV04q3j1svCIX9kT0FoI/z7TGZ2hkEnyB7201k0SQn8KhA6W69a1d6Uim+eV0LrzLct6Yz7LetDkuHapFIqc9eDltHfcS2u9rh2007EtQ0TvENGHrY5tf956/w4iepuIjhLR94lo8bfFOSwZtCPaqgAeMcZsB/AZAI8T0YMA/grA3xpj7gIwBeCbn9wyHW52tFP7bwBcyjpLtv4zAB4B8PXW+zsB/BmAf1horkq5hIP7m30oPOt267Wr2EtdK5xTPFlGXTzLXUxGRvarcWOn2JNbKWoTVtbGJbJsZvspnfPsQYyz7ixtCDPfE0HVYmlOjcuIRqvLV29WvOFlXEc3euGo4oVCTJkKi55yWc8vr7ObtURWI8liO+jnZ1FV57A3RITAC3WOdq26+D5W7fZH8ludSC4AeBnAMQDTxphL3/oZNNsBOtymaGsjGWMCY8xnAKwFsAPAlqt8JILs2BYErmPbUsWizH9jzDSAXQA+D6CPiC6d/WsBnJ3nM1HHNt93HduWKtrp2LYcQN0YM01EWQCPoalo7wLwVQDPoc2ObUFQR3Gyqf9MXNCyPSxxbX5lVrdxKYjE/coUJ2Ttf/dlNa42zXpRtW5dtS5CDqHQP84eP6bGpYl1pJmijor3LV/Pc5TZBK9W9TjpGSiXdXHBREmY+A2d/C+T6uoiPFMsaR2mnmc9iEib6tVZ/h7PC3dFOqXDSTLcU6ro9VOwePO/HT/SEICd1FyxB+B5Y8yPiWgEwHNE9BcA3kezPaDDbYp2rLaP0GyJbL9/HE19ycEBNF+y1ifyMKJxNPtNDgKYuMrw2wU3+3fxKWPMVfvcdHQjRQ8l2mOMuf/qI5c+lsp34YK2DrHAbSSHWHCjNtLTN+i5NyOWxHdxQ3Qkh6UHJ9ocYkFHNxIRPU5Eh1o5TLfdRYFL+TbOjom2lmf8MJohljMAdgN40hgzsuAHlxBat0gNGWPeI6JuAO8C+E0Avwdg0hjzVOsH1m+MWfASxZsNnTyRdgA4aow5boypoRmje6KDz7/hMMaMGmPea9EFAPI2zp2tYTvR3Fy3FDq5kdYAOC1e39Y5TEvtNk6nbN8AXOttnDczOrmRzgIYFq/nzWFayrie2zhvZnRyI+0GsLFVfZIC8DU0b6G8bdDGbZxAm7ldNxs6Hf3/MoC/A+ADeMYY85cde/hNACJ6CMBrAPaCr2v7Dpp60vMA1qF1G6cxZvG9ZW4gnGfbIRY4ZdshFriN5BAL3EZyiAVuIznEAreRHGKB20gOscBtJIdY4DaSQyz4fxIjw3790krJAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 144x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "dataiter = iter(loader)\n",
    "images, labels = dataiter.next()\n",
    "\n",
    "util.displayGTSRB(images[0].numpy())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here, we want to control neuron 3 and neuron 4 such that there sign is positive (activated) and negative (deactivated)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nndependability.atg.gradient import gratestgen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 0.0000,  0.0000,  0.0000,  0.0000,  1.5866,  0.0000,  0.0000,  0.2675,\n",
      "          8.7566,  9.3015,  0.0000, 20.9700,  0.0000,  0.0000,  0.0000,  0.0000,\n",
      "          0.0000,  0.0000,  0.0000,  0.0000,  0.1225,  0.0000,  0.0000,  0.0000,\n",
      "         13.6080,  0.0000,  0.0000,  0.0000,  0.0000, 18.3025,  0.0000,  0.0000,\n",
      "          0.0000,  0.0000, 10.2891, 12.4542,  0.0000,  5.4251,  0.0000,  0.0000,\n",
      "          0.0000,  0.0000,  4.2513,  0.0000, 11.0453,  7.3342, 21.0321,  0.0000,\n",
      "          0.0000,  0.0000,  0.0000,  2.0281,  0.0000,  8.6155,  0.0000,  0.0000,\n",
      "          4.3820,  0.0000,  0.0000,  0.0000,  0.0000,  6.8748,  2.7324,  0.0000,\n",
      "          3.9678,  0.0000,  0.0000,  4.7795,  0.0000,  0.0000,  0.0000,  0.0000,\n",
      "          0.0000,  0.0000,  0.0000,  0.0000,  1.2829, 23.2207, 13.6945,  4.0878,\n",
      "          4.3188,  0.0000,  0.0000,  3.0776]], grad_fn=<ReluBackward>)\n",
      "tensor([[ 0.0000,  0.0000,  0.0000,  1.0000, -1.0000,  0.0000,  0.0000,  0.2675,\n",
      "          8.7566,  9.3015,  0.0000, 20.9700,  0.0000,  0.0000,  0.0000,  0.0000,\n",
      "          0.0000,  0.0000,  0.0000,  0.0000,  0.1225,  0.0000,  0.0000,  0.0000,\n",
      "         13.6080,  0.0000,  0.0000,  0.0000,  0.0000, 18.3025,  0.0000,  0.0000,\n",
      "          0.0000,  0.0000, 10.2891, 12.4542,  0.0000,  5.4251,  0.0000,  0.0000,\n",
      "          0.0000,  0.0000,  4.2513,  0.0000, 11.0453,  7.3342, 21.0321,  0.0000,\n",
      "          0.0000,  0.0000,  0.0000,  2.0281,  0.0000,  8.6155,  0.0000,  0.0000,\n",
      "          4.3820,  0.0000,  0.0000,  0.0000,  0.0000,  6.8748,  2.7324,  0.0000,\n",
      "          3.9678,  0.0000,  0.0000,  4.7795,  0.0000,  0.0000,  0.0000,  0.0000,\n",
      "          0.0000,  0.0000,  0.0000,  0.0000,  1.2829, 23.2207, 13.6945,  4.0878,\n",
      "          4.3188,  0.0000,  0.0000,  3.0776]], grad_fn=<CopySlices>)\n",
      "0: 0.0, 1.5866172\n",
      "Found an image to successfully create the required pattern:\n"
     ]
    }
   ],
   "source": [
    "targetedNeuronIndex= list()\n",
    "desiredNAP = list()\n",
    "\n",
    "# Specify the target - we want to contril neuron 3 and neuron 4, with sign being positiove and negative\n",
    "targetedNeuronIndex.append(3)\n",
    "targetedNeuronIndex.append(4)\n",
    "\n",
    "desiredNAP.append(1)\n",
    "desiredNAP.append(-1)\n",
    "\n",
    "# Trigger gradient-based test case generation\n",
    "new_image, testcaseGenSuccessful = gratestgen.generateTestCase(images[0].unsqueeze(0), targetedNeuronIndex, desiredNAP, net)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "neuron activation: 0.32120165, 0.0\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAJIAAACPCAYAAAARM4LLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAF2NJREFUeJztXWmMHNdx/qq759yZPbjLc0mKokRTomjLhyyfcQLHCuQktgM4CGwDgQMYyI84QALkRwz/SoAczp8kCBAkURDDShDYlpNYNhz5UGwJimxJlmjZIimK1JLitVxyd7nX7Fzd0/3yY0ZdVS8mOUu2huTyfQDBmqk33W9637w6XxUZY+DgcK3wrvcEHNYH3EJyyARuITlkAreQHDKBW0gOmcAtJIdM4BaSQya4poVERA8S0VEimiKiz2U1KYebD3S1Dkki8gEcA/AAgLMAngfwSWPMy9lNz+FmQXANn70fwJQx5gQAENFXAHwMwCUXUrE0ZKrVUQCAgV7AnsdTIV9vlGSSlI5ifr9UHVbj2gszKV0P7XsXUjqO+SJD1RE1rlQqprTv68cTR+2UXr44m9KNVlONy/n5lC6Pb1G8Spl5SBQLnZDfWK0vpLQXlPX1A0rpqNNWvILHvHqrk9LFkr4GefyM43ZLz0M87/nzp+eNMRtxBVzLQpoEcEa8PgvgXZf7QLU6io9//PcAACE6ileujKW0P1ZVvGK7kdLTNf7cve9/QI2beuQvU/qZ0/qvtHffnSm9uryU0vd/6CNq3D1v3pPSo1X9/Banj6f0d/7t71P6wCuH1Lgd49t4jp/SEv/979yV0qauWJg/s5LSP3rmayld2vhmNW5yAy/GmYsnFG+3+ME8O3Uxpffue4saV6jwwlo8fkzxFqPVlP6nv/jsKfSBN1zZJqLfJaIXiOiFVrN+5Q843JS4lh1pGsAO8Xp77z0FY8xDAB4CgK3bd5mJnd1f6/yFhho3tpV3jNisKt6OzXyb/ALz2jOvqXFzC3z75UV9jddOsijdtfu2lC6Uc2pc4FVSevrcK4o3deCHKX30tSN8vXs/oMZFKzyPsQ0lxSPDorOdaJF46rVnU7opRPPkaEGNm5nl6+9/p96VT/7o0ZS+Y8/bU3rTkBaBZ6anUvr03Izilapr15uvZUd6HsAeIrqdiPIAPgHgm9dwPYebGFe9IxljOkT0+wC+C8AH8EVjzOHMZuZwU+FaRBuMMY8BeCyjuTjcxLimhbRWGOMhNF2dJMScZubelJIjlUnF2rmXdST/xNGUpsq4HredraWfvPK04nnl/Sn9gbezXjE3f0aNe/T5p5h37rTizZ5h66zeiFK6sjyvxi3OL6f09NRJxVs+9mpKHz38rOJt3HlXSu/fezfP4/yUGpcfZh1v7qf6ex46sZjSe0ushx47oy2zl46xBbply92KN+GvfVm4EIlDJnALySETDFS0kYkQNHse4dJmxbv9ThZtsLzS+Qp7sDdPsNg7NKWdca0hFnXNek3zhCPz24//R0rXlrS/ben8Wb7GqvZ7mYTFGYgf3WsHnlTjcoFwCn717/Q1DF9jeHKv4m3aza6H2jI7E48c+rEat30/jztzXPOCMj+fVotF7tyiFuFbhejcf/c79Twm9N+mH7gdySETuIXkkAncQnLIBAPVkarDw/jghz8EAPjxjw4o3tEj7Muc2LZT8TZ7rPsUhfweO3tSjfvZNJvr+ZyveLWTz6X01GusL9mRb4qZl3RiaIjQgUi/iT07k4HDLrXVeYvH+lN7UfNWZlmPqZR4/osRqXFLL3AAoZDX+sw4LqT0kUNs4vtWutDkbZz14JXHFC+MdPiqH7gdySETuIXkkAkGKtpQKCLZtQ8A8O44r1jfeIRN8uNtnas0uo2Tw8Z93vKpUFTjynkWKdWizmlqiUSxZoOj7p615RuRGBaQfjyUsKgz8jfo6dynUIyLrZ9q4LMobYc6Q6FdZ5N/Qdw7CnT0v1HhjIKRHVqEb5pgs/72ERZ7E5u2qXErS5w58cyTOoti28QE1gq3IzlkAreQHDLBQEVbFBucr3XFynvv2aV4v/KL703pR3/wHcV77Osc6HzHvvel9MUjz6lxhw5yULW5pIPCScyJXZ5h8dWMtBs9ElKqXKooXqU8lNKxsOD8nH6MUcjX7ITaKowNi+0g1FahSfjmJERbsaittoI3yryyVhEqO+5I6bEhTqetlkfVuI5hz/9YU+cjrsxpL3g/cDuSQyZwC8khE7iF5JAJBqoj1WoNPPH9lwAAkx++V/GSLZzUdfebtDl68IUXU/qpR19K6VPHdOS7doEj936ok9194ek2xPpNPj+kxlVH2IteGN6geJUy6xwk3AZJos3/Tod1pKStE/zDGie9RU09R7T4dYdYlyLLRZGssDsgmF9QvLPH+MBCsJv1pdX582oclTal9N336b9FvHQWa4XbkRwygVtIDplgoKKtubyIQ9/+KgDgpQl9VHqIWPRUbnuf4u1f4mSwI1//Z77e/Kwa54csDjzSJnMu4OsHIte7MrZJjfOEt7wTaQ+7NMnz4npRW4uojvBEB4H2PHt5cTS9Znu2+aStEaeL43akxwlR2jih59hKxBxL7BoYCfSesTLDwV3bc76JtDjuB25HcsgEbiE5ZAK3kBwywWCT/zsh/PmuO35mUSdPbRpmnWa8rCP303OcsLZ8kUMfUdNKShOhD+PrM/1D45wUP7SRswk8q4RO1OawRcsqGdMROlhR6BxaSwH8HN87X9Dfxetw5D7xrLoABdZpOuLwQrSyrMaFwk0Qta1QEDhMdGGY9dDyFl1eJ2jy+bdTP7ugeMdr57BWXHFHIqIvEtEsER0S720goseJ6NXe/2OXu4bD+kc/ou1LAB603vscgO8bY/YA+H7vtcMtjCuKNmPMU0S0y3r7YwB+qUc/DOBJAH98pWv5FGEs6G6bzz35LcX76K/9ekpfPPmi4v30RT5G3RAms2f9DGSlscKoNusDkesdCs+zaerofxCzFznXtszzGo9dEs7soTFdOa5QFlXfLDdEKJL2Om0d/Q9yfJ3CCJvkHbu6XZ3VgkaoveqmxXNenuFzfxd8LaYj4V7wA50g6LWXsFZcrbK92RjzelGd8wDWfqLOYV3hmq02061mesnKTLJiWxhFlxrmcJPjaq22C0S01RgzQ0RbAcxeaqCs2LZt8yZz/9vuAwB894iuEpYf4nqNsy//QPFmz/PYlkgUs73XxTJbKUNWEdBmzIt4+TxbJXFLi6/xKltSZU97fIMmb/kNIaJKG3VVFIR81PvVGZ0ktrLAIqVoBYwnJvgZVIfZ2guqlvgViXN+qK3fuMle6eYcf88FK/nO8/nZlSr6e5qGVSW1D1ztjvRNAJ/u0Z8G8I2rvI7DOkE/5v+XATwDYC8RnSWizwD4AoAHiOhVAB/qvXa4hdGP1fbJS7B+OeO5ONzEGKhnO1esYOs97wEAeAf+VfGOn+OqZLWzOrGt3WA9wDO8ieZy2mzNDbNftFjRBcplEr7piKQ06GuUSpwkX85p3SESHuWYRNF363h4Y5U90caK3HviAAEsHa/e5CQ1P8dzrJZ0gn9JJNgllgHTFEXl4xbrao0VrcaObeeqb7mctpVmVrUnvR+4WJtDJnALySETDFS0ebkihjZ1K7MN690aM8e5AHo4r10DnvBYSylC1s+gVOFzaIGvxU2rxSZzLJLoqiM6qArhQlhp66pvK+JMWmGExWhsJba1GixSNozqNhRj4vphy+oBIoqari6zeCkEOne8IILafkMnoZGorkIibz2JtfgiiIopyzqfe/PkrpQ+duin6AduR3LIBG4hOWQCt5AcMsFgE9u8AF65G9/dPK6Tul45xufVyqsriheK/mphLM7Hezp6Hnb4d+FZpXGkkVweYZ0jP6RN/FWRzLY4p3WHRGQK0Bjfe6GmK68tt3n+wyNav6mOsn5jmlpRXBRlbmp1ccYNVpJent0GOSsxzxeVd+NYlABq6Qq9tTk+u7Zp41bFM2QpsH3A7UgOmcAtJIdMMNiKbQDQK9y5YVgngy0e41xjq5gbQmG6RsLmJ0+b+LK9qd2qN59jUVoosGiwE88iUYw0ZyVjN0UVuNwQm/HxgjbBO5Gcr76GL4SssfwXBfG7bumvpkDCo2+sviEdUS0OqsC8nsjwZj4iXx6y8tbhRJvDdYJbSA6ZYKCiLUkMGr0KHBM77tG8p55P6ZVEB1KlCJB2Glne2pbw6voFbRXKOtPyerYIbAgvdWwdaSqIoPCoOGZEw1q0+XkxX0v8yhhuzk46L3BA1zRFLXDr6LgnanDL6nMAkJfXFDRBj4sj9pw32nqOY+O7sVa4HckhE7iF5JAJ3EJyyASDrWoLwlzS9SRvf/O7FG/72JdT+vSyTtbKFdkczYtErqZVKa0tkuIrlrlbCljfEXltSEjrQaE47xVZ1daH85xdEAX86IIhncSfCLN72XIh1FZ4ztWcnn8kTPdQeKzL1u/dyGpx0NeQKplM8CfSyqAvPlYcu13xNtiHGfqA25EcMoFbSA6ZYKCibWV5CY8/9nUAwMc/+hHF2ytakZ784QuK54lEtKIQUYFt+kJu5VZ/EFFtLRSB2WZTm+61VfE6r10ISZ7dEvVEVICDDvyGFXYNtHRuHJoRuyhaseX2rjMvSXgeHegoQDtk0dm2Cs6Tlm2Cof/UsQh4l6r6oHSB7PZiV4bbkRwygVtIDpnALSSHTDBY8782j5mnvgQA+N6Qrmr7tju5Zbj/9LP6cx3Wd3wR3gisPmkQ5+A9q8WoinZ02CaPmvrsf01WgSvqOXoiVJFriCS3jjatOyKDoN2xTHdhkudJhyakZpK0RAKfngZaYo7/L3wino8vD0B4lj4p3BdRXZ95W/Z0Ml4/6OfI9g4ieoKIXiaiw0T0B733XdU2hxT9iLYOgD8yxuwD8G4AnyWifXBV2xwE+jn7PwNgpkfXiOgIgElcRdW2Qj7AHTu7bS6P/s8/Kt7SLs4GsKsodYQnV3poyVheXdGTrWOJtlh0vo6lSDRaNORLbOIXy7pfWyCOi7elF9o6Nh22OT86ibUpPVRkV4Gp6/NweWHy54o8j7xvic4al9dJIn0NkwgRVhRZAoH24HcM/+nrCzo3PVy5ZJWiS2JNynavBODbADwHV7XNQaDvhUREFQD/CeAPjTHqmMflqra5im23BvpaSESUQ3cR/bsx5r96b1/oVWvD5aq2GWMeMsbcZ4y5L5/L/bwhDusAV9SRiIgA/AuAI8aYvxas16u2fQF9Vm3zc0UM9ZLOd4dHFe/Qc/+d0ivWmbRikaPrgQhbe5HWkWJx5q1d06VZymIRF2RDGqvZy8YtfMartEG3LW+J0AQ1+HoetJ4SGA5pUKBDGFVxJi0O9fcsDXNJHU8Ucyfoa4QRuzmMZ7sehNtAmP+BVXpntf3z9TEAaC2vvaxNP36k9wH4bQAHiej1igKfR3cBPdKr4HYKwG+t+e4O6wb9WG1PA1bCL8NVbXMAMGDPdpwkqK12w+G+FXHeMM7JVItndS+MltiufWHGhsYq6SJM7SDSIitucBg+ECVphid02RkSvcuMlZwf5EWPkYR5DWgT3yuwK3q4pK+RL4gshEh71XM5TuAzohdJZ1mPI9mTzdfiPRHiLF/lZ2osdTisi5byVe0qGRleu2/ZxdocMoFbSA6ZYLBHtskDgq41YoyVly0qmeUD7WmNRJDVG2HLRiZ/AYARItBYLbJMi11fUoSQ5ZIIOywaEl+rhuWSyNMW4qxQ0I+RhPhNrO8Zihxr3zpubYSXurUsume3rGclP2cdzDN5Tqqrjoj+K6G2xNpi/uGKLvoebdHivh+4HckhE7iF5JAJ3EJyyAQD1ZECnzAx2tVPVq3I97bbtqX06sWTijdbY11l++13pnR7Vk//4jnWrToNLfc7Qs9oiqyBdl33JkuE6a6KqwMIAjan5eECz9KlinnWWxotK/EsYt2kvbioeGGd9aKkIQrMJ1pHSsShAS/QJWhGt3Ih9m2796X0wtljatyySMYrj2hXTLulK+b1A7cjOWQCt5AcMsFARVu+VMHO/b8AABit6KQxiBajWNBdnxcOcpftbTvfktK5DaNq3JToMH3ulO5nUhe9sAtNFnv1SBfpjIg9yklgnVcT3a7LJcGzcral0O5E2mssW4Am1pk0iGS5WIizyBpnfL53dUznV0/c+Y6U3n7X/pQukTb/myenU3p5TveU8ws6iNsP3I7kkAncQnLIBG4hOWSCgepIJgFaPRVhKdSH4mORMD+xWZdZMQe5l5sZ2pnS79h/t76BaPCyMK+LqMeiRbsnQi652AqzCPM87mheKMId4bIR47QOE3VEApxVuTYRGcmelZ0jM0hlGlq5qqPxhTFuRZ8f1uGMRoO/d11kEJAVSllZnUvpoKT1rA0TuoB7P3A7kkMmcAvJIRMMtheJn0dhwyQAYMuoZVqL8jJm8azi5f3vpbRqAToxqW+QsEc2tHI6SZi0oUgG8z0d/Zet0L3EKrcmz9TJz1jVb2WeWxBYx7JFlTljHbzxRRmdSpXdI4Wqzh3fuG1PSg9VtFd6tcFnMM6ffpWvYeXB53x+HoHlAqktarWgH7gdySETuIXkkAkGbLV1EDe6gcmG0Vu+n+PO0WMT2ooYFwlli4u8DRurZ4ZfZHEwNq4tv4UFtvxWxTEmOz+5GAsLLLY8yvL4kBH54YmWo0b0CvHyVl8PUUUtyWvxTuK1SuazCqYWxRGq+qpu2xoLKzFssRc9yOs/dRCIo+lWMdVw+Q0+su3gcCm4heSQCdxCcsgEA9WRGvUVvPhM15S/6557FW9kmBPKyOhp5UqsP507dTClZy7uUeMoxzpSK9QHCFrC/DWiCu3WHbqBi4nYDbF0QbshWuLotJGRestr7InzcLmi1sHyopqsX9St4n2hT3WE62GoskmNa4gEuFZinb0rswuhWODndvGCjvDXpNe7pN0LVLxMs7hLoJ+KbUUi+jER/axXse1Pe+/fTkTPEdEUEX2V6CoaoTqsG/Qj2toAPmiMuRfAWwE8SETvBvBXAP7GGHMngEUAn3njpulwo6Ofs/8GwOtZZ7nePwPggwA+1Xv/YQB/AuAfLnetVrOBVw5361B4eZ08Nblle0qHlklLIi+5Ps1VTF4+fFiNu3D6REq36zo5Lm6zWMpV2b1QLFkJduKJ1K0qZ3URFPaFb7vR0AHoYoXF2fi2Nyne5Bgn483OH1c8EjeXOeayPwoA+KKHSW1Wi9+O6COXjAs3Smgl2An3u5foHO12e+11rPqtj+T3KpHMAngcwHEAS8akdfPOolsO0OEWRV8LyRgTG2PeCmA7gPsB3HWFj6SQFduS2FVsW69Yk/lvjFkC8ASA9wAYJUobXGwHMH2Jz6QV2zw7uOmwbtBPxbaNACJjzBIRlQA8gK6i/QSA3wTwFfRZsS3pRKgvdEvWzM/qdZc0+HxZa2VO8WoiWb8lItOHDzyuxrWXWC9qhTopTZa4NG3WaaaPT6lRBWGer9S1bjIywWfGkuZFcS89zoheca2WjqwvtFjfSzrWmTcj+ukKnWalod0LkTxvZ/04WyvsGphp8TUKVjgmFvU86y09f4rXbv7340faCuBhIvLR3cEeMcZ8i4heBvAVIvozAC+iWx7Q4RZFP1bbS+iWRLbfP4GuvuTgADJ2v/I38mZEc+jWm5wAsPbsqfWJG/1Z3GaMuWKdm4EupPSmRC8YY+4b+I1vQKyXZ+GCtg6ZwC0kh0xwvRbSQ9fpvjci1sWzuC46ksP6gxNtDplgoAuJiB4koqO9HKZbrlHgeu7GOTDR1vOMH0M3xHIWwPMAPmmMeXkgE7gB0OsitdUY8xMiqgI4AOA3APwOgAVjzBd6P7AxY8xlmyjeaBjkjnQ/gCljzAljTIhujO5jA7z/dYcxZsYY85MeXQMgu3E+3Bv2MLqL66bCIBfSJACZOHxL5zCtt26cTtm+Drjabpw3Mga5kKYB7BCvL5nDtJ5xLd04b2QMciE9D2BP7/RJHsAn0O1Cecugj26cQJ+5XTcaBh39/1UAf4tuQbIvGmP+fGA3vwFARO8H8L8ADgJpd5vPo6snPQJgJ3rdOI0xCz/3IjconGfbIRM4ZdshE7iF5JAJ3EJyyARuITlkAreQHDKBW0gOmcAtJIdM4BaSQyb4P03FEU/+KKq6AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 144x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "if testcaseGenSuccessful: \n",
    "    util.displayGTSRB(new_image[0].numpy())\n",
    "    out, intermediate = net.forwardWithIntermediate(new_image)\n",
    "    print(\"neuron activation: \"+str(intermediate.detach().numpy().squeeze(0)[targetedNeuronIndex[0]]) + \", \"+ str(intermediate.detach().numpy().squeeze(0)[targetedNeuronIndex[1]]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAJIAAACPCAYAAAARM4LLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAETdJREFUeJztXV2IJNd1/k5V/8z0zppRElnoz44hwsYvsUEoDvZDsCMQeVEeQrAMwQGBXhJIwA8WfoogAeXFyVtAEJF9CJEFMtgEgzFGwQ4ER4rzY1vGtiIQXrGxouhndzUz3V1VJw/dO/c7p6Z7e7qvuntnzgfLVtW9detWzel7/s8VVUUgsCqKTU8gcDYQhBTIgiCkQBYEIQWyIAgpkAVBSIEsCEIKZMFKhCQiD4nIT0TkZRF5PNekArceZFmDpIiUAH4K4EEAlwG8AOARVX0p3/QCtwo6K9z7AICXVfUVABCRZwA8DGAmIQ0GA93f31/hkecHQseb9D1cuXLlDVW9/Wb9ViGkuwH8nM4vA/iNeTfs7+/jscceW+GRHjL31EBnnsy5Rf0FepbQ4ewHi1vxzRBzJiySenqm0fCF1ieY9xFOjyeeeOLVRfq958K2iDwmIi+KyIsHBwfv9eMCG8IqK9JrAO6l83um1wxU9SkATwHAXXfdtdBS0JLbhNvo2K0YBf9Qi+V+I3bRcWPIyQyncfPlXn6xKnRBpsXv4gYxI8z5VubhrcU1Xcixhq2yIr0A4D4R+ZCI9AB8FsDXM8wpcAti6RVJVSsR+WMA3wRQAnhaVX+UbWaBWwqrsDao6jcAfCPTXAK3MFYipJww8o7XRFhDMve4ftzWkh14DO7Z2Hk0dF6Us8eoucGOUdep0ctPZUnPtrcZkUYk/WkKJ+/Nnj2gM75Qy1w4W3xaSmYKF0kgC4KQAlmwNayN2U1LZaalnVlW0zhjnzZ0bMcwaj39fLyKX/KzWot+Oi86qV/huhWd9FlLN0ZdVzxh01bR+6jQu8yxuxaO/ZpPMs+6oJ4p8gNOz9xiRQpkQRBSIAuCkAJZsD0ykjHtz+nIVgLvECXVvSVnkSxk1OzSyUhlkjlaYgTdV8y4DgAdUt1LaQlQx4c1KtOkOjo+rjS1jUfOBUNzLlsfK51rnV6gLP2aweerxxfEihTIgiCkQBZsjrXNYV+tmBo65SVaC+cVb9hc6/gS6egVNXVqy14qWuY7cJbtmlVyZpVejR+nW9xPtUOsEz07fr+7k8Yc8ryMGd1ENrRMFPRuDVvpW94CPl7d/x8rUiALgpACWbA51jbH8tyoXcq1Tq1lSc5MH3jG1vHW42gMWsqPqqHpNxomdrDb2TVtO8SWqjrdV3S6pl/N8x861knsrCv2vg5Zy7u9XrruuDSz9Mpbx8lhbDRcH6OHeSbwsGwHNoQgpEAWBCEFsmBjMtJc47WTCWpSY8ekWnveXtCoXaeSd0m+qUmukMrOpCdk2a5mW5SNaWDsZDpzYsdvDpLMNHJhA0pqfcNmgq41E6iwjGSaUDdkGqBvIO5bzVX5lzB0x4oUyIIgpEAWbI3TluGDtXqk7pJxuRW8VtBSXjhrcMlshHjnha79BLzij0cj06YVqec0x8qZEGoyLxeOxSq11WNrGhjRz7rT8LzsHGty/FaNixcnltshU0kjzjre8BxNUyuQbhHEihTIgiCkQBYEIQWyYGMy0jwNs2x5qjnYjEwB8yL8nZxV0G+mpOB8Ke0Y46MktwzHVkaqxunZfZpk7VVrkrv6OwM7DzIVHAzHpk1IrjOv4swLDcuMlZWRKKYOHTZXOHXfBErMqZiyKG66IonI0yLyuoj8kK79koh8S0R+Nv3/tiWeHThDWIS1/R2Ah9y1xwF8W1XvA/Dt6XngHOOmrE1VvyMiv+ouPwzgt6bHlwD8E4AvrjIRXnlbGVecy0bHheOBFXnduy5GuSJW1yEPuahlX4OCIw3sTK5fu358PGrSGL0Lln31BylqgFXwyUTSYddFOQhZs/k2n4NWEmtjtg8AtWFhlDruVXrO7as3F9h2h6pemR7/D4A7Vp5J4JbGylqbTiS1mfJZVGw7H1hWa/uFiNypqldE5E4Ar8/qOLNi2xzVwC/DNZmza1quC7es98oUKFZ41iZs5U0P77mIr4u3pbjp9xe/bNquvvHG8fGbb71zfNwfXLDzJZZ79eiaaRsfpHfpuoC4vZ3+ifOvqkPTTynOXJwZumjSNzEp7S78nAucqHfgLlHpeNkV6esAPj89/jyAry05TuCMYBH1/x8A/AuAD4vIZRF5FMCTAB4UkZ8B+O3peeAcYxGt7ZEZTZ/JPJfALYztyWvjamie75O6q3Vi9r2ib/qxHNDpuJw3DrQnIaxyqu+oTp+kU1oZpthNstDeMN23M7BJAtpN8xrru6btuhylZ1XW+19QQsFeScH/Xr5pOGDNMpVRcXJpH64AN22dcbxUVZvwtQXyIAgpkAUbDGybnUtVuFjmkngWB6yJC9bqkDrtK5WMDxPbGJOluOcs228dpPN3nN2ry3lnFxI7q0YuZlvTfT0X2LbTTWOMrM8W9VG6cEBsr9+xv3dTqNQ7uClmuyAfQSOz2VdL24+Y7cCmEIQUyIIgpEAWbFBGapVUOz5se//TFa782lRWvukqlZpRq1oPKYisod9P03NBXRTwPx7aoP59ClI7atL49ZF1YWA3yWoX9myo1m3d1DY+sm/akOujJhmpdr/3HlW8bVxiW0Hz4tI7ndaaQd/xhEoJp0WsSIEsCEIKZMEGY7bt8tkQ+2ocfStbornYphuzoGAzX0Sdg8HY6N1R+wl4md9xFdVqsnQLmSRGzjtfUGz30JmJezRHVfeenErui5jyHClIr3EFTWv+BvQRpHCld5qTLeDAcrtQxooUyIIgpEAWbI1luxpTzJtzUnK5Mbu7ldV6RqRxle43UnDcMwVEO+MyGmJZvZ2eaVPyng5I47p40e4cLgNy2rrgtdFhctq246hTUF1Vp36lSznirbpqx5a6FLFWmsBv26+tqVHXNQa2BQIGQUiBLAhCCmTB+mWkqTosLhqdM6wLX4idrMFcHe2wcSnPVWorXT7ZDnnQTWmccsf0q6skm9Su2joHkR2Qer67a/Pa0E/PvurkuGtX03lHbBtHF/CeKF7eY6iTddjMUdJ4TauMMKv/q68nsSIFsiAIKZAF62dtU9XS7xxdksrZ2kaErc0dimV2GmwjycrrN8geU8DXqCbHrFrHLM9q5HLeOE1MK7IuOwt498Le8fHwTRsc939H6Xm+6sqA2Nn7BsSiCvtnGlPKuS/Y3icTBW+zyhZvAGga9hA4x29sRRrYFIKQAlkQhBTIgrXLSDf4cZtnUx9voR8mmabTS+6HHVeRtqqSOaDlPK9T25if3Vj55ojC6g4K61kfU+BZc5jG6x7tmX4XWQZzfpCS5swyEQAM2L1Rc4Ca6YYhuUz8VqpSsIzEEf4+V5Bk0mXK2DoskrJ9r4g8LyIviciPRORPptejalvgGIuwtgrAF1T1owA+AeCPROSjiKptAcIiuf9XAFyZHl8TkR8DuBtLVm07Dppy/Mssrl7/p64NWbMLl4bMllzvwR5SbHNNdoN+x3r4ubD7yE1jlIzeGJLb/frouul3gNRxsGst57fvJzZ4wVm2LxCbGr2b2kbOgq/EmkuX82ZKANFHrZ23oG5me/iXYXSnEranJQA/DuB7iKptAcLChCQiewCeA/CnqnqV2+ZVbYuKbecDCxGSTEp5PAfg71X1q9PLv5hWa8O8qm2q+pSq3q+q9w8Gg5O6BM4AbiojyUSH/FsAP1bVL1PTjaptT+IUVdtm7RPGufrqN2Al7zQXKO/4CH8uXVNbuYJNBT2Wn/ruE1BU5NgNXxepRI1y/r3LfwPti1sM7Pte3EsyUuk23oEm2Wp4xLKOc8FQ/QBfg2ZEG+WUFL1QdFySA+cHOqloTt7BTCxiR/okgD8A8AMR+Y/ptS9hQkDPTiu4vQrg90//+MBZwSJa2z9jtiAfVdsCADZh2b7h/fd7Y5CsXrf2GKFlnpbkSn1ZG9oKveu2QqcSOJy3pa4ou1IaOJeIAYA9qsTW30ljvOuSzDla4X1O7VZKxR5W1mww5u3myeo96LnKdGQt7zjTAHsMarKUi4sg4Ly5wlm25+XUzUL42gJZEIQUyIK1s7YbHM0HUwnHGrt7mNOxD7R2rI19rE3XazrpVXkb0ZGbx/VDChpz8daDnXQfO3e7uy7+nNKja7Ws55A0utJXUeM47T7tZ+JYm5DJXWo7R1Otha3XhWPh3LZM9VGHWJECWRCEFMiCIKRAFmxA/T/5esHWbG+xJnBwe93YwLOG5IVq5LZQJ7mooHz8Q1c0nQugtbfoTC6eskPmBJfIsEPRBQcjK390TPF5p5KTJ19IGDx0Pkql92zclu+85SjLeOJL6NC0xEdbeM/CAogVKZAFQUiBLFi/+n/jf8cO2BrcuKAr5jCsaPu07CGxumpsWdt18sD2aSU/csXWR+wg9vl1Xdplm8wLOrYqfkMV27q+ojqNKY6FKKWj81anQ8emmZ15xb1vLPq8v4vtx6KEt2RHWZvAxhCEFMiCIKRAFmwgr236v1M5OWhdnWuCISR/lC6ni+URcSX3zB5w7D13MgxbFHyEglLe3JAC96tqdv5b6cZn84J/dpdkvoJcNz3nuVeqNVC5soBU2ccGwNUuiI4jIFb3kMSKFMiDIKRAFmywqq2FNwcY8NabxG5ae5CR6l64wDZevW3gnEt5LlNbp7VFp2cPE/Rc6jhPq+OiEMoxF2z3AXzpsEvv2QoCVI4gcKV32GJNXK9VBc/s1+Yt+KdHrEiBLAhCCmTB1myzZXy2Lv2GeQWzDa/cNSa9x7EDegA7M9VZ0UtmKY4dlCYWTOm6/T32SWP071KT09lXW+OXG5Mm6FkbB8T5zbMLZu8cs+03LzNjest2VGwLbAhBSIEsCEIKZMHmZKR5KqcPRmcDrbnPq8WzFVdONuAAOP9LYq944Uu/mKA0yq+D3RKV8+bq2uedkXzjJ0nB/ywzes+9+QJOPmPziFHx2/u7pqNl9H2HRSq27YjIv4rIf04rtj0xvf4hEfmeiLwsIl8Rkd7NxgqcXSzC2oYAPq2qvw7gYwAeEpFPAPhLAH+lqr8G4C0Aj7530wxsOxbJ/VcAN3KLu9N/CuDTAD43vX4JwJ8B+JtFH9xaTXV228ytMX3VN6O2ysy+vKdZqzj5nBwvoZlxNz8/Vp+bltpNqdK+aD2f0haptRvDKu6tfURP7ucLvBj+OPtbLYpF6yOV00okrwP4FoD/BvC26vGe6JcxKQcYOKdYiJBUtVbVjwG4B8ADAD6y6AOiYtv5wKnUf1V9G8DzAH4TwL7IsV31HgCvzbgnKradAyxSse12AGNVfVtEdgE8iImg/TyA3wPwDE5Rse143Lmtc/TdOb5pa9mfN8bsyzJHPphlXvCyDoschdugl8cvvJuIg/VJlvIV1eZJMDxFne0FwZwm3OyvcxIWsSPdCeCSiJSYrGDPquo/ishLAJ4RkT8H8O+YlAcMnFMsorX9FyYlkf31VzCRlwIByDI5TEs/TOR/Mak3+SsA3ljbg7cb2/4tPqiqt9+s01oJ6fihIi+q6v1rf/AW4qx8i3DaBrIgCCmQBZsipKc29NxtxJn4FhuRkQJnD8HaAlmwVkISkYdE5CfTGKZzt1HgWd6Nc22sbWoZ/ykmLpbLAF4A8IiqvrSWCWwBprtI3amq3xeRiwD+DcDvAvhDAG+q6pPTH9htqnrTTRS3CetckR4A8LKqvqKqI0x8dA+v8fkbh6peUdXvT4+vAeDdOC9Nu13ChLhuKayTkO4G8HM6P9cxTGdtN84QtjeAZXfj3Gask5BeA3Avnc+MYTrLWGU3zm3GOgnpBQD3TbNPegA+i8kulOcGC+zGCSwR27UNWLf3/3cA/DUmxWmfVtW/WNvDtwAi8ikA3wXwA6T6tl/CRE56FsAHMN2NU1Xf3Mgkl0RYtgNZEMJ2IAuCkAJZEIQUyIIgpEAWBCEFsiAIKZAFQUiBLAhCCmTB/wO2rwu1LL049wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 144x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "if testcaseGenSuccessful: \n",
    "    util.displayGTSRB((new_image[0].numpy() - images[0].numpy()) + 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}