EvenMore-pytorch-models-Accuracies-Copy1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Even More pretrained models in PyTorch\n",
    "\n",
    "####  https://github.com/Cadene/pretrained-models.pytorch\n",
    "\n",
    "Can we correlate power law exponents with accuracy ?\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-27T08:06:08.823252Z",
     "start_time": "2018-08-27T08:06:06.545222Z"
    }
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision.models as models\n",
    "import torchvision.transforms as transforms\n",
    "import torch.nn as nn\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "import numpy as np\n",
    "import powerlaw\n",
    "\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "\n",
    "import matplotlib\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-27T08:06:11.499813Z",
     "start_time": "2018-08-27T08:06:08.826030Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "importing Jupyter notebook from RMT_Util.ipynb\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/charleshmartin/anaconda3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6\n",
      "  return f(*args, **kwds)\n",
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.6.5 |Anaconda, Inc.| (default, Apr 26 2018, 08:42:37) \n",
      "[GCC 4.2.1 Compatible Clang 4.0.1 (tags/RELEASE_401/final)]\n",
      "numpy version 1.14.3\n",
      "scipy version 1.1.0\n",
      "tensforflow version 1.10.1\n",
      "keras version 2.2.2\n",
      "sklearn version 0.19.1\n",
      "powerlaw version 1.4.3\n"
     ]
    }
   ],
   "source": [
    "import import_ipynb\n",
    "import RMT_Util"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-27T08:06:11.506733Z",
     "start_time": "2018-08-27T08:06:11.501899Z"
    }
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['figure.figsize'] = [5,5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pretrainedmodels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# does not work...I made symbolic link instead\n",
    "# %env TORCH_MODEL_ZOO=\"~/work-tmp/pretrainedmodels\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 45 Available models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['fbresnet152',\n",
       " 'bninception',\n",
       " 'resnext101_32x4d',\n",
       " 'resnext101_64x4d',\n",
       " 'inceptionv4',\n",
       " 'inceptionresnetv2',\n",
       " 'alexnet',\n",
       " 'densenet121',\n",
       " 'densenet169',\n",
       " 'densenet201',\n",
       " 'densenet161',\n",
       " 'resnet18',\n",
       " 'resnet34',\n",
       " 'resnet50',\n",
       " 'resnet101',\n",
       " 'resnet152',\n",
       " 'inceptionv3',\n",
       " 'squeezenet1_0',\n",
       " 'squeezenet1_1',\n",
       " 'vgg11',\n",
       " 'vgg11_bn',\n",
       " 'vgg13',\n",
       " 'vgg13_bn',\n",
       " 'vgg16',\n",
       " 'vgg16_bn',\n",
       " 'vgg19_bn',\n",
       " 'vgg19',\n",
       " 'nasnetamobile',\n",
       " 'nasnetalarge',\n",
       " 'dpn68',\n",
       " 'dpn68b',\n",
       " 'dpn92',\n",
       " 'dpn98',\n",
       " 'dpn131',\n",
       " 'dpn107',\n",
       " 'xception',\n",
       " 'senet154',\n",
       " 'se_resnet50',\n",
       " 'se_resnet101',\n",
       " 'se_resnet152',\n",
       " 'se_resnext50_32x4d',\n",
       " 'se_resnext101_32x4d',\n",
       " 'cafferesnet101',\n",
       " 'pnasnet5large',\n",
       " 'polynet']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pretrainedmodels.model_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "62"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracies = pd.read_csv('accuracies', delimiter=\"\\t\")\n",
    "len(accuracies)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TODO: remove duplcates when can use our ourporting\n",
    "\n",
    "1. get all ourporting\n",
    "2. get the other pytorch models\n",
    "\n",
    "I think this is enough"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Tensorflow', 'ourporting', 'Caffe', 'Torch7', 'Keras', 'Pytorch'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracies['platform'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(24,                   model    platform    acc1    acc5\n",
       " 1       PNASNet-5-Large  ourporting  82.736  95.992\n",
       " 3        NASNet-A-Large  ourporting  82.566  96.086\n",
       " 5              SENet154  ourporting  81.304  95.498\n",
       " 7               PolyNet  ourporting  81.002  95.624\n",
       " 10  SE-ResNeXt101_32x4d  ourporting  80.236  95.028\n",
       " 12    InceptionResNetV2  ourporting  80.170  95.234\n",
       " 13          InceptionV4  ourporting  80.062  94.926\n",
       " 14    DualPathNet107_5k  ourporting  79.746  94.684\n",
       " 16       DualPathNet131  ourporting  79.432  94.574\n",
       " 17     DualPathNet92_5k  ourporting  79.400  94.620\n",
       " 18        DualPathNet98  ourporting  79.224  94.488\n",
       " 19   SE-ResNeXt50_32x4d  ourporting  79.076  94.434\n",
       " 22     ResNeXt101_64x4d  ourporting  78.956  94.252\n",
       " 23             Xception  ourporting  78.888  94.292\n",
       " 26         SE-ResNet152  ourporting  78.658  94.374\n",
       " 28         SE-ResNet101  ourporting  78.396  94.258\n",
       " 30     ResNeXt101_32x4d  ourporting  78.188  93.886\n",
       " 33          SE-ResNet50  ourporting  77.636  93.752\n",
       " 36          FBResNet152  ourporting  77.386  93.594\n",
       " 39    DualPathNet68b_5k  ourporting  77.034  93.590\n",
       " 41       CaffeResnet101  ourporting  76.200  92.766\n",
       " 44        DualPathNet68  ourporting  75.868  92.774\n",
       " 48      NASNet-A-Mobile  ourporting  74.080  91.740\n",
       " 50          BNInception  ourporting  73.522  91.560)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "our_acc = accuracies[accuracies['platform']=='ourporting']\n",
    "len(our_acc), our_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(21,             model platform    acc1    acc5\n",
       " 27      ResNet152  Pytorch  78.428  94.110\n",
       " 34    DenseNet161  Pytorch  77.560  93.798\n",
       " 35      ResNet101  Pytorch  77.438  93.672\n",
       " 37    InceptionV3  Pytorch  77.294  93.454\n",
       " 38    DenseNet201  Pytorch  77.152  93.548\n",
       " 42    DenseNet169  Pytorch  76.026  92.992\n",
       " 43       ResNet50  Pytorch  76.002  92.980\n",
       " 45    DenseNet121  Pytorch  74.646  92.136\n",
       " 46       VGG19_BN  Pytorch  74.266  92.066\n",
       " 49       ResNet34  Pytorch  73.554  91.456\n",
       " 51       VGG16_BN  Pytorch  73.518  91.608\n",
       " 52          VGG19  Pytorch  72.080  90.822\n",
       " 53          VGG16  Pytorch  71.636  90.354\n",
       " 54       VGG13_BN  Pytorch  71.508  90.494\n",
       " 55       VGG11_BN  Pytorch  70.452  89.818\n",
       " 56       ResNet18  Pytorch  70.142  89.274\n",
       " 57          VGG13  Pytorch  69.662  89.264\n",
       " 58          VGG11  Pytorch  68.970  88.746\n",
       " 59  SqueezeNet1_1  Pytorch  58.250  80.800\n",
       " 60  SqueezeNet1_0  Pytorch  58.108  80.428\n",
       " 61        Alexnet  Pytorch  56.432  79.194)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "py_acc = accuracies[accuracies['platform']=='Pytorch']\n",
    "len(py_acc), py_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- merge dataframes\n",
    "- get model names\n",
    "- load pretrained\n",
    "- compute alpha\n",
    "- store in data frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>platform</th>\n",
       "      <th>acc1</th>\n",
       "      <th>acc5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>ResNet152</td>\n",
       "      <td>Pytorch</td>\n",
       "      <td>78.428</td>\n",
       "      <td>94.110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>DenseNet161</td>\n",
       "      <td>Pytorch</td>\n",
       "      <td>77.560</td>\n",
       "      <td>93.798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>ResNet101</td>\n",
       "      <td>Pytorch</td>\n",
       "      <td>77.438</td>\n",
       "      <td>93.672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>InceptionV3</td>\n",
       "      <td>Pytorch</td>\n",
       "      <td>77.294</td>\n",
       "      <td>93.454</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>DenseNet201</td>\n",
       "      <td>Pytorch</td>\n",
       "      <td>77.152</td>\n",
       "      <td>93.548</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          model platform    acc1    acc5\n",
       "27    ResNet152  Pytorch  78.428  94.110\n",
       "34  DenseNet161  Pytorch  77.560  93.798\n",
       "35    ResNet101  Pytorch  77.438  93.672\n",
       "37  InceptionV3  Pytorch  77.294  93.454\n",
       "38  DenseNet201  Pytorch  77.152  93.548"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df =  pd.concat([py_acc, our_acc])\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Minimum dimension for W"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "MIN_DIM = 50"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### All Linear Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-27T13:12:09.422260Z",
     "start_time": "2018-08-27T13:12:09.341227Z"
    }
   },
   "outputs": [],
   "source": [
    "def count_layers(model):\n",
    "    \"\"\"Loops over a pytorch model and counts all the weight matrices, linear, conv2d, and conv2d-slices\"\"\"\n",
    "    num_linear, num_conv2d, num_conv2d_slices = 0, 0, 0\n",
    "    \n",
    "    for im, m in enumerate(model.modules()):\n",
    "        #print(im,m)\n",
    "        if isinstance(m, nn.Linear):     \n",
    "            W = np.array(m.weight.data.clone().cpu())\n",
    "            if W is not None:\n",
    "                #print(im,m, W.shape)\n",
    "                num_linear += 1\n",
    "        elif isinstance(m, nn.Conv2d):\n",
    "            #print(\"conv2d\")\n",
    "            Wtensor = np.array(m.weight.data.clone().cpu())\n",
    "            #print(Wtensor.shape)\n",
    "            Wmats = RMT_Util.get_conv2D_Wmats(Wtensor)\n",
    "            #print(im,m)\n",
    "            num_conv2d += 1\n",
    "            for W in Wmats:\n",
    "                #print(W.shape)\n",
    "                if (np.min(W.shape) >= MIN_DIM):\n",
    "                    num_conv2d_slices += 1\n",
    "                \n",
    "    return num_linear, num_conv2d, num_conv2d_slices\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def iter_pytotch_layers(model, min_dim=MIN_DIM):\n",
    "    \"\"\"Iterate over a pytorch layers and  weight matrix.  \n",
    "    Supports linear, conv2d, and conv2d-slices\"\"\"\n",
    "    for im, m in enumerate(model.modules()):\n",
    "        if isinstance(m, nn.Linear):     \n",
    "            W = np.array(m.weight.data.clone().cpu())\n",
    "            if W is not None:\n",
    "                yield W\n",
    "        elif isinstance(m, nn.Conv2d):\n",
    "            Wtensor = np.array(m.weight.data.clone().cpu())\n",
    "            Wmats = RMT_Util.get_conv2D_Wmats(Wtensor)\n",
    "            for W in Wmats:\n",
    "                if (np.min(W.shape) >= min_dim):\n",
    "                    yield W"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calc_model_alphas(model):\n",
    "    try:\n",
    "        results = []\n",
    "        \n",
    "        for W in tqdm(iter_pytotch_layers(model)):\n",
    "            M, N = np.min(W.shape), np.max(W.shape)\n",
    "            _, sv, _ = np.linalg.svd(W)            \n",
    "    \n",
    "            evals = sv*sv\n",
    "            fit = powerlaw.Fit(evals, xmax=np.max(evals), verbose=False)\n",
    "            \n",
    "            rank_loss = M - np.linalg.matrix_rank(W)\n",
    "            results.append([fit.alpha, np.min(sv), N, M, rank_loss])\n",
    "            #print([fit.alpha, np.min(sv), N, M, rank_loss])\n",
    "\n",
    "    except Exception as e:\n",
    "        print(\"error \", model_name)\n",
    "        print(e)\n",
    "    \n",
    "    finally:\n",
    "        pass\n",
    "    \n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VGG(\n",
       "  (_features): Sequential(\n",
       "    (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (1): ReLU(inplace)\n",
       "    (2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (3): ReLU(inplace)\n",
       "    (4): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (5): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (6): ReLU(inplace)\n",
       "    (7): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (8): ReLU(inplace)\n",
       "    (9): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (10): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (11): ReLU(inplace)\n",
       "    (12): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (13): ReLU(inplace)\n",
       "    (14): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (15): ReLU(inplace)\n",
       "    (16): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (17): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (18): ReLU(inplace)\n",
       "    (19): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (20): ReLU(inplace)\n",
       "    (21): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (22): ReLU(inplace)\n",
       "    (23): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (24): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (25): ReLU(inplace)\n",
       "    (26): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (27): ReLU(inplace)\n",
       "    (28): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (29): ReLU(inplace)\n",
       "    (30): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "  )\n",
       "  (linear0): Linear(in_features=25088, out_features=4096, bias=True)\n",
       "  (relu0): ReLU(inplace)\n",
       "  (dropout0): Dropout(p=0.5)\n",
       "  (linear1): Linear(in_features=4096, out_features=4096, bias=True)\n",
       "  (relu1): ReLU(inplace)\n",
       "  (dropout1): Dropout(p=0.5)\n",
       "  (last_linear): Linear(in_features=4096, out_features=1000, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_name = \"vgg16\"\n",
    "model = pretrainedmodels.__dict__[model_name](num_classes=1000, pretrained='imagenet')\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "83b4dd5d31e24858a462aaf8b7662f63",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/powerlaw.py:700: RuntimeWarning: divide by zero encountered in true_divide\n",
      "  (Theoretical_CDF * (1 - Theoretical_CDF))\n",
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/powerlaw.py:700: RuntimeWarning: invalid value encountered in true_divide\n",
      "  (Theoretical_CDF * (1 - Theoretical_CDF))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "model = pretrainedmodels.__dict__[model_name](num_classes=1000, pretrained='imagenet')\n",
    "model.eval()\n",
    "results = calc_model_alphas(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VGG(\n",
       "  (linear0): Linear(in_features=25088, out_features=4096, bias=True)\n",
       "  (relu0): ReLU(inplace)\n",
       "  (dropout0): Dropout(p=0.5)\n",
       "  (linear1): Linear(in_features=4096, out_features=4096, bias=True)\n",
       "  (relu1): ReLU(inplace)\n",
       "  (dropout1): Dropout(p=0.5)\n",
       "  (last_linear): Linear(in_features=4096, out_features=1000, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_name = \"vgg16_bn\"\n",
    "model = pretrainedmodels.__dict__[model_name](num_classes=1000, pretrained='imagenet')\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "811e96b1aefd401e9743a37bd21378d5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/powerlaw.py:700: RuntimeWarning: divide by zero encountered in true_divide\n",
      "  (Theoretical_CDF * (1 - Theoretical_CDF))\n",
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/powerlaw.py:700: RuntimeWarning: invalid value encountered in true_divide\n",
      "  (Theoretical_CDF * (1 - Theoretical_CDF))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "model = pretrainedmodels.__dict__[model_name](num_classes=1000, pretrained='imagenet')\n",
    "model.eval()\n",
    "results = calc_model_alphas(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['fbresnet152',\n",
       " 'bninception',\n",
       " 'resnext101_32x4d',\n",
       " 'resnext101_64x4d',\n",
       " 'inceptionv4',\n",
       " 'inceptionresnetv2',\n",
       " 'alexnet',\n",
       " 'densenet121',\n",
       " 'densenet169',\n",
       " 'densenet201',\n",
       " 'densenet161',\n",
       " 'resnet18',\n",
       " 'resnet34',\n",
       " 'resnet50',\n",
       " 'resnet101',\n",
       " 'resnet152',\n",
       " 'inceptionv3',\n",
       " 'squeezenet1_0',\n",
       " 'squeezenet1_1',\n",
       " 'vgg11',\n",
       " 'vgg11_bn',\n",
       " 'vgg13',\n",
       " 'vgg13_bn',\n",
       " 'vgg16',\n",
       " 'vgg16_bn',\n",
       " 'vgg19_bn',\n",
       " 'vgg19',\n",
       " 'nasnetamobile',\n",
       " 'nasnetalarge',\n",
       " 'dpn68',\n",
       " 'dpn68b',\n",
       " 'dpn92',\n",
       " 'dpn98',\n",
       " 'dpn131',\n",
       " 'dpn107',\n",
       " 'xception',\n",
       " 'senet154',\n",
       " 'se_resnet50',\n",
       " 'se_resnet101',\n",
       " 'se_resnet152',\n",
       " 'se_resnext50_32x4d',\n",
       " 'se_resnext101_32x4d',\n",
       " 'cafferesnet101',\n",
       " 'pnasnet5large',\n",
       " 'polynet']"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pretrainedmodels.model_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/torchvision/models/squeezenet.py:94: UserWarning: nn.init.kaiming_uniform is now deprecated in favor of nn.init.kaiming_uniform_.\n",
      "  init.kaiming_uniform(m.weight.data)\n",
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/torchvision/models/squeezenet.py:92: UserWarning: nn.init.normal is now deprecated in favor of nn.init.normal_.\n",
      "  init.normal(m.weight.data, mean=0.0, std=0.01)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "SqueezeNet(\n",
       "  (features): Sequential(\n",
       "    (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(2, 2))\n",
       "    (1): ReLU(inplace)\n",
       "    (2): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=True)\n",
       "    (3): Fire(\n",
       "      (squeeze): Conv2d(64, 16, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (squeeze_activation): ReLU(inplace)\n",
       "      (expand1x1): Conv2d(16, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (expand1x1_activation): ReLU(inplace)\n",
       "      (expand3x3): Conv2d(16, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (expand3x3_activation): ReLU(inplace)\n",
       "    )\n",
       "    (4): Fire(\n",
       "      (squeeze): Conv2d(128, 16, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (squeeze_activation): ReLU(inplace)\n",
       "      (expand1x1): Conv2d(16, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (expand1x1_activation): ReLU(inplace)\n",
       "      (expand3x3): Conv2d(16, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (expand3x3_activation): ReLU(inplace)\n",
       "    )\n",
       "    (5): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=True)\n",
       "    (6): Fire(\n",
       "      (squeeze): Conv2d(128, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (squeeze_activation): ReLU(inplace)\n",
       "      (expand1x1): Conv2d(32, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (expand1x1_activation): ReLU(inplace)\n",
       "      (expand3x3): Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (expand3x3_activation): ReLU(inplace)\n",
       "    )\n",
       "    (7): Fire(\n",
       "      (squeeze): Conv2d(256, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (squeeze_activation): ReLU(inplace)\n",
       "      (expand1x1): Conv2d(32, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (expand1x1_activation): ReLU(inplace)\n",
       "      (expand3x3): Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (expand3x3_activation): ReLU(inplace)\n",
       "    )\n",
       "    (8): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=True)\n",
       "    (9): Fire(\n",
       "      (squeeze): Conv2d(256, 48, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (squeeze_activation): ReLU(inplace)\n",
       "      (expand1x1): Conv2d(48, 192, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (expand1x1_activation): ReLU(inplace)\n",
       "      (expand3x3): Conv2d(48, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (expand3x3_activation): ReLU(inplace)\n",
       "    )\n",
       "    (10): Fire(\n",
       "      (squeeze): Conv2d(384, 48, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (squeeze_activation): ReLU(inplace)\n",
       "      (expand1x1): Conv2d(48, 192, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (expand1x1_activation): ReLU(inplace)\n",
       "      (expand3x3): Conv2d(48, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (expand3x3_activation): ReLU(inplace)\n",
       "    )\n",
       "    (11): Fire(\n",
       "      (squeeze): Conv2d(384, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (squeeze_activation): ReLU(inplace)\n",
       "      (expand1x1): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (expand1x1_activation): ReLU(inplace)\n",
       "      (expand3x3): Conv2d(64, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (expand3x3_activation): ReLU(inplace)\n",
       "    )\n",
       "    (12): Fire(\n",
       "      (squeeze): Conv2d(512, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (squeeze_activation): ReLU(inplace)\n",
       "      (expand1x1): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (expand1x1_activation): ReLU(inplace)\n",
       "      (expand3x3): Conv2d(64, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (expand3x3_activation): ReLU(inplace)\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.5)\n",
       "  (last_conv): Conv2d(512, 1000, kernel_size=(1, 1), stride=(1, 1))\n",
       "  (relu): ReLU(inplace)\n",
       "  (avgpool): AvgPool2d(kernel_size=13, stride=1, padding=0)\n",
       ")"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_name = \"squeezenet1_1\"\n",
    "model = pretrainedmodels.__dict__[model_name](num_classes=1000, pretrained='imagenet')\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "67e8976ac9d34bb5b156584707ec5b10",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/powerlaw.py:700: RuntimeWarning: divide by zero encountered in true_divide\n",
      "  (Theoretical_CDF * (1 - Theoretical_CDF))\n",
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/powerlaw.py:700: RuntimeWarning: invalid value encountered in true_divide\n",
      "  (Theoretical_CDF * (1 - Theoretical_CDF))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/powerlaw.py:1178: RuntimeWarning: overflow encountered in double_scalars\n",
      "  return (self.alpha-1) * self.xmin**(self.alpha-1)\n",
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/powerlaw.py:825: RuntimeWarning: invalid value encountered in multiply\n",
      "  likelihoods = f*C\n",
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/scipy/optimize/optimize.py:563: RuntimeWarning: invalid value encountered in subtract\n",
      "  numpy.max(numpy.abs(fsim[0] - fsim[1:])) <= fatol):\n"
     ]
    }
   ],
   "source": [
    "results = calc_model_alphas(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "resnet152\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b05879825449432fa2530cb838f8fa0a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/powerlaw.py:700: RuntimeWarning: divide by zero encountered in true_divide\n",
      "  (Theoretical_CDF * (1 - Theoretical_CDF))\n",
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/powerlaw.py:700: RuntimeWarning: invalid value encountered in true_divide\n",
      "  (Theoretical_CDF * (1 - Theoretical_CDF))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "densenet161\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/torchvision/models/densenet.py:212: UserWarning: nn.init.kaiming_normal is now deprecated in favor of nn.init.kaiming_normal_.\n",
      "  nn.init.kaiming_normal(m.weight.data)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "error  densenet161\n",
      "Cannot initialize CUDA without ATen_cuda library. PyTorch splits its backend into two shared libraries: a CPU library and a CUDA library; this error has occurred because you are trying to use some CUDA functionality, but the CUDA library has not been loaded by the dynamic linker for some reason.  The CUDA library MUST be loaded, EVEN IF you don't directly use any symbols from the CUDA library! One common culprit is a lack of -Wl,--no-as-needed in your link arguments; many dynamic linkers will delete dynamic library dependencies if you don't depend on any of their symbols.  You can check if this has occurred by using ldd on your binary to see if there is a dependency on *_cuda.so library.\n",
      "resnet101\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "10cb1829936741bb9843ca06f4a4b8df",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "inceptionv3\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "93d16a6fdb03466a9492668e06b8e1d4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "densenet201\n",
      "error  densenet201\n",
      "Cannot initialize CUDA without ATen_cuda library. PyTorch splits its backend into two shared libraries: a CPU library and a CUDA library; this error has occurred because you are trying to use some CUDA functionality, but the CUDA library has not been loaded by the dynamic linker for some reason.  The CUDA library MUST be loaded, EVEN IF you don't directly use any symbols from the CUDA library! One common culprit is a lack of -Wl,--no-as-needed in your link arguments; many dynamic linkers will delete dynamic library dependencies if you don't depend on any of their symbols.  You can check if this has occurred by using ldd on your binary to see if there is a dependency on *_cuda.so library.\n",
      "densenet169\n",
      "error  densenet169\n",
      "Cannot initialize CUDA without ATen_cuda library. PyTorch splits its backend into two shared libraries: a CPU library and a CUDA library; this error has occurred because you are trying to use some CUDA functionality, but the CUDA library has not been loaded by the dynamic linker for some reason.  The CUDA library MUST be loaded, EVEN IF you don't directly use any symbols from the CUDA library! One common culprit is a lack of -Wl,--no-as-needed in your link arguments; many dynamic linkers will delete dynamic library dependencies if you don't depend on any of their symbols.  You can check if this has occurred by using ldd on your binary to see if there is a dependency on *_cuda.so library.\n",
      "resnet50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9b4d7b908c59468ab14d51a22375e34f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "densenet121\n",
      "error  densenet121\n",
      "Cannot initialize CUDA without ATen_cuda library. PyTorch splits its backend into two shared libraries: a CPU library and a CUDA library; this error has occurred because you are trying to use some CUDA functionality, but the CUDA library has not been loaded by the dynamic linker for some reason.  The CUDA library MUST be loaded, EVEN IF you don't directly use any symbols from the CUDA library! One common culprit is a lack of -Wl,--no-as-needed in your link arguments; many dynamic linkers will delete dynamic library dependencies if you don't depend on any of their symbols.  You can check if this has occurred by using ldd on your binary to see if there is a dependency on *_cuda.so library.\n",
      "vgg19_bn\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5cc7b75bb1bd43c3a19ab50cdd4f22e6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "resnet34\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a68e1b71a5bf4eac993bf6fd5c0c0f0a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "vgg16_bn\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3edf6828e36c414a9a9558b324f683b1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "vgg19\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "29efb3941ab34838b9e8db577bef3d43",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "vgg16\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d290f4e6eb764b188adac55525a34ad0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "vgg13_bn\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fa2d7bfabdb2435b99ea3771c94e069a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "vgg11_bn\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d19e53b5c4ff4e9ca299a3e08b41cc59",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "resnet18\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "325094703d624c64a4f82ad017447eed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "vgg13\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "582bf847b5aa4013bb43f34931925052",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "vgg11\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ed87e62fc2694b4d9b1f382134b61792",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "squeezenet1_1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/torchvision/models/squeezenet.py:94: UserWarning: nn.init.kaiming_uniform is now deprecated in favor of nn.init.kaiming_uniform_.\n",
      "  init.kaiming_uniform(m.weight.data)\n",
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/torchvision/models/squeezenet.py:92: UserWarning: nn.init.normal is now deprecated in favor of nn.init.normal_.\n",
      "  init.normal(m.weight.data, mean=0.0, std=0.01)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "921851d0dbfd4555b0d03d108d83eebc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "squeezenet1_0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/powerlaw.py:1178: RuntimeWarning: overflow encountered in double_scalars\n",
      "  return (self.alpha-1) * self.xmin**(self.alpha-1)\n",
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/powerlaw.py:825: RuntimeWarning: invalid value encountered in multiply\n",
      "  likelihoods = f*C\n",
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/scipy/optimize/optimize.py:563: RuntimeWarning: invalid value encountered in subtract\n",
      "  numpy.max(numpy.abs(fsim[0] - fsim[1:])) <= fatol):\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c8ddde4d5e3b4768846396270955acb3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "alexnet\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1b1c2d24a4e04977802b5ee4f9a124f6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "pnasnet5large\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b4fa1853f4884094b8825ee976156e88",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nasnetalarge\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3af409d6fed94e0b9f3b5fa592d1d55d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "senet154\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5330011d091c4ca2a28dd47a561d4b1e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "polynet\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f5cacd1178794abc8770b5b0d86700b4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/powerlaw.py:1174: RuntimeWarning: overflow encountered in power\n",
      "  return x**-self.alpha\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "inceptionresnetv2\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d8af872b6dc94a109f6969b4da0425af",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "inceptionv4\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5002469816514716a828e1da2fb19032",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "resnext101_64x4d\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a01ad855e5564fde8a07a8f647abe9f3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "xception\n",
      "error  xception\n",
      "Error(s) in loading state_dict for Xception:\n",
      "\tsize mismatch for block1.rep.0.pointwise.weight: copying a param of torch.Size([128, 64, 1, 1]) from checkpoint, where the shape is torch.Size([128, 64]) in current model.\n",
      "\tsize mismatch for block1.rep.3.pointwise.weight: copying a param of torch.Size([128, 128, 1, 1]) from checkpoint, where the shape is torch.Size([128, 128]) in current model.\n",
      "\tsize mismatch for block2.rep.1.pointwise.weight: copying a param of torch.Size([256, 128, 1, 1]) from checkpoint, where the shape is torch.Size([256, 128]) in current model.\n",
      "\tsize mismatch for block2.rep.4.pointwise.weight: copying a param of torch.Size([256, 256, 1, 1]) from checkpoint, where the shape is torch.Size([256, 256]) in current model.\n",
      "\tsize mismatch for block3.rep.1.pointwise.weight: copying a param of torch.Size([728, 256, 1, 1]) from checkpoint, where the shape is torch.Size([728, 256]) in current model.\n",
      "\tsize mismatch for block3.rep.4.pointwise.weight: copying a param of torch.Size([728, 728, 1, 1]) from checkpoint, where the shape is torch.Size([728, 728]) in current model.\n",
      "\tsize mismatch for block4.rep.1.pointwise.weight: copying a param of torch.Size([728, 728, 1, 1]) from checkpoint, where the shape is torch.Size([728, 728]) in current model.\n",
      "\tsize mismatch for block4.rep.4.pointwise.weight: copying a param of torch.Size([728, 728, 1, 1]) from checkpoint, where the shape is torch.Size([728, 728]) in current model.\n",
      "\tsize mismatch for block4.rep.7.pointwise.weight: copying a param of torch.Size([728, 728, 1, 1]) from checkpoint, where the shape is torch.Size([728, 728]) in current model.\n",
      "\tsize mismatch for block5.rep.1.pointwise.weight: copying a param of torch.Size([728, 728, 1, 1]) from checkpoint, where the shape is torch.Size([728, 728]) in current model.\n",
      "\tsize mismatch for block5.rep.4.pointwise.weight: copying a param of torch.Size([728, 728, 1, 1]) from checkpoint, where the shape is torch.Size([728, 728]) in current model.\n",
      "\tsize mismatch for block5.rep.7.pointwise.weight: copying a param of torch.Size([728, 728, 1, 1]) from checkpoint, where the shape is torch.Size([728, 728]) in current model.\n",
      "\tsize mismatch for block6.rep.1.pointwise.weight: copying a param of torch.Size([728, 728, 1, 1]) from checkpoint, where the shape is torch.Size([728, 728]) in current model.\n",
      "\tsize mismatch for block6.rep.4.pointwise.weight: copying a param of torch.Size([728, 728, 1, 1]) from checkpoint, where the shape is torch.Size([728, 728]) in current model.\n",
      "\tsize mismatch for block6.rep.7.pointwise.weight: copying a param of torch.Size([728, 728, 1, 1]) from checkpoint, where the shape is torch.Size([728, 728]) in current model.\n",
      "\tsize mismatch for block7.rep.1.pointwise.weight: copying a param of torch.Size([728, 728, 1, 1]) from checkpoint, where the shape is torch.Size([728, 728]) in current model.\n",
      "\tsize mismatch for block7.rep.4.pointwise.weight: copying a param of torch.Size([728, 728, 1, 1]) from checkpoint, where the shape is torch.Size([728, 728]) in current model.\n",
      "\tsize mismatch for block7.rep.7.pointwise.weight: copying a param of torch.Size([728, 728, 1, 1]) from checkpoint, where the shape is torch.Size([728, 728]) in current model.\n",
      "\tsize mismatch for block8.rep.1.pointwise.weight: copying a param of torch.Size([728, 728, 1, 1]) from checkpoint, where the shape is torch.Size([728, 728]) in current model.\n",
      "\tsize mismatch for block8.rep.4.pointwise.weight: copying a param of torch.Size([728, 728, 1, 1]) from checkpoint, where the shape is torch.Size([728, 728]) in current model.\n",
      "\tsize mismatch for block8.rep.7.pointwise.weight: copying a param of torch.Size([728, 728, 1, 1]) from checkpoint, where the shape is torch.Size([728, 728]) in current model.\n",
      "\tsize mismatch for block9.rep.1.pointwise.weight: copying a param of torch.Size([728, 728, 1, 1]) from checkpoint, where the shape is torch.Size([728, 728]) in current model.\n",
      "\tsize mismatch for block9.rep.4.pointwise.weight: copying a param of torch.Size([728, 728, 1, 1]) from checkpoint, where the shape is torch.Size([728, 728]) in current model.\n",
      "\tsize mismatch for block9.rep.7.pointwise.weight: copying a param of torch.Size([728, 728, 1, 1]) from checkpoint, where the shape is torch.Size([728, 728]) in current model.\n",
      "\tsize mismatch for block10.rep.1.pointwise.weight: copying a param of torch.Size([728, 728, 1, 1]) from checkpoint, where the shape is torch.Size([728, 728]) in current model.\n",
      "\tsize mismatch for block10.rep.4.pointwise.weight: copying a param of torch.Size([728, 728, 1, 1]) from checkpoint, where the shape is torch.Size([728, 728]) in current model.\n",
      "\tsize mismatch for block10.rep.7.pointwise.weight: copying a param of torch.Size([728, 728, 1, 1]) from checkpoint, where the shape is torch.Size([728, 728]) in current model.\n",
      "\tsize mismatch for block11.rep.1.pointwise.weight: copying a param of torch.Size([728, 728, 1, 1]) from checkpoint, where the shape is torch.Size([728, 728]) in current model.\n",
      "\tsize mismatch for block11.rep.4.pointwise.weight: copying a param of torch.Size([728, 728, 1, 1]) from checkpoint, where the shape is torch.Size([728, 728]) in current model.\n",
      "\tsize mismatch for block11.rep.7.pointwise.weight: copying a param of torch.Size([728, 728, 1, 1]) from checkpoint, where the shape is torch.Size([728, 728]) in current model.\n",
      "\tsize mismatch for block12.rep.1.pointwise.weight: copying a param of torch.Size([728, 728, 1, 1]) from checkpoint, where the shape is torch.Size([728, 728]) in current model.\n",
      "\tsize mismatch for block12.rep.4.pointwise.weight: copying a param of torch.Size([1024, 728, 1, 1]) from checkpoint, where the shape is torch.Size([1024, 728]) in current model.\n",
      "\tsize mismatch for conv3.pointwise.weight: copying a param of torch.Size([1536, 1024, 1, 1]) from checkpoint, where the shape is torch.Size([1536, 1024]) in current model.\n",
      "\tsize mismatch for conv4.pointwise.weight: copying a param of torch.Size([2048, 1536, 1, 1]) from checkpoint, where the shape is torch.Size([2048, 1536]) in current model.\n",
      "resnext101_32x4d\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b65eb1e56af24ca8968d9e332295705a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "fbresnet152\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "229183e984b449119b9027cc3c448b43",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "cafferesnet101\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "57b84d0777474960b70cabb64d697d4a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nasnetamobile\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "35a48a0ea3ae4b02ae65a2aec46d9abb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "bninception\n",
      "error  bninception\n",
      "Error(s) in loading state_dict for BNInception:\n",
      "\tsize mismatch for conv1_7x7_s2_bn.weight: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for conv1_7x7_s2_bn.bias: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for conv1_7x7_s2_bn.running_mean: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for conv1_7x7_s2_bn.running_var: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for conv2_3x3_reduce_bn.weight: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for conv2_3x3_reduce_bn.bias: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for conv2_3x3_reduce_bn.running_mean: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for conv2_3x3_reduce_bn.running_var: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for conv2_3x3_bn.weight: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for conv2_3x3_bn.bias: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for conv2_3x3_bn.running_mean: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for conv2_3x3_bn.running_var: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_3a_1x1_bn.weight: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3a_1x1_bn.bias: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3a_1x1_bn.running_mean: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3a_1x1_bn.running_var: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3a_3x3_reduce_bn.weight: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3a_3x3_reduce_bn.bias: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3a_3x3_reduce_bn.running_mean: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3a_3x3_reduce_bn.running_var: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3a_3x3_bn.weight: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3a_3x3_bn.bias: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3a_3x3_bn.running_mean: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3a_3x3_bn.running_var: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3a_double_3x3_reduce_bn.weight: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3a_double_3x3_reduce_bn.bias: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3a_double_3x3_reduce_bn.running_mean: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3a_double_3x3_reduce_bn.running_var: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3a_double_3x3_1_bn.weight: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_3a_double_3x3_1_bn.bias: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_3a_double_3x3_1_bn.running_mean: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_3a_double_3x3_1_bn.running_var: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_3a_double_3x3_2_bn.weight: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_3a_double_3x3_2_bn.bias: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_3a_double_3x3_2_bn.running_mean: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_3a_double_3x3_2_bn.running_var: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_3a_pool_proj_bn.weight: copying a param of torch.Size([32]) from checkpoint, where the shape is torch.Size([1, 32]) in current model.\n",
      "\tsize mismatch for inception_3a_pool_proj_bn.bias: copying a param of torch.Size([32]) from checkpoint, where the shape is torch.Size([1, 32]) in current model.\n",
      "\tsize mismatch for inception_3a_pool_proj_bn.running_mean: copying a param of torch.Size([32]) from checkpoint, where the shape is torch.Size([1, 32]) in current model.\n",
      "\tsize mismatch for inception_3a_pool_proj_bn.running_var: copying a param of torch.Size([32]) from checkpoint, where the shape is torch.Size([1, 32]) in current model.\n",
      "\tsize mismatch for inception_3b_1x1_bn.weight: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3b_1x1_bn.bias: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3b_1x1_bn.running_mean: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3b_1x1_bn.running_var: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3b_3x3_reduce_bn.weight: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3b_3x3_reduce_bn.bias: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3b_3x3_reduce_bn.running_mean: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3b_3x3_reduce_bn.running_var: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3b_3x3_bn.weight: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_3b_3x3_bn.bias: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_3b_3x3_bn.running_mean: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_3b_3x3_bn.running_var: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_3b_double_3x3_reduce_bn.weight: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3b_double_3x3_reduce_bn.bias: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3b_double_3x3_reduce_bn.running_mean: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3b_double_3x3_reduce_bn.running_var: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3b_double_3x3_1_bn.weight: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_3b_double_3x3_1_bn.bias: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_3b_double_3x3_1_bn.running_mean: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_3b_double_3x3_1_bn.running_var: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_3b_double_3x3_2_bn.weight: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_3b_double_3x3_2_bn.bias: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_3b_double_3x3_2_bn.running_mean: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_3b_double_3x3_2_bn.running_var: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_3b_pool_proj_bn.weight: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3b_pool_proj_bn.bias: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3b_pool_proj_bn.running_mean: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3b_pool_proj_bn.running_var: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3c_3x3_reduce_bn.weight: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_3c_3x3_reduce_bn.bias: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_3c_3x3_reduce_bn.running_mean: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_3c_3x3_reduce_bn.running_var: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_3c_3x3_bn.weight: copying a param of torch.Size([160]) from checkpoint, where the shape is torch.Size([1, 160]) in current model.\n",
      "\tsize mismatch for inception_3c_3x3_bn.bias: copying a param of torch.Size([160]) from checkpoint, where the shape is torch.Size([1, 160]) in current model.\n",
      "\tsize mismatch for inception_3c_3x3_bn.running_mean: copying a param of torch.Size([160]) from checkpoint, where the shape is torch.Size([1, 160]) in current model.\n",
      "\tsize mismatch for inception_3c_3x3_bn.running_var: copying a param of torch.Size([160]) from checkpoint, where the shape is torch.Size([1, 160]) in current model.\n",
      "\tsize mismatch for inception_3c_double_3x3_reduce_bn.weight: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3c_double_3x3_reduce_bn.bias: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3c_double_3x3_reduce_bn.running_mean: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3c_double_3x3_reduce_bn.running_var: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_3c_double_3x3_1_bn.weight: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_3c_double_3x3_1_bn.bias: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_3c_double_3x3_1_bn.running_mean: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_3c_double_3x3_1_bn.running_var: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_3c_double_3x3_2_bn.weight: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_3c_double_3x3_2_bn.bias: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_3c_double_3x3_2_bn.running_mean: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_3c_double_3x3_2_bn.running_var: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_4a_1x1_bn.weight: copying a param of torch.Size([224]) from checkpoint, where the shape is torch.Size([1, 224]) in current model.\n",
      "\tsize mismatch for inception_4a_1x1_bn.bias: copying a param of torch.Size([224]) from checkpoint, where the shape is torch.Size([1, 224]) in current model.\n",
      "\tsize mismatch for inception_4a_1x1_bn.running_mean: copying a param of torch.Size([224]) from checkpoint, where the shape is torch.Size([1, 224]) in current model.\n",
      "\tsize mismatch for inception_4a_1x1_bn.running_var: copying a param of torch.Size([224]) from checkpoint, where the shape is torch.Size([1, 224]) in current model.\n",
      "\tsize mismatch for inception_4a_3x3_reduce_bn.weight: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_4a_3x3_reduce_bn.bias: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_4a_3x3_reduce_bn.running_mean: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_4a_3x3_reduce_bn.running_var: copying a param of torch.Size([64]) from checkpoint, where the shape is torch.Size([1, 64]) in current model.\n",
      "\tsize mismatch for inception_4a_3x3_bn.weight: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_4a_3x3_bn.bias: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_4a_3x3_bn.running_mean: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_4a_3x3_bn.running_var: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_4a_double_3x3_reduce_bn.weight: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_4a_double_3x3_reduce_bn.bias: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_4a_double_3x3_reduce_bn.running_mean: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_4a_double_3x3_reduce_bn.running_var: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_4a_double_3x3_1_bn.weight: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4a_double_3x3_1_bn.bias: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4a_double_3x3_1_bn.running_mean: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4a_double_3x3_1_bn.running_var: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4a_double_3x3_2_bn.weight: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4a_double_3x3_2_bn.bias: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4a_double_3x3_2_bn.running_mean: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4a_double_3x3_2_bn.running_var: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4a_pool_proj_bn.weight: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4a_pool_proj_bn.bias: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4a_pool_proj_bn.running_mean: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4a_pool_proj_bn.running_var: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4b_1x1_bn.weight: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_4b_1x1_bn.bias: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_4b_1x1_bn.running_mean: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_4b_1x1_bn.running_var: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_4b_3x3_reduce_bn.weight: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_4b_3x3_reduce_bn.bias: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_4b_3x3_reduce_bn.running_mean: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_4b_3x3_reduce_bn.running_var: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_4b_3x3_bn.weight: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4b_3x3_bn.bias: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4b_3x3_bn.running_mean: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4b_3x3_bn.running_var: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4b_double_3x3_reduce_bn.weight: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_4b_double_3x3_reduce_bn.bias: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_4b_double_3x3_reduce_bn.running_mean: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_4b_double_3x3_reduce_bn.running_var: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_4b_double_3x3_1_bn.weight: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4b_double_3x3_1_bn.bias: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4b_double_3x3_1_bn.running_mean: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4b_double_3x3_1_bn.running_var: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4b_double_3x3_2_bn.weight: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4b_double_3x3_2_bn.bias: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4b_double_3x3_2_bn.running_mean: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4b_double_3x3_2_bn.running_var: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4b_pool_proj_bn.weight: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4b_pool_proj_bn.bias: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4b_pool_proj_bn.running_mean: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4b_pool_proj_bn.running_var: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4c_1x1_bn.weight: copying a param of torch.Size([160]) from checkpoint, where the shape is torch.Size([1, 160]) in current model.\n",
      "\tsize mismatch for inception_4c_1x1_bn.bias: copying a param of torch.Size([160]) from checkpoint, where the shape is torch.Size([1, 160]) in current model.\n",
      "\tsize mismatch for inception_4c_1x1_bn.running_mean: copying a param of torch.Size([160]) from checkpoint, where the shape is torch.Size([1, 160]) in current model.\n",
      "\tsize mismatch for inception_4c_1x1_bn.running_var: copying a param of torch.Size([160]) from checkpoint, where the shape is torch.Size([1, 160]) in current model.\n",
      "\tsize mismatch for inception_4c_3x3_reduce_bn.weight: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4c_3x3_reduce_bn.bias: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4c_3x3_reduce_bn.running_mean: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4c_3x3_reduce_bn.running_var: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4c_3x3_bn.weight: copying a param of torch.Size([160]) from checkpoint, where the shape is torch.Size([1, 160]) in current model.\n",
      "\tsize mismatch for inception_4c_3x3_bn.bias: copying a param of torch.Size([160]) from checkpoint, where the shape is torch.Size([1, 160]) in current model.\n",
      "\tsize mismatch for inception_4c_3x3_bn.running_mean: copying a param of torch.Size([160]) from checkpoint, where the shape is torch.Size([1, 160]) in current model.\n",
      "\tsize mismatch for inception_4c_3x3_bn.running_var: copying a param of torch.Size([160]) from checkpoint, where the shape is torch.Size([1, 160]) in current model.\n",
      "\tsize mismatch for inception_4c_double_3x3_reduce_bn.weight: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4c_double_3x3_reduce_bn.bias: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4c_double_3x3_reduce_bn.running_mean: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4c_double_3x3_reduce_bn.running_var: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4c_double_3x3_1_bn.weight: copying a param of torch.Size([160]) from checkpoint, where the shape is torch.Size([1, 160]) in current model.\n",
      "\tsize mismatch for inception_4c_double_3x3_1_bn.bias: copying a param of torch.Size([160]) from checkpoint, where the shape is torch.Size([1, 160]) in current model.\n",
      "\tsize mismatch for inception_4c_double_3x3_1_bn.running_mean: copying a param of torch.Size([160]) from checkpoint, where the shape is torch.Size([1, 160]) in current model.\n",
      "\tsize mismatch for inception_4c_double_3x3_1_bn.running_var: copying a param of torch.Size([160]) from checkpoint, where the shape is torch.Size([1, 160]) in current model.\n",
      "\tsize mismatch for inception_4c_double_3x3_2_bn.weight: copying a param of torch.Size([160]) from checkpoint, where the shape is torch.Size([1, 160]) in current model.\n",
      "\tsize mismatch for inception_4c_double_3x3_2_bn.bias: copying a param of torch.Size([160]) from checkpoint, where the shape is torch.Size([1, 160]) in current model.\n",
      "\tsize mismatch for inception_4c_double_3x3_2_bn.running_mean: copying a param of torch.Size([160]) from checkpoint, where the shape is torch.Size([1, 160]) in current model.\n",
      "\tsize mismatch for inception_4c_double_3x3_2_bn.running_var: copying a param of torch.Size([160]) from checkpoint, where the shape is torch.Size([1, 160]) in current model.\n",
      "\tsize mismatch for inception_4c_pool_proj_bn.weight: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4c_pool_proj_bn.bias: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4c_pool_proj_bn.running_mean: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4c_pool_proj_bn.running_var: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4d_1x1_bn.weight: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_4d_1x1_bn.bias: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_4d_1x1_bn.running_mean: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_4d_1x1_bn.running_var: copying a param of torch.Size([96]) from checkpoint, where the shape is torch.Size([1, 96]) in current model.\n",
      "\tsize mismatch for inception_4d_3x3_reduce_bn.weight: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4d_3x3_reduce_bn.bias: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4d_3x3_reduce_bn.running_mean: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4d_3x3_reduce_bn.running_var: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4d_3x3_bn.weight: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_4d_3x3_bn.bias: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_4d_3x3_bn.running_mean: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_4d_3x3_bn.running_var: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_4d_double_3x3_reduce_bn.weight: copying a param of torch.Size([160]) from checkpoint, where the shape is torch.Size([1, 160]) in current model.\n",
      "\tsize mismatch for inception_4d_double_3x3_reduce_bn.bias: copying a param of torch.Size([160]) from checkpoint, where the shape is torch.Size([1, 160]) in current model.\n",
      "\tsize mismatch for inception_4d_double_3x3_reduce_bn.running_mean: copying a param of torch.Size([160]) from checkpoint, where the shape is torch.Size([1, 160]) in current model.\n",
      "\tsize mismatch for inception_4d_double_3x3_reduce_bn.running_var: copying a param of torch.Size([160]) from checkpoint, where the shape is torch.Size([1, 160]) in current model.\n",
      "\tsize mismatch for inception_4d_double_3x3_1_bn.weight: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_4d_double_3x3_1_bn.bias: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_4d_double_3x3_1_bn.running_mean: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_4d_double_3x3_1_bn.running_var: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_4d_double_3x3_2_bn.weight: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_4d_double_3x3_2_bn.bias: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_4d_double_3x3_2_bn.running_mean: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_4d_double_3x3_2_bn.running_var: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_4d_pool_proj_bn.weight: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4d_pool_proj_bn.bias: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4d_pool_proj_bn.running_mean: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4d_pool_proj_bn.running_var: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4e_3x3_reduce_bn.weight: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4e_3x3_reduce_bn.bias: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4e_3x3_reduce_bn.running_mean: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4e_3x3_reduce_bn.running_var: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_4e_3x3_bn.weight: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_4e_3x3_bn.bias: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_4e_3x3_bn.running_mean: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_4e_3x3_bn.running_var: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_4e_double_3x3_reduce_bn.weight: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_4e_double_3x3_reduce_bn.bias: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_4e_double_3x3_reduce_bn.running_mean: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_4e_double_3x3_reduce_bn.running_var: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_4e_double_3x3_1_bn.weight: copying a param of torch.Size([256]) from checkpoint, where the shape is torch.Size([1, 256]) in current model.\n",
      "\tsize mismatch for inception_4e_double_3x3_1_bn.bias: copying a param of torch.Size([256]) from checkpoint, where the shape is torch.Size([1, 256]) in current model.\n",
      "\tsize mismatch for inception_4e_double_3x3_1_bn.running_mean: copying a param of torch.Size([256]) from checkpoint, where the shape is torch.Size([1, 256]) in current model.\n",
      "\tsize mismatch for inception_4e_double_3x3_1_bn.running_var: copying a param of torch.Size([256]) from checkpoint, where the shape is torch.Size([1, 256]) in current model.\n",
      "\tsize mismatch for inception_4e_double_3x3_2_bn.weight: copying a param of torch.Size([256]) from checkpoint, where the shape is torch.Size([1, 256]) in current model.\n",
      "\tsize mismatch for inception_4e_double_3x3_2_bn.bias: copying a param of torch.Size([256]) from checkpoint, where the shape is torch.Size([1, 256]) in current model.\n",
      "\tsize mismatch for inception_4e_double_3x3_2_bn.running_mean: copying a param of torch.Size([256]) from checkpoint, where the shape is torch.Size([1, 256]) in current model.\n",
      "\tsize mismatch for inception_4e_double_3x3_2_bn.running_var: copying a param of torch.Size([256]) from checkpoint, where the shape is torch.Size([1, 256]) in current model.\n",
      "\tsize mismatch for inception_5a_1x1_bn.weight: copying a param of torch.Size([352]) from checkpoint, where the shape is torch.Size([1, 352]) in current model.\n",
      "\tsize mismatch for inception_5a_1x1_bn.bias: copying a param of torch.Size([352]) from checkpoint, where the shape is torch.Size([1, 352]) in current model.\n",
      "\tsize mismatch for inception_5a_1x1_bn.running_mean: copying a param of torch.Size([352]) from checkpoint, where the shape is torch.Size([1, 352]) in current model.\n",
      "\tsize mismatch for inception_5a_1x1_bn.running_var: copying a param of torch.Size([352]) from checkpoint, where the shape is torch.Size([1, 352]) in current model.\n",
      "\tsize mismatch for inception_5a_3x3_reduce_bn.weight: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_5a_3x3_reduce_bn.bias: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_5a_3x3_reduce_bn.running_mean: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_5a_3x3_reduce_bn.running_var: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_5a_3x3_bn.weight: copying a param of torch.Size([320]) from checkpoint, where the shape is torch.Size([1, 320]) in current model.\n",
      "\tsize mismatch for inception_5a_3x3_bn.bias: copying a param of torch.Size([320]) from checkpoint, where the shape is torch.Size([1, 320]) in current model.\n",
      "\tsize mismatch for inception_5a_3x3_bn.running_mean: copying a param of torch.Size([320]) from checkpoint, where the shape is torch.Size([1, 320]) in current model.\n",
      "\tsize mismatch for inception_5a_3x3_bn.running_var: copying a param of torch.Size([320]) from checkpoint, where the shape is torch.Size([1, 320]) in current model.\n",
      "\tsize mismatch for inception_5a_double_3x3_reduce_bn.weight: copying a param of torch.Size([160]) from checkpoint, where the shape is torch.Size([1, 160]) in current model.\n",
      "\tsize mismatch for inception_5a_double_3x3_reduce_bn.bias: copying a param of torch.Size([160]) from checkpoint, where the shape is torch.Size([1, 160]) in current model.\n",
      "\tsize mismatch for inception_5a_double_3x3_reduce_bn.running_mean: copying a param of torch.Size([160]) from checkpoint, where the shape is torch.Size([1, 160]) in current model.\n",
      "\tsize mismatch for inception_5a_double_3x3_reduce_bn.running_var: copying a param of torch.Size([160]) from checkpoint, where the shape is torch.Size([1, 160]) in current model.\n",
      "\tsize mismatch for inception_5a_double_3x3_1_bn.weight: copying a param of torch.Size([224]) from checkpoint, where the shape is torch.Size([1, 224]) in current model.\n",
      "\tsize mismatch for inception_5a_double_3x3_1_bn.bias: copying a param of torch.Size([224]) from checkpoint, where the shape is torch.Size([1, 224]) in current model.\n",
      "\tsize mismatch for inception_5a_double_3x3_1_bn.running_mean: copying a param of torch.Size([224]) from checkpoint, where the shape is torch.Size([1, 224]) in current model.\n",
      "\tsize mismatch for inception_5a_double_3x3_1_bn.running_var: copying a param of torch.Size([224]) from checkpoint, where the shape is torch.Size([1, 224]) in current model.\n",
      "\tsize mismatch for inception_5a_double_3x3_2_bn.weight: copying a param of torch.Size([224]) from checkpoint, where the shape is torch.Size([1, 224]) in current model.\n",
      "\tsize mismatch for inception_5a_double_3x3_2_bn.bias: copying a param of torch.Size([224]) from checkpoint, where the shape is torch.Size([1, 224]) in current model.\n",
      "\tsize mismatch for inception_5a_double_3x3_2_bn.running_mean: copying a param of torch.Size([224]) from checkpoint, where the shape is torch.Size([1, 224]) in current model.\n",
      "\tsize mismatch for inception_5a_double_3x3_2_bn.running_var: copying a param of torch.Size([224]) from checkpoint, where the shape is torch.Size([1, 224]) in current model.\n",
      "\tsize mismatch for inception_5a_pool_proj_bn.weight: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_5a_pool_proj_bn.bias: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_5a_pool_proj_bn.running_mean: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_5a_pool_proj_bn.running_var: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_5b_1x1_bn.weight: copying a param of torch.Size([352]) from checkpoint, where the shape is torch.Size([1, 352]) in current model.\n",
      "\tsize mismatch for inception_5b_1x1_bn.bias: copying a param of torch.Size([352]) from checkpoint, where the shape is torch.Size([1, 352]) in current model.\n",
      "\tsize mismatch for inception_5b_1x1_bn.running_mean: copying a param of torch.Size([352]) from checkpoint, where the shape is torch.Size([1, 352]) in current model.\n",
      "\tsize mismatch for inception_5b_1x1_bn.running_var: copying a param of torch.Size([352]) from checkpoint, where the shape is torch.Size([1, 352]) in current model.\n",
      "\tsize mismatch for inception_5b_3x3_reduce_bn.weight: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_5b_3x3_reduce_bn.bias: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_5b_3x3_reduce_bn.running_mean: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_5b_3x3_reduce_bn.running_var: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_5b_3x3_bn.weight: copying a param of torch.Size([320]) from checkpoint, where the shape is torch.Size([1, 320]) in current model.\n",
      "\tsize mismatch for inception_5b_3x3_bn.bias: copying a param of torch.Size([320]) from checkpoint, where the shape is torch.Size([1, 320]) in current model.\n",
      "\tsize mismatch for inception_5b_3x3_bn.running_mean: copying a param of torch.Size([320]) from checkpoint, where the shape is torch.Size([1, 320]) in current model.\n",
      "\tsize mismatch for inception_5b_3x3_bn.running_var: copying a param of torch.Size([320]) from checkpoint, where the shape is torch.Size([1, 320]) in current model.\n",
      "\tsize mismatch for inception_5b_double_3x3_reduce_bn.weight: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_5b_double_3x3_reduce_bn.bias: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_5b_double_3x3_reduce_bn.running_mean: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_5b_double_3x3_reduce_bn.running_var: copying a param of torch.Size([192]) from checkpoint, where the shape is torch.Size([1, 192]) in current model.\n",
      "\tsize mismatch for inception_5b_double_3x3_1_bn.weight: copying a param of torch.Size([224]) from checkpoint, where the shape is torch.Size([1, 224]) in current model.\n",
      "\tsize mismatch for inception_5b_double_3x3_1_bn.bias: copying a param of torch.Size([224]) from checkpoint, where the shape is torch.Size([1, 224]) in current model.\n",
      "\tsize mismatch for inception_5b_double_3x3_1_bn.running_mean: copying a param of torch.Size([224]) from checkpoint, where the shape is torch.Size([1, 224]) in current model.\n",
      "\tsize mismatch for inception_5b_double_3x3_1_bn.running_var: copying a param of torch.Size([224]) from checkpoint, where the shape is torch.Size([1, 224]) in current model.\n",
      "\tsize mismatch for inception_5b_double_3x3_2_bn.weight: copying a param of torch.Size([224]) from checkpoint, where the shape is torch.Size([1, 224]) in current model.\n",
      "\tsize mismatch for inception_5b_double_3x3_2_bn.bias: copying a param of torch.Size([224]) from checkpoint, where the shape is torch.Size([1, 224]) in current model.\n",
      "\tsize mismatch for inception_5b_double_3x3_2_bn.running_mean: copying a param of torch.Size([224]) from checkpoint, where the shape is torch.Size([1, 224]) in current model.\n",
      "\tsize mismatch for inception_5b_double_3x3_2_bn.running_var: copying a param of torch.Size([224]) from checkpoint, where the shape is torch.Size([1, 224]) in current model.\n",
      "\tsize mismatch for inception_5b_pool_proj_bn.weight: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_5b_pool_proj_bn.bias: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_5b_pool_proj_bn.running_mean: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n",
      "\tsize mismatch for inception_5b_pool_proj_bn.running_var: copying a param of torch.Size([128]) from checkpoint, where the shape is torch.Size([1, 128]) in current model.\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "results_4model = {}\n",
    "counts_4model = {}\n",
    "models = [m.lower().replace('-','') for m in df['model'].values]\n",
    "\n",
    "for model_name in models:\n",
    "    if(model_name in pretrainedmodels.model_names):\n",
    "        print(model_name)\n",
    "        \n",
    "        try:\n",
    "            model = pretrainedmodels.__dict__[model_name](num_classes=1000, pretrained='imagenet')\n",
    "            model.eval()\n",
    "             \n",
    "            results_4model[model_name] = calc_model_alphas(model)\n",
    "            counts_4model[model_name] = count_layers(model)\n",
    "            \n",
    "        except Exception as e:\n",
    "            print(\"error \", model_name)\n",
    "            print(e)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['resnet152', 'resnet101', 'inceptionv3', 'resnet50', 'vgg19_bn', 'resnet34', 'vgg16_bn', 'vgg19', 'vgg16', 'vgg13_bn', 'vgg11_bn', 'resnet18', 'vgg13', 'vgg11', 'squeezenet1_1', 'squeezenet1_0', 'alexnet', 'pnasnet5large', 'nasnetalarge', 'senet154', 'polynet', 'inceptionresnetv2', 'inceptionv4', 'resnext101_64x4d', 'resnext101_32x4d', 'fbresnet152', 'cafferesnet101', 'nasnetamobile'])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_4model.keys()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['modelname'] = df['model'].apply(lambda x: x.lower().replace('-',''))\n",
    "df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "avg_alphas = {}\n",
    "avg_alphas_m1 = {}\n",
    "len_alphas = {}\n",
    "for model_name in models:\n",
    "    avg_alphas[model_name] = 0.0\n",
    "    len_alphas[model_name] = 0.0\n",
    "    avg_alphas_m1[model_name] = 0.0\n",
    "    \n",
    "    if(model_name in alphas_4model):\n",
    "        avg_alphas[model_name] = np.average(alphas_4model[model_name][0])\n",
    "        avg_alphas_m1[model_name] = np.average(alphas_4model[model_name][0][:-1])\n",
    "\n",
    "        len_alphas[model_name] = len(alphas_4model[model_name][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['avg_alpha'] = df['modelname'].apply(lambda x: avg_alphas[x] )\n",
    "df['avg_alpha_m1'] = df['modelname'].apply(lambda x: avg_alphas_m1[x] )\n",
    "\n",
    "df['num_layers'] = df['modelname'].apply(lambda x: int(len_alphas[x]) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.sort_values('model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_vgg11 = df[(df['model']=='VGG11')| (df['model']=='VGG11_BN')]\n",
    "df_vgg11['legend']='VGG11'\n",
    "\n",
    "df_vgg13 = df[(df['model']=='VGG13')| (df['model']=='VGG13_BN')]\n",
    "df_vgg13['legend']='VGG13'\n",
    "\n",
    "df_vgg16 = df[(df['model']=='VGG16')| (df['model']=='VGG16_BN')]\n",
    "df_vgg16['legend']='VGG16'\n",
    "\n",
    "df_vgg19 = df[(df['model']=='VGG19')| (df['model']=='VGG19_BN')]\n",
    "df_vgg19['legend']='VGG19'\n",
    "\n",
    "df_squeezenet = df[(df['model']=='SqueezeNet1_0')| (df['model']=='SqueezeNet1_1')]\n",
    "df_squeezenet['legend']='SqueezeNet'\n",
    "\n",
    "df_resnxt = df[(df['model']=='ResNeXt101_32x4d')| (df['model']=='ResNeXt101_64x4d')]\n",
    "df_resnxt['legend']='ResNeXT101'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_vgg19"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for mf in [df_resnxt, df_vgg11, df_vgg13, df_squeezenet]:\n",
    "#for mf in [df_vgg11, df_vgg13]:\n",
    "    x = mf['acc5'].values\n",
    "    y = mf['avg_alpha'].values\n",
    "    label = mf['legend'].values[0]\n",
    "    print(label)\n",
    "    plt.scatter(x,y,label=label)\n",
    "\n",
    "plt.legend()\n",
    "plt.title(r\"Accuracy vs Avg$(\\alpha)$\"+\"\\nPretrained ImageNet Models\")\n",
    "plt.xlabel(r\"Test Accuracy\")\n",
    "plt.ylabel(r\"Avg$(\\alpha)$\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "alphas_4model['vgg16'][0], alphas_4model['vgg16_bn'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "alphas_4model['vgg19'][0], alphas_4model['vgg19_bn'][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Redo without Final Layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for mf in [df_resnxt, df_vgg11, df_vgg13, df_vgg16, df_vgg19, df_squeezenet]:\n",
    "#for mf in [df_vgg11, df_vgg13]:\n",
    "    x = mf['acc5'].values\n",
    "    y = mf['avg_alpha_m1'].values\n",
    "    label = mf['legend'].values[0]\n",
    "    print(label)\n",
    "    plt.scatter(x,y,label=label)\n",
    "\n",
    "plt.legend()\n",
    "plt.title(r\"Accuracy vs Avg$(\\alpha])$\"+\"\\nPretrained ImageNet Models\\n(average excludes final layer)\")\n",
    "plt.xlabel(r\"Test Accuracy\")\n",
    "plt.ylabel(r\"Avg$(\\alpha)$\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### No gross correlation\n",
    "\n",
    "Have to be more clever than this"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.scatter(df['acc5'].values, df['avg_alpha_m1'].values)\n",
    "plt.ylim([1,5])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ResNet\n",
    "\n",
    "Only exception is InceptionResNetv2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x, y = [], []\n",
    "for name in df.modelname:\n",
    "    if name.startswith('resnet'):\n",
    "        print(name)\n",
    "        mf = df[df.modelname==name]\n",
    "        x = mf['acc5'].values\n",
    "        y = mf['avg_alpha'].values\n",
    "        \n",
    "        plt.scatter(x,y, label=name)\n",
    "        \n",
    "plt.title(r\"Accuracy vs Avg$(\\alpha])$\"+\"\\nPretrained Resnet### Models\")\n",
    "plt.xlabel(r\"Test Accuracy\")\n",
    "plt.ylabel(r\"Avg$(\\alpha)$\")\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x, y = [], []\n",
    "for name in df.modelname:\n",
    "    if 'resnet' in name:\n",
    "        #print(name)\n",
    "        mf = df[df.modelname==name]\n",
    "        x = mf['acc5'].values\n",
    "        y = mf['avg_alpha_m1'].values\n",
    "        if y > 0:\n",
    "            plt.scatter(x,y, label=name)\n",
    "        \n",
    "plt.title(r\"Accuracy vs Avg$(\\alpha])$\"+\"\\nPretrained ResNet Models\\n(average excludes final layer)\")\n",
    "plt.xlabel(r\"Test Accuracy\")\n",
    "plt.ylabel(r\"Avg$(\\alpha)$\")\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x, y = [], []\n",
    "for name in df.modelname:\n",
    "    if 'resnet' in name and 'inception' not in name:\n",
    "        #print(name)\n",
    "        mf = df[df.modelname==name]\n",
    "        x = mf['acc5'].values\n",
    "        y = mf['avg_alpha'].values\n",
    "        if y > 0:\n",
    "            plt.scatter(x,y, label=name)\n",
    "        \n",
    "plt.title(r\"Accuracy vs Avg$(\\alpha])$\"+\"\\nPretrained ResNet  Models\")\n",
    "plt.xlabel(r\"Test Accuracy\")\n",
    "plt.ylabel(r\"Avg$(\\alpha)$\")\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x, y = [], []\n",
    "for name in df.modelname: \n",
    "    if 'inception' in name and 'resnet' not in name:\n",
    "        #print(name)\n",
    "        mf = df[df.modelname==name]\n",
    "        x = mf['acc5'].values\n",
    "        y = mf['avg_alpha'].values\n",
    "        if y > 0:\n",
    "            plt.scatter(x,y, label=name)\n",
    "        \n",
    "plt.title(r\"Accuracy vs Avg$(\\alpha])$\"+\"\\nPretrained Inception3,4 Models\")\n",
    "plt.xlabel(r\"Test Accuracy\")\n",
    "plt.ylabel(r\"Avg$(\\alpha)$\")\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for mf inacc = [df_vgg11, df_vgg13]:\n",
    "    x = mf['acc5'].values\n",
    "    y = mf['avg_alpha'].values\n",
    "    label = mf['legend'].values[0]\n",
    "    print(label)\n",
    "    plt.scatter(x,y,label=label)\n",
    "\n",
    "plt.legend()\n",
    "plt.title(r\"Accuracy vs Avg$(\\alpha)$\"+\"\\nPretrained ImageNet Models\")\n",
    "plt.xlabel(r\"Test Accuracy\")\n",
    "plt.ylabel(r\"Avg$(\\alpha)$\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot for each layer .. BIG SCATTER PLOT"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "CHECK for RANK COLLAPSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "widgets": {
   "state": {
    "52ecdae62c394a67868cd3ee83e19861": {
     "views": [
      {
       "cell_index": 14
      }
     ]
    },
    "e232317699e24554b5ed09fa6c78c662": {
     "views": [
      {
       "cell_index": 8
      }
     ]
    }
   },
   "version": "1.2.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}