EvenMore-pytorch-models.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Even More pretrained models in PyTorch\n",
    "\n",
    "####  https://github.com/Cadene/pretrained-models.pytorch\n",
    "\n",
    "Upto 7000 layers from 34 ImageNet models\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-27T08:06:08.823252Z",
     "start_time": "2018-08-27T08:06:06.545222Z"
    }
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision.models as models\n",
    "import torchvision.transforms as transforms\n",
    "import torch.nn as nn\n",
    "\n",
    "import numpy as np\n",
    "import powerlaw\n",
    "\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "\n",
    "import matplotlib\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-27T08:06:11.499813Z",
     "start_time": "2018-08-27T08:06:08.826030Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "importing Jupyter notebook from RMT_Util.ipynb\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/charleshmartin/anaconda3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6\n",
      "  return f(*args, **kwds)\n",
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.6.5 |Anaconda, Inc.| (default, Apr 26 2018, 08:42:37) \n",
      "[GCC 4.2.1 Compatible Clang 4.0.1 (tags/RELEASE_401/final)]\n",
      "numpy version 1.14.3\n",
      "scipy version 1.1.0\n",
      "tensforflow version 1.10.1\n",
      "keras version 2.2.2\n",
      "sklearn version 0.19.1\n",
      "powerlaw version 1.4.3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import import_ipynb\n",
    "import RMT_Util"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-27T08:06:11.506733Z",
     "start_time": "2018-08-27T08:06:11.501899Z"
    }
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['figure.figsize'] = [5,5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pretrainedmodels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# does not work...I made symbolic link instead\n",
    "# %env TORCH_MODEL_ZOO=\"~/work-tmp/pretrainedmodels\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 45 Available models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "45"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pretrainedmodels.model_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Minimum dimension for W"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "MIN_DIM = 50"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### All Linear Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-27T13:12:09.422260Z",
     "start_time": "2018-08-27T13:12:09.341227Z"
    }
   },
   "outputs": [],
   "source": [
    "def count_layers(model):\n",
    "    \"\"\"Loops over a pytorch model and counts all the weight matrices, linear, conv2d, and conv2d-slices\"\"\"\n",
    "    num_linear, num_conv2d, num_conv2d_slices = 0, 0, 0\n",
    "    \n",
    "    for im, m in enumerate(model.modules()):\n",
    "        #print(im,m)\n",
    "        if isinstance(m, nn.Linear):     \n",
    "            W = np.array(m.weight.data.clone().cpu())\n",
    "            if W is not None:\n",
    "                #print(im,m, W.shape)\n",
    "                num_linear += 1\n",
    "        elif isinstance(m, nn.Conv2d):\n",
    "            #print(\"conv2d\")\n",
    "            Wtensor = np.array(m.weight.data.clone().cpu())\n",
    "            #print(Wtensor.shape)\n",
    "            Wmats = RMT_Util.get_conv2D_Wmats(Wtensor)\n",
    "            #print(im,m)\n",
    "            num_conv2d += 1\n",
    "            for W in Wmats:\n",
    "                #print(W.shape)\n",
    "                if (np.min(W.shape) >= MIN_DIM):\n",
    "                    num_conv2d_slices += 1\n",
    "                \n",
    "    return num_linear, num_conv2d, num_conv2d_slices\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FBResNet(\n",
       "  (conv1): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3))\n",
       "  (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (relu): ReLU(inplace)\n",
       "  (maxpool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
       "  (layer1): Sequential(\n",
       "    (0): Bottleneck(\n",
       "      (conv1): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "      (downsample): Sequential(\n",
       "        (0): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "        (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (1): Bottleneck(\n",
       "      (conv1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (2): Bottleneck(\n",
       "      (conv1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "  )\n",
       "  (layer2): Sequential(\n",
       "    (0): Bottleneck(\n",
       "      (conv1): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "      (downsample): Sequential(\n",
       "        (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2))\n",
       "        (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (1): Bottleneck(\n",
       "      (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (2): Bottleneck(\n",
       "      (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (3): Bottleneck(\n",
       "      (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (4): Bottleneck(\n",
       "      (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (5): Bottleneck(\n",
       "      (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (6): Bottleneck(\n",
       "      (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (7): Bottleneck(\n",
       "      (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "  )\n",
       "  (layer3): Sequential(\n",
       "    (0): Bottleneck(\n",
       "      (conv1): Conv2d(512, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "      (downsample): Sequential(\n",
       "        (0): Conv2d(512, 1024, kernel_size=(1, 1), stride=(2, 2))\n",
       "        (1): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (1): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (2): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (3): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (4): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (5): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (6): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (7): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (8): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (9): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (10): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (11): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (12): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (13): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (14): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (15): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (16): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (17): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (18): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (19): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (20): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (21): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (22): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (23): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (24): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (25): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (26): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (27): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (28): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (29): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (30): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (31): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (32): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (33): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (34): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (35): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "  )\n",
       "  (layer4): Sequential(\n",
       "    (0): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "      (downsample): Sequential(\n",
       "        (0): Conv2d(1024, 2048, kernel_size=(1, 1), stride=(2, 2))\n",
       "        (1): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (1): Bottleneck(\n",
       "      (conv1): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (2): Bottleneck(\n",
       "      (conv1): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (bn3): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "  )\n",
       "  (avgpool): AvgPool2d(kernel_size=7, stride=7, padding=0)\n",
       "  (last_linear): Linear(in_features=2048, out_features=1000, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pretrainedmodels.fbresnet152(num_classes=1000, pretrained='imagenet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "479ed3a67dc04aa3a676e2dfb3072e69",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=42), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "resnext101_64x4d\n",
      "inceptionv4\n",
      "inceptionresnetv2\n",
      "alexnet\n",
      "densenet121\n",
      "densenet169\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/torchvision/models/densenet.py:212: UserWarning: nn.init.kaiming_normal is now deprecated in favor of nn.init.kaiming_normal_.\n",
      "  nn.init.kaiming_normal(m.weight.data)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "densenet201\n",
      "densenet161\n",
      "resnet18\n",
      "resnet34\n",
      "resnet50\n",
      "resnet101\n",
      "resnet152\n",
      "inceptionv3\n",
      "squeezenet1_0\n",
      "squeezenet1_1\n",
      "vgg11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/torchvision/models/squeezenet.py:94: UserWarning: nn.init.kaiming_uniform is now deprecated in favor of nn.init.kaiming_uniform_.\n",
      "  init.kaiming_uniform(m.weight.data)\n",
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/torchvision/models/squeezenet.py:92: UserWarning: nn.init.normal is now deprecated in favor of nn.init.normal_.\n",
      "  init.normal(m.weight.data, mean=0.0, std=0.01)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vgg11_bn\n",
      "vgg13\n",
      "vgg13_bn\n",
      "vgg16\n",
      "vgg16_bn\n",
      "vgg19_bn\n",
      "vgg19\n",
      "nasnetamobile\n",
      "nasnetalarge\n",
      "dpn68\n",
      "dpn68b\n",
      "dpn92\n",
      "dpn98\n",
      "dpn131\n",
      "dpn107\n",
      "xception\n",
      "senet154\n",
      "se_resnet50\n",
      "se_resnet101\n",
      "se_resnet152\n",
      "se_resnext50_32x4d\n",
      "se_resnext101_32x4d\n",
      "cafferesnet101\n",
      "pnasnet5large\n",
      "polynet\n",
      "\n",
      "34 48 4095 6944\n"
     ]
    }
   ],
   "source": [
    "num_models, num_linear, num_conv2d, num_conv2d_slices = 0, 0, 0, 0\n",
    "\n",
    "for model_name in tqdm(pretrainedmodels.model_names[3:]):\n",
    "    print(model_name)\n",
    "    try:\n",
    "        model = pretrainedmodels.__dict__[model_name](num_classes=1000, pretrained='imagenet')\n",
    "        model.eval()\n",
    "        num_l, num_c, num_cs = count_layers(model)\n",
    "        num_models += 1\n",
    "        num_linear += num_l\n",
    "        num_conv2d += num_c\n",
    "        num_conv2d_slices += num_cs\n",
    "        #print(\"made it\")\n",
    "        #print(num_linear, num_conv2d, num_conv2d_slices)\n",
    "    except Exception as e: \n",
    "        #print(e)\n",
    "        pass\n",
    "    \n",
    "   \n",
    "print(num_models, num_linear, num_conv2d, num_conv2d_slices)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def iter_pytotch_layers(model, min_dim=MIN_DIM):\n",
    "    \"\"\"Iterate over a pytorch layers and  weight matrix.  \n",
    "    Supports linear, conv2d, and conv2d-slices\"\"\"\n",
    "    for im, m in enumerate(model.modules()):\n",
    "        if isinstance(m, nn.Linear):     \n",
    "            W = np.array(m.weight.data.clone().cpu())\n",
    "            if W is not None:\n",
    "                yield W\n",
    "        elif isinstance(m, nn.Conv2d):\n",
    "            Wtensor = np.array(m.weight.data.clone().cpu())\n",
    "            Wmats = RMT_Util.get_conv2D_Wmats(Wtensor)\n",
    "            for W in Wmats:\n",
    "                if (np.min(W.shape) >= min_dim):\n",
    "                    yield W"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "model = pretrainedmodels.fbresnet152(num_classes=1000, pretrained='imagenet')\n",
    "\n",
    "alphas, min_svs, Qs = [], [], []\n",
    "for W in tqdm(iter_pytotch_layers(model)):\n",
    "    M, N = np.min(W), np.max(W)\n",
    "    Q = N/M\n",
    "    Qs.append(Q)\n",
    "    \n",
    "    _, sv, _ = np.linalg.svd(W)\n",
    "    min_svs.append(np.min(sv))\n",
    "\n",
    "    evals = sv*sv\n",
    "    fit = powerlaw.Fit(evals, xmax=np.max(evals), verbose=False)\n",
    "    alphas.append(fit.alpha)\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "96996d5c1dae46b7b886a4978b163b74",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=45), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fbresnet152\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6d0e2f1352604b9d804669387c8545db",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/powerlaw.py:700: RuntimeWarning: divide by zero encountered in true_divide\n",
      "  (Theoretical_CDF * (1 - Theoretical_CDF))\n",
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/powerlaw.py:700: RuntimeWarning: invalid value encountered in true_divide\n",
      "  (Theoretical_CDF * (1 - Theoretical_CDF))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bninception\n",
      "skipping  bninception\n",
      "resnext101_32x4d\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8362580acbde45b59b36c921d9ada24c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "resnext101_64x4d\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "16fb6ee667d54b6d812b8b2e94258db1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inceptionv4\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a04338ae702d49c48d96879a7c2cecab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inceptionresnetv2\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cfb8a708788948f481273de3425bd88c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "alexnet\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9adc43f5305a4a3ba9c3380f33ba20df",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "densenet121\n",
      "skipping  densenet121\n",
      "densenet169\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/torchvision/models/densenet.py:212: UserWarning: nn.init.kaiming_normal is now deprecated in favor of nn.init.kaiming_normal_.\n",
      "  nn.init.kaiming_normal(m.weight.data)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "skipping  densenet169\n",
      "densenet201\n",
      "skipping  densenet201\n",
      "densenet161\n",
      "skipping  densenet161\n",
      "resnet18\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a6d9b83ce344209bfb50af56c330289",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "resnet34\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cb9775fa76824f6b92af90a9c7b072cd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "resnet50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "494d8e5af86642cea107d63f03cc9694",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "resnet101\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1f7db3d9c2584022918774cd1ca1e6c7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "resnet152\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4e56222e03dd410fb0d86edb5a5ec4d7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inceptionv3\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9e54fbe3b0f54ba19e768515352f51d6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "squeezenet1_0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/torchvision/models/squeezenet.py:94: UserWarning: nn.init.kaiming_uniform is now deprecated in favor of nn.init.kaiming_uniform_.\n",
      "  init.kaiming_uniform(m.weight.data)\n",
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/torchvision/models/squeezenet.py:92: UserWarning: nn.init.normal is now deprecated in favor of nn.init.normal_.\n",
      "  init.normal(m.weight.data, mean=0.0, std=0.01)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4177016560f9446a9722583180c8ff80",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "squeezenet1_1\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "45cf8ec6b223404f8b68686ea2b8b27a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/powerlaw.py:1178: RuntimeWarning: overflow encountered in double_scalars\n",
      "  return (self.alpha-1) * self.xmin**(self.alpha-1)\n",
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/powerlaw.py:825: RuntimeWarning: invalid value encountered in multiply\n",
      "  likelihoods = f*C\n",
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/scipy/optimize/optimize.py:563: RuntimeWarning: invalid value encountered in subtract\n",
      "  numpy.max(numpy.abs(fsim[0] - fsim[1:])) <= fatol):\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vgg11\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8d68445dc3a7444ba727947408c33576",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vgg11_bn\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f2d257903641488cb88b4f067373172e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vgg13\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2ff37aa857b44400a3961723d4f0702a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vgg13_bn\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "201230603ef648588d63447286f81b58",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vgg16\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "08db6ac69d04406e9c7d21655d3d11c5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vgg16_bn\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6491d543fa914e87a990cff4a2e37370",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vgg19_bn\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "91a6ed0e25e0438fb900c3a26095cda4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vgg19\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2aa15bbc47444895bcc0533da111c0ac",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nasnetamobile\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "98266a60538247798332237b1aa21fc8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nasnetalarge\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "085300571b21442fad9ae83d7560b1c2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dpn68\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2f21d3470cdb4796835295fd89064816",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dpn68b\n",
      "skipping  dpn68b\n",
      "dpn92\n",
      "skipping  dpn92\n",
      "dpn98\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9e877621b9734acd8d59607e0936f181",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dpn131\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "70f07787db384a48a7f0f535aee2c5a1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dpn107\n",
      "skipping  dpn107\n",
      "xception\n",
      "skipping  xception\n",
      "senet154\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b6121a0b134a4b84bed47c1fb13b2858",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "se_resnet50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5dda826998374c59b269b179dadbb7e8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "se_resnet101\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eaab6634c0314cfcb144b2d7c690e6c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "se_resnet152\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cc75238511f949d0aff2b4b8ff9f5754",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "se_resnext50_32x4d\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4beddbf0ed56486198db1c4c8a9c8383",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "se_resnext101_32x4d\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c2c93fcb590d474f87d5b671c7b5a929",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cafferesnet101\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "62711d67b53b4d509d6275f96f6f7e15",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pnasnet5large\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a0f90e3088834ca69bfb9d8b8b1d15ad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "polynet\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f3c36a762a7c462ebf105d802691335f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/powerlaw.py:1174: RuntimeWarning: overflow encountered in power\n",
      "  return x**-self.alpha\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "alphas, min_svs, Qs = [], [], []\n",
    "\n",
    "num_models, num_matrices = 0, 0\n",
    "\n",
    "\n",
    "for model_name in tqdm(pretrainedmodels.model_names):\n",
    "    try:\n",
    "        print(model_name)\n",
    "\n",
    "        model = pretrainedmodels.__dict__[model_name](num_classes=1000, pretrained='imagenet')\n",
    "        model.eval()\n",
    "        \n",
    "        for W in tqdm(iter_pytotch_layers(model)):\n",
    "            M, N = np.min(W.shape), np.max(W.shape)\n",
    "            Q = N/M\n",
    "            Qs.append(Q)\n",
    "\n",
    "            _, sv, _ = np.linalg.svd(W)\n",
    "            min_svs.append(np.min(sv))\n",
    "\n",
    "            evals = sv*sv\n",
    "            fit = powerlaw.Fit(evals, xmax=np.max(evals), verbose=False)\n",
    "            alphas.append(fit.alpha)\n",
    "            num_matrices+=1\n",
    "            \n",
    "        num_models+=1           \n",
    "    \n",
    "    except:\n",
    "        print(\"skipping \", model_name)\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUIAAAFkCAYAAABVWDK3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHi5JREFUeJzt3XmcXFWd9/HP14R9X4JCEghLRIV5gExYFFAEHFkGg44IAwIimkHADR0IDA64PcI8DiAuOAwgQUFB2YUZQXYQgmFfIhBCQkICBEjYF5Hf88c5ndxUqruq01Wp7j7f9+tVr77Lufeee6vq2+feqntKEYGZWcne1ekKmJl1moPQzIrnIDSz4jkIzax4DkIzK56D0MyK5yA0s+I5CM2seA5CMyueg9DMiucgtD6RNF3Srp2uh1lfOAj7uRw0r0t6RdIzkn4haeVO12tpqdn/rsdPOl2vvvA/j/7HQTgw7BURKwNjgK2B45d2BSQNXdrbrNgrIlauPI7sYF1sEHIQDiAR8RTwP8DmAJLeL+lGSfMlPSTpE3n6IZKu7FpO0lRJF1XGZ0raMg+vJ+liSXMlPSHpK5Vy0yUdI+l+4NVGYShpgqTHJb0s6WFJn6zMa7pOzZK0saQXJI2p7Mtzknaq1P/YXJd5uTW9fGX5usevsuw3Jd0v6UVJF9Ys2+i41V1W0i+B9YErc+v26HyMn8rH7RFJu3Szv0MlfSuv/3lJ++fl/603x83qiAg/+vEDmA7smodHAg8B3wWWAaYCxwHLAjsDLwObAhsB80n/6NYFZgBP5XVsBMzL894F3AX8e17HRsA04OOVbd+bt7tCE/XbB1gvr3df4FVg3cp2G9app/V3s/0vAlOAFYE/AD+sWfbBXP81gduA7+V53R6/yrJ35v1ZM2/jsDyvmeNWd9k6x2xTYCawXh4fBWzczb6eBPwRWA3YK6/3YWCVTr9OB/qj4xXwo8ETlN40r+QQmQH8DFgB2BF4uhoewK+BE/PwTNKp9H7AmfmN+T7gEOCKXGZb4Mma7R0L/KKy7c83Ub+6QUUK0XGV8YZ1arD/XY8v1pS5AngAuB9YrmbZagDtATyehxsdv+nAZyvz/gP4eS+OW91la48ZsAnwLLArsEwPx3lV4A1gozy+DhDAcZ1+jQ6GRyev+1jz9o6IP1YnSFoPmBkR71QmzwCG5+GbgJ1Ib7SbSAHyEeCDeRxgA2A9SfMr6xgC3FIZn9lsJSUdBBxFatUArAysXSnSTJ3qWWz/a/w3KQzHR8SbNfOq9Z9BaqWR//Z0/CAFZZfXKss2c9y6W3YRETFV0teAE4HNJP0BOCoiZtcU3Rl4NCKm5fFlgReBH9dbr/WOrxEOXLOBkZKqz+H6wFN5uCt0dszDN5FC5yMsDJ2ZwBMRsXrlsUpE7FFZZ1NdmEvagBRIRwJrRcTqpNNSVYo1U6deyZ+gnwacDZwoac2aIiMrw+uTjhs0Pn49aea49WSRYxoRF0TEDqSADeDkOsusV6k7wHjSpYWXm9ym9cBBOHBNIl2DO1rSMvkDgr2A3+T5NwEfJV3bm0VqrewGrAXck8vcCbyUL9avIGmIpM0lbb0E9VmJ9CaeC+nDEfKHOhXN1Km3fgTcFRFfAK4Cfl4z/whJI3JAHgdcmKc3On496etxe4Z0XRFJm0raWdJypFPf14G/1VlmFrClpHUlbQscCKwjadkmt2k9cBAOUBHxFvAJYHfgOdK1w4Mi4i95/qOka2u35PGXSBf0b4uIv+VpfyO9+bcEnsjrOYt0Mb639XkY+E/gdtIb/e9IH05UyzSsUze6PmHtelwKIGkcKUgPy+WOAsZIOqCy7AXANXk704Dv5W33ePwa7Gtfj9sPgOPzqfW+pA9BniOdTq9DCuxa/5v3YwrpWuanSNdgr29ym9YD5QuvZoOOpOnAFxpcXzRzi9DMzEFoZsXzqbGZFc8tQjMrnoPQzIrnIDSz4jkIs/zF1ntyDyBfabyElUTSzyV9q8my50r6Xrvr1Cq5552dOl2PTnIQLnQ0cGO+Ver0vq4sv3GqXwJ+U9LLlfk3SnqjMv+RmuX3lzRD0quSLqveOiZpTUmX5nkzJO3fQz1OlPSrvu5Pq1X2f2Rl2q75u3/NLN/jfuXut66umfZYN9P2a7S9iDgsIr7bTN0akRSSNulh/udymVNqpu+dp5/b5HaaCuSI2CwibmxmnYOVg3ChDUhdXPWa6vTTl984CzoTJd0N8NuaYkdWymxaWd9mwH+RbqN6N+mm/Z9Vlvsp8FaedwBwRl5moHkVaKqVtQRuBraXNARA0ntIXW+NqZm2SS7b3zwO7Fvz2joIeLRVG6j3ui2VgxCQdD3pHtif5NbZeyWtJuk8pY43Z0g6vusG/fwf+zZJp0p6gdRzSE/rXwn4J2Bik1U6ALgyIm6OiFdIYfEpSatU1vWtiHglIm4l9bxyYJP7GpIOzy2hlyV9V6mD09slvSTpoq77VyWtIen3+RjMy8MjKuvaUNLNeT1/lPTTaitN0naS/qTU8el9dU6/Tgf+ubvWkbrp/FTSbqTb0PbNz9d9dRb/Myn4ujp7/TBwA/BIzbTHu3p6kfQ+Sdcqdfb6iKTPVOqySOtKqUPUOZJmS/pCnVbeGpKuysdmkqSN83JdoXtfrvu+9faddLvdA8DH83JrAh8iPdfVY/RbSU8rdQB7c9c/REnjSa+jo/N2rszTF+tsV5WfDlC6b/o4Lexg966uVnuD47OHUge4Lyt1MvvNbvarf+p0P2D95QHcSLodq2v8POByYBVSt1KPAofmeZ8D3ga+DAylm05LK+s6iHSfq2q2N5d0j+ltwE6VeZcDx9Ss4xXg74GtgNdr5n2TFJz1tn0i8KvKeJDeTKsCmwFvAteROgFYjdTR58G57Fqk0F0xH4ffApdV1nU78ENSl1A7AC91bYvUndXzpD4A3wV8LI8Pqx5v4JTKMrsC0/Nwo85PF9mvbvb9BuDrefgnwOeB79dMOycPr0TqVeaQ/JyOyc/NZnn+uSzs1HU3UlBtlo/NL/Nx3aRS9gVgm7yu84Hf1DwHm/RQ788BtwL7AxfmaYeTzhK+B5xbKfv5/NwsR+qF597KvAV1rkybTk1nuyzaP+K/kgJ4U1LPQVvk10Gj4zMH2DEPrwGM6fR7ujcPtwjryKdO+wLHRsTLETGd1KFAtdU1OyJ+HBFvR8TrDVZ5MHBe5FdJdgzpzT2c1EnplV2tBlI/fi/WrONF0gu+p3nNOjkiXoqIh0hdZV0TEdMi4kXSTwFsBRARz0fExRHxWqTunr5P6jILSeuTfj/l3yPirVjYMu3yWeDqiLg6It6JiGuByaRgrPoBsFedU/utSaH5nbz+aaRuvhpez6u4idTqg9T11y35UZ3W1f3XP5JC+Bf5Ob0buBj4dJ31fobUCetDEfEa8O06ZS6JiDsj4m1SEPbqZwiyS4GdJK1G+md6Xm2BiDgnv0bfJP1z2CKX78npETGzm9ftF4DjI+KRSO6LiOdpfHz+CnxA0qoRMS/PHzAchPWtTWqFzKhMq+20s6kOS/NpxUeoeRFHxKSuF3BETCS1CrtC4hVSi61qVVJX8j3Na9YzleHX64yvnOu+oqT/ypcGXiJdS1s9/6NYD3ghB0GX6jHZANgnnxbPV+ppZQdSN/0LRMRcUsvsOzV1XND5aWX540jXRZt1M7CDpDVIofoY8CfgQ3na5iy8PrgBsG3N9g4A3lNnvevV7Gu910Jtx6y9/uXBHFRXkX6sa+2IWKQ3n3wae1I+jX2J1LKDRTvDraen1+5I0vXJWo2Ozz+RXr8zJN0k6YMN6tCv+GJpfc+R/sNtQDpVhMU77Wz23sSDgD/Fwp6FuxMs7MT0IdIpCQCSNiKd+jwKvAMMlTQ6v7HJZZfog54GvkE6Rdo2Ip5W+nGle3I95wBrSlqxEobVTlBnAr+MiC82sZ3/RzrtvbNm+SciYnQ3yzRz/G8nne6PJ3cJFhEvSZqdp82OiCcq27spIj7WxHrnACMq4yO7K9gC55G62qrX6twfGEe+pEDa13ksfB11d4x6OnYzgY1JZwq107s9PhHxZ2CcpGVInfNeRHuPS0u5RVhHpP7mLgK+r/QBxQakvu6W5GsoB5Gu1SwgaXVJH5e0fL5YfQDpdO0Pucj5pNPFHZU+HPkO6VTr5Yh4FbgE+I6klSRtT3oz/HIJ6tbIKqQW4vx8sf6ErhkRMYN0qnuipGVzC2CvyrK/yvvw8dxyWV7STqp82FJZ13zSpYejK5MbdX76DDBKi/YwXbve13Mdj2LRbvRvzdOqnxb/HnivpAOVOmpdRtLWkt5fZ9UXAYco/QreiqTrmL2xoGPWJtxEur5ar0v+VUjXeJ8nXav8v33YTpezgO9KGq3k/0haix6OT37+D5C0WkT8lXStuKf+JfsdB2H3vkz6esc00hvnAuCc3qwgh8MIFv/azDKki95dH5Z8mfS7HI8A5Gt3h5EC8VnSC/7wyvKHk37A6VnS13K+lJdptdPydp4D7iB1Dlp1AOn3Rp7P+3Mh6Y1JRMwkBfRxpP2cSboQ391r7kdU3jzRuPPTrmP6vKSerkfdROrs9NbKtFvytAVBmK+B/gPpGuRs0qntyaSW+CIi4n9In3jfQPolvNvzrNrfS+nOicDEfIr5mZ4K5ut010XEC3Vmn0f+NUDSmcsdNfPPJl23my/psibrdgop6K8hBdrZpA9VGh2fA4Hp+RT9MNI14gHDvc9Yy0i6EPhLRJzQsPAgkluND5J+Qe/tTtfHes8tQlti+dRoY0nvUvpu3zig2ZbHgCbpk/mUcA1Sy+hKh+DA5SC0vngP6fuAr5BOFb8UEUv6I0wDzb+QTvkfJ53Sf6mz1bG+8KmxmRWvX3x9Zu21145Ro0Z1uhpmNsjcddddz0XEsEbl+kUQjho1ismTJ3e6GmY2yEia0biUrxGamTkIzcwchGZWPAehmRXPQWhmxXMQmlnxHIRmVjwHoZkVz0FoZsVzEJpZ8RyEZla8fnGvcauNmnDVguHpJ+3ZwZqY2UDgFqGZFc9BaGbFcxCaWfEchGZWPAehmRXPQWhmxXMQmlnxHIRmVjwHoZkVz0FoZsVzEJpZ8RyEZlY8B6GZFc9BaGbFcxCaWfEchGZWPAehmRXPQWhmxXMQmlnxHIRmVjwHoZkVb9AH4agJVy3yq3ZmZrUGfRCamTXiIDSz4jkIzax4DkIzK56D0MyK5yA0s+I5CM2seA5CMyueg9DMiucgNLPiOQjNrHgOQjMrnoPQzIrXVBBK+rqkhyQ9KOnXkpaXtKGkSZIek3ShpGVz2eXy+NQ8f1Q7d8DMrK8aBqGk4cBXgLERsTkwBNgPOBk4NSJGA/OAQ/MihwLzImIT4NRczsys32r21HgosIKkocCKwBxgZ+B3ef5EYO88PC6Pk+fvIkmtqa6ZWes1DMKIeAr4IfAkKQBfBO4C5kfE27nYLGB4Hh4OzMzLvp3Lr9XaapuZtU4zp8ZrkFp5GwLrASsBu9cpGl2L9DCvut7xkiZLmjx37tzma2xm1mLNnBrvCjwREXMj4q/AJcCHgNXzqTLACGB2Hp4FjATI81cDXqhdaUScGRFjI2LssGHD+rgbZmZLrpkgfBLYTtKK+VrfLsDDwA3Ap3OZg4HL8/AVeZw8//qIWKxFaGbWXzRzjXAS6UOPu4EH8jJnAscAR0maSroGeHZe5GxgrTz9KGBCG+ptZtYyQxsXgYg4ATihZvI0YJs6Zd8A9ul71czMlg7fWWJmxXMQmlnxHIRmVjwHoZkVz0FoZsVzEJpZ8RyEZlY8B6GZFc9BaGbFcxCaWfEchGZWPAehmRXPQWhmxXMQmlnxHIRmVjwHoZkVz0FoZsVzEJpZ8RyEZlY8B6GZFc9BaGbFcxCaWfEchGZWPAehmRXPQWhmxXMQmlnxHIRmVjwHoZkVz0FoZsVzEJpZ8RyEZlY8B6GZFc9BaGbFcxCaWfEchGZWPAehmRXPQWhmxXMQmlnxHIRmVjwHoZkVz0FoZsUb2ukKLC2jJly1YHj6SXt2sCZm1t+4RWhmxXMQmlnxHIRmVrxBdY2weh3QzKxZTbUIJa0u6XeS/iJpiqQPSlpT0rWSHst/18hlJel0SVMl3S9pTHt3wcysb5o9Nf4R8L8R8T5gC2AKMAG4LiJGA9flcYDdgdH5MR44o6U1NjNrsYZBKGlV4MPA2QAR8VZEzAfGARNzsYnA3nl4HHBeJHcAq0tat+U1NzNrkWZahBsBc4FfSLpH0lmSVgLeHRFzAPLfdXL54cDMyvKz8jQzs36pmSAcCowBzoiIrYBXWXgaXI/qTIvFCknjJU2WNHnu3LlNVdbMrB2aCcJZwKyImJTHf0cKxme6Tnnz32cr5UdWlh8BzK5daUScGRFjI2LssGHDlrT+ZmZ91jAII+JpYKakTfOkXYCHgSuAg/O0g4HL8/AVwEH50+PtgBe7TqHNzPqjZr9H+GXgfEnLAtOAQ0ghepGkQ4EngX1y2auBPYCpwGu5rJlZv9VUEEbEvcDYOrN2qVM2gCP6WC8zs6XGt9iZWfEchGZWPAehmRXPQWhmxXMQmlnxHIRmVjwHoZkVz0FoZsVzEJpZ8RyEZlY8B6GZFc9BaGbFcxCaWfEchGZWPAehmRXPQWhmxXMQmlnxHIRmVjwHoZkVz0FoZsVzEJpZ8RyEZlY8B6GZFc9BaGbFcxCaWfEchGZWPAehmRXPQWhmxXMQmlnxHIRmVjwHoZkVz0FoZsUb2ukKdMKoCVctGJ5+0p4drImZ9QduEZpZ8RyEZlY8B6GZFc9BaGbFcxCaWfEchGZWPAehmRXPQWhmxXMQmlnxHIRmVjwHoZkVz0FoZsVzEJpZ8ZoOQklDJN0j6fd5fENJkyQ9JulCScvm6cvl8al5/qj2VN3MrDV60yL8KjClMn4ycGpEjAbmAYfm6YcC8yJiE+DUXM7MrN9qKggljQD2BM7K4wJ2Bn6Xi0wE9s7D4/I4ef4uubyZWb/UbIvwNOBo4J08vhYwPyLezuOzgOF5eDgwEyDPfzGXX4Sk8ZImS5o8d+7cJay+mVnfNQxCSf8IPBsRd1Un1ykaTcxbOCHizIgYGxFjhw0b1lRlzczaoZmu+rcHPiFpD2B5YFVSC3F1SUNzq28EMDuXnwWMBGZJGgqsBrzQ8pqbmbVIwxZhRBwbESMiYhSwH3B9RBwA3AB8Ohc7GLg8D1+Rx8nzr4+IxVqEZmb9RV++R3gMcJSkqaRrgGfn6WcDa+XpRwET+lZFM7P26tWv2EXEjcCNeXgasE2dMm8A+7SgbmZmS4XvLDGz4jkIzax4DkIzK17xQThqwlWMmnBVp6thZh1UfBCamTkIzax4DkIzK56D0MyK5yA0s+I5CM2seA5CMyueg9DMiucgNLPiOQjNrHgOQjMrnoPQzIrXq45ZB7NqxwvTT9qzgzUxs6XNLUIzK56D0MyK5yA0s+I5CM2seA5CMyueg9DMiucgrMO/Y2JWFgehmRXPQWhmxXMQmlnxHIRmVjwHoZkVz0FoZsVzEJpZ8QZ8N1z+vp+Z9ZVbhGZWPAehmRVvwJ8at5N7rTYrg1uEZlY8B6GZFc9BaGbF8zXCXvJ1Q7PBx0HYJH9f0Wzw8qmxmRXPQWhmxXMQmlnxHIRmVjwHoZkVz0FoZsVrGISSRkq6QdIUSQ9J+mqevqakayU9lv+ukadL0umSpkq6X9KYdu9Ep/hnP80Gh2ZahG8D34iI9wPbAUdI+gAwAbguIkYD1+VxgN2B0fkxHjij5bXuxxyOZgNPwyCMiDkRcXcefhmYAgwHxgETc7GJwN55eBxwXiR3AKtLWrflNTcza5FeXSOUNArYCpgEvDsi5kAKS2CdXGw4MLOy2Kw8rXZd4yVNljR57ty5va+5mVmLNH2LnaSVgYuBr0XES5K6LVpnWiw2IeJM4EyAsWPHLjZ/IPGpsNnA1lSLUNIypBA8PyIuyZOf6TrlzX+fzdNnASMri48AZremumZmrdfMp8YCzgamRMQplVlXAAfn4YOByyvTD8qfHm8HvNh1Cm1m1h81c2q8PXAg8ICke/O044CTgIskHQo8CeyT510N7AFMBV4DDmlpjc3MWqxhEEbErdS/7gewS53yARzRx3qZmS01A7Y/Qn9AYWat4lvszKx4DkIzK96APTXu7/zbJmYDh1uEZlY8B6GZFc9BaGbFcxCaWfEchGZWPAfhUuDOWs36NwehmRXPQWhmxXMQmlnxHIRmVjwHoZkVz/caL0W+/9isf3KL0MyK5yA0s+I5CM2seA5CMyueg9DMiucgNLPiOQg7xB0xmPUf/h5hh3UXhv6eodnS4xahmRXPQWhmxXMQmlnxHIRmVjwHoZkVz0FoZsVzEPZT/p6h2dLjIDSz4jkIzax4vrOkn3Ov1mbt5xahmRXPQWhmxXMQmlnxHIQDiL9SY9YeDsIBzuFo1nf+1HgAcvCZtZaDcJDw12zMlpxPjQchny6b9Y6D0MyK5yAshFuJZt3zNcLC1AtDX1O00rUlCCXtBvwIGAKcFREntWM71rNmW4D+JT0rXctPjSUNAX4K7A58APhnSR9o9XbMzFqlHS3CbYCpETENQNJvgHHAw23YlrVRb68pNmpBNnta7q8C2dLWjiAcDsysjM8Ctm3Ddqwf6+tpee38aiDWm9bssmb1tCMIVWdaLFZIGg+Mz6NvSnqwDXXpb9YGnut0JdpFJy8YbPl+Vtbd47Rml22BQf1cVgz0/dygmULtCMJZwMjK+Ahgdm2hiDgTOBNA0uSIGNuGuvQr3s/Bo4R9hHL2sx3fI/wzMFrShpKWBfYDrmjDdszMWqLlLcKIeFvSkcAfSF+fOSciHmr1dszMWqUt3yOMiKuBq3uxyJntqEc/5P0cPErYRyhkPxWx2OcYZmZF8b3GZla8jgehpN0kPSJpqqQJna5Pu0iaLukBSfdKmtzp+rSKpHMkPVv9+pOkNSVdK+mx/HeNTtaxr7rZxxMlPZWfz3sl7dHJOvaVpJGSbpA0RdJDkr6apw+q57I7HQ3CAm/H+2hEbDnIvo5wLrBbzbQJwHURMRq4Lo8PZOey+D4CnJqfzy3zdfGB7G3gGxHxfmA74Ij8Xhxsz2VdnW4RLrgdLyLeArpux7MBIiJuBl6omTwOmJiHJwJ7L9VKtVg3+zioRMSciLg7D78MTCHdJTaonsvudDoI692ON7xDdWm3AK6RdFe+q2Ywe3dEzIH0BgPW6XB92uVISffnU+dBc8ooaRSwFTCJQp7LTgdhU7fjDRLbR8QY0mWAIyR9uNMVsj45A9gY2BKYA/xnZ6vTGpJWBi4GvhYRL3W6PktLp4OwqdvxBoOImJ3/PgtcSrosMFg9I2ldgPz32Q7Xp+Ui4pmI+FtEvAP8N4Pg+ZS0DCkEz4+IS/LkQf9cQueDsIjb8SStJGmVrmHgH4DB3MnEFcDBefhg4PIO1qUtusIh+yQD/PmUJOBsYEpEnFKZNeifS+gHX6jOXzs4jYW3432/oxVqA0kbkVqBkO7muWCw7KekXwM7kXopeQY4AbgMuAhYH3gS2CciBuyHDd3s406k0+IApgP/0nUtbSCStANwC/AA8E6efBzpOuGgeS670/EgNDPrtE6fGpuZdZyD0MyK5yA0s+I5CM2seA5CMyueg9DMiucgNLPiOQitoyR9JfeBd36b1r+CpJskDZH0d5JmSPpSZf6ykm6W1JafrbCBwUFonXY4sEdEHNBMYSW9ed1+Hrgk3xf8AOk2zoO6Zubu364D9u3FOm2QcRBa20jaTNIfJT0q6VuSfixp68r8nwMbAVdI+nqedpSkB/Pja3naqNxq/BlwN4t21IGk30i6UNKk3OLbszL7ABa9P/ZZYLOaql6Wy1mhfIudtYWk5UmhtQ8wDfgLcFdEfKqm3HRgbEQ8J+nvSb1Bb0fqom0S8FlgXl7HhyLijjrbmgJcFhHH5ntmT4mIbXJHHk9GxHsqZX8LfAJ4b0TMyNOGAE9HxLBWHgMbONwitHbZFbgnIh6KiNeBZWncZ98OwKUR8WpEvAJcAuyY583oJgRXIHWG8O086WGgq5PUtYH5lbK7ASsBV1FpFUbE34C3unoIsvI4CK1dtiK1CJG0HvBKRNzWYJl6HfV2ebWb6ZsDj0XEG3l8DHBfHn4dWD7XYXngP0jXJB/Iy1UtB7yBFclBaO3yJqmjXYAfkFqEjdwM7C1pxdxv4ydJXUP1ZAtgfUnL52W+DZwKEBHzgCE5BI8HzouI6dQEoaS1gLkR8ddmd84GFwehtcsFwIclPUJqod0u6bSeFsg/HnQucCfp+uBZEXFPg+1sAZwP3Ejq6PeMmpbnNaQORT9G6vcSFm8RfhQY6L9CZ33gD0tsQJN0M/DFiHikm/lbAUdFxIE9rOMS4Nju1mGDn1uENtBtDDzW3czcorwhfzK8mPzJ8mUOwbK5RWhmxXOL0MyK5yA0s+I5CM2seA5CMyueg9DMiucgNLPiOQjNrHgOQjMr3v8H6GylWgfbbKAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(alphas, bins=100);\n",
    "plt.title(r'Power law Exponents $\\alpha$'+\"\\nfor 7500 ImageNet Weight Matrices\")\n",
    "plt.xlabel(r'$\\alpha$ for $p(\\lambda)~\\lambda^{-alpha}$')\n",
    "plt.savefig(\"img/even-more-alphas.png\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(87.91150206584032, 5.557776889244303)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = np.array(alphas)\n",
    "100.0*len(a[a<5])/len(a), 100.0*len(a[a<1.5])/len(a)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Rank Loss\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUIAAAEyCAYAAACPj9ldAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAEYZJREFUeJzt3X+M5Hddx/Hnm5YWo4aj7RaPu9a9htMAGvlxgSaEhLSApUWuaKtHCFyg5qLSBINGDjGK4h9XjRZJDE2lhIMQ2lrFXrgaqP0h0aSFO3oUSsFu60mPu/QO+wO1UlN5+8d8tox7s7uzs9/Znd3385FM5vvjMzPv+c7Maz/fnxuZiSRV9qzVLkCSVptBKKk8g1BSeQahpPIMQknlGYSSyjMIJZVnEEoqzyCUVN6pq10AwFlnnZXT09OrXYakdebgwYPfzcypxdpNRBBOT09z4MCB1S5D0joTEf82TDtXjSWVZxBKKs8glFSeQSipPINQUnkGoaTyDEJJ5RmEksozCCWVZxBKKs8glFTeRJxrLGn8pnfvf2b48J5LVrGSyWOPUFJ5BqGk8gxCSeUZhJLKMwgllWcQSirPIJRUnkEoqTyDUFJ5BqGk8gxCSeUZhJLKMwgllefVZ6R1rv+qMxrMHqGk8gxCSeUZhJLKMwgllWcQSirPIJRUnkEoqTyDUFJ5BqGk8gxCSeUZhJLKMwgllWcQSipv6CCMiFMi4p6I+Fwb3xIRd0fEAxFxQ0Sc1qaf3sZn2vzp8ZQuSd1YSo/wPcD9feNXAVdn5lbgMeCKNv0K4LHMfCFwdWsnSRNrqCCMiM3AJcDH2ngAFwA3tSZ7gUvb8PY2Tpt/YWsvSRNp2B7hh4HfAX7Qxs8EHs/Mp9v4EWBTG94EPAzQ5j/R2kvSRFo0CCPiTcDxzDzYP3lA0xxiXv/z7oqIAxFx4MSJE0MVK0njMEyP8NXAmyPiMHA9vVXiDwMbImL2Uv+bgaNt+AhwDkCb/1zg0blPmpnXZua2zNw2NTW1rDchaWmmd+/3Ev59Fg3CzHx/Zm7OzGlgB3B7Zr4NuAO4rDXbCdzchve1cdr82zPzpB6hJE2K5RxH+D7gvRExQ28b4HVt+nXAmW36e4HdyytRksZrSf/FLjPvBO5sww8BrxzQ5vvA5R3UJkkrwjNLJJVnEEoqzyCUVJ5BKKk8g1BSeQahpPIMQknlLek4Qklrg6fPLY09QknlGYSSyjMIJZVnEEoqzyCUVJ5BKKk8g1BSeQahpPIMQknlGYSSyjMIJZVnEEoqzyCUVJ5BKKk8g1BSeQahpPIMQknlGYSSyjMIJZVnEEoqzyCUVJ5BKKk8g1BSeQahpPIMQknlGYSSyjMIJZVnEEoqzyCUVJ5BKKm8U1e7AEmrZ3r3/meGD++5ZBUrWV32CCWVZxBKKs8glFSeQSipPINQUnkGoaTyPHxG0oqZ1MN17BFKKs8glFTeokEYEc+JiC9FxFcj4r6I+MM2fUtE3B0RD0TEDRFxWpt+ehufafOnx/sWJGl5hukRPgVckJk/B7wUuCgizgeuAq7OzK3AY8AVrf0VwGOZ+ULg6tZOkibWokGYPf/ZRp/dbglcANzUpu8FLm3D29s4bf6FERGdVSxJHRtqG2FEnBIRh4DjwK3Ag8Djmfl0a3IE2NSGNwEPA7T5TwBnDnjOXRFxICIOnDhxYnnvQpKWYaggzMz/zcyXApuBVwIvGtSs3Q/q/eVJEzKvzcxtmbltampq2HolqXNL2mucmY8DdwLnAxsiYvY4xM3A0TZ8BDgHoM1/LvBoF8VK0jgMs9d4KiI2tOEfAV4H3A/cAVzWmu0Ebm7D+9o4bf7tmXlSj1CSJsUwZ5ZsBPZGxCn0gvPGzPxcRHwDuD4i/hi4B7iutb8O+FREzNDrCe4YQ92S1JlFgzAz7wVeNmD6Q/S2F86d/n3g8k6qk6QV4JklksozCCWVZxBKKs/LcEnrSP9lrjQ8e4SSyjMIJZVnEEoqzyCUVJ5BKKk8g1BSeR4+I2nZJvW/0w3LHqGk8gxCSeUZhJLKMwgllWcQSirPIJRUnkEoaV7Tu/eXuKKNQSipPINQUnkGoaTyPMVO0thN+nZGe4SSgDo7RgaxRyjp/6kYhvYIJZVnEEoqzyCUVJ5BKKk8g1BSeQahpPIMQknlGYSSyjMIJZVnEEoqzyCUVJ5BKKk8g1BSeQahpPK8DJeksVhLl/OyRyipPINQUnkGoaTyDEJJ5RmEksozCCWVZxBKKs8glFSeQSipvEWDMCLOiYg7IuL+iLgvIt7Tpp8REbdGxAPt/nltekTERyJiJiLujYiXj/tNSNJyDNMjfBr4rcx8EXA+8O6IeDGwG7gtM7cCt7VxgDcCW9ttF/DRzquWpA4tGoSZeSwzv9KG/wO4H9gEbAf2tmZ7gUvb8Hbgk9lzF7AhIjZ2XrmkNW169/6JOR95SRddiIhp4GXA3cDzM/MY9MIyIs5uzTYBD/c97EibdmzOc+2i12Pk3HPPHaF0STB5FzeYtHqGMfTOkoj4MeBvgN/MzO8t1HTAtDxpQua1mbktM7dNTU0NW4YkdW6oIIyIZ9MLwU9n5t+2yY/MrvK2++Nt+hHgnL6HbwaOdlOuJHVvmL3GAVwH3J+Zf943ax+wsw3vBG7um/6Otvf4fOCJ2VVoSZpEw2wjfDXwduBrEXGoTftdYA9wY0RcAXwbuLzNuwW4GJgBngTe2WnFktSxRYMwM/+Jwdv9AC4c0D6Bdy+zLklaMZ5ZIqk8g1BSeQahpPIMQknlGYSSyvP/GktaVP9pc4f3XLKKlYyHPUJJ5RmEksozCCWVZxBKKs8glFSeQSipPINQUnkGoaTyDEJJ5RmEksozCCWVZxBKKs8glFSeQSipPINQUnkGoaTyDEJJ5RmEksrzUv2SRtJ/+f61zh6hpPIMQknluWosrVHradV0tdkjlFSeQSipPINQUnkGoaTyDEJJ5RmEksrz8BlJq6r/MKDDey5ZlRrsEUoqzyCUVJ5BKGliTO/evypnzBiEkspzZ4m0hnh+8XjYI5RUnkEoqTyDUFJ5BqGk8gxCSeUZhJLKMwgllWcQSipv0SCMiI9HxPGI+HrftDMi4taIeKDdP69Nj4j4SETMRMS9EfHycRYvSV0Ypkf4CeCiOdN2A7dl5lbgtjYO8EZga7vtAj7aTZmSND6LBmFmfhF4dM7k7cDeNrwXuLRv+iez5y5gQ0Rs7KpYSRqHUbcRPj8zjwG0+7Pb9E3Aw33tjrRpJ4mIXRFxICIOnDhxYsQyJGn5ut5ZEgOm5aCGmXltZm7LzG1TU1MdlyFJwxs1CB+ZXeVt98fb9CPAOX3tNgNHRy9PksZv1Mtw7QN2Anva/c1906+MiOuBVwFPzK5CSxqNl94av0WDMCI+A7wWOCsijgB/QC8Ab4yIK4BvA5e35rcAFwMzwJPAO8dQszSvSfhHQFp7Fg3CzHzrPLMuHNA2gXcvtyhJWkmeWSKpPINQUnkGoaTyDEJJ5flf7CQtyXo8nMceoaTyDEJJ5RmEksozCCWVZxBKKs8g1Lo1vXv/utzDqe4ZhJLKMwgllecB1VoXXAXWchiEWrOGDT+vUajFuGosqTx7hNKEcnV/5dgjlFSePUKtOfaU1DV7hJLKMwglleeqsTSklTgMx9X+npU+5MkeoaTyDEJJ5RmEWhKv6KL1yG2EHRsUEp7WtXxdh6+n3amfPUJJ5dkjXAGzvQ97HqtvUlfrJ7WuKgxCjdV6XQX1j9v64qqxpPIMQknluWo8JFeFNMh6XfWvxh6hpPLsES7APXmrz89AK8Eg1FispQBzs4cMwg6spR+9pJO5jVBSefYI1Rl7xkvj8pocBqFGMsphI26L0yhW4ntjEGpVTVI4rtQxgfYEJ49BqInggclaTQahNEb2/tYGg3CAlfzy2hOabAZZDQbhEhlcy7dYuKzV8Jmk7Z1aGoNQi1qvwSXNWvdB6P8Q0SgM91rWZRCO8iVeqcdImjxjCcKIuAj4C+AU4GOZuWccr6NuuN2zW/6BXHs6D8KIOAX4S+D1wBHgyxGxLzO/0fVrdaHyl3bQe6+8PFTXOHqErwRmMvMhgIi4HtgOTEwQrtaPfSk9r4W2bQ673XPQ6xl00skiM7t9wojLgIsy81fb+NuBV2XmlfM9Ztu2bXngwIElvY4/aKmWUTbbRMTBzNy2WLtx9AhjwLST0jYidgG72uhTEfH1MdSyFGcB313lGmAy6rCGH5qEOqwBiKtGquEnh2k0jiA8ApzTN74ZODq3UWZeC1wLEBEHhkntcZqEGialDmuYrDqsYfw1jOPCrF8GtkbElog4DdgB7BvD60hSJzrvEWbm0xFxJfB5eofPfDwz7+v6dSSpK2M5jjAzbwFuWcJDrh1HHUs0CTXAZNRhDT80CXVYQ8/Yauh8r7EkrTX+8yZJ5RmEkspbsSCMiD+NiG9GxL0R8dmI2NA37/0RMRMR34qIn5/n8Vsi4u6IeCAibmh7pJdaw+URcV9E/CAitvVNf1tEHOq7/SAiXjrg8R+MiO/0tbu4wxqmI+K/+577mnkef0ZE3NqWw60R8byl1rBIHa+PiIMR8bV2f8E8jx/bsmjzVuQ7Mef5buh7P4cj4tA87Q635XMoIpZ2JsBwdQy1bCPiorZ8ZiJid8c1zPt7ndOu82Wx2PuKiNPbZzXTPv/pZb9oZq7IDXgDcGobvgq4qg2/GPgqcDqwBXgQOGXA428EdrTha4BfH6GGFwE/DdwJbJunzc8CD80z74PAby9zOQysAZgGvj7E4/8E2N2Gd88uxw7reBnwgjb8M8B3VmFZrNh3YoHa/gz4/XnmHQbO6uq1Rlm29I7IeBA4DzitLa8Xd1jDwN/ruJfFMO8L+A3gmja8A7hhua+7Yj3CzPxCZj7dRu+id6A19M5Dvj4zn8rMfwVm6J2v/IyICOAC4KY2aS9w6Qg13J+Z31qk2VuBzyz1uTuuYSHb6b1/GHE5LFRHZt6TmbMHwN8HPCciTh+p0hFrYAW/E4O05/5lxvg96MAz5/Rn5v8As+f0d2KB3+u4DfO++n8DNwEXts9sZKu1jfBdwN+34U3Aw33zjrRp/c4EHu/7YAa16cqvsPAP4Mq2uvDxUVdLF7AlIu6JiH+MiNfM0+b5mXkMoN2f3XEN/X4JuCczn5pn/riWxWp/J14DPJKZD8wzP4EvtE0Hu+Zps1yLLdthllFX+n+vc3W9LIZ5X8+0aZ//E/S+DyPr9DjCiPgH4CcGzPpAZt7c2nwAeBr49OzDBrSfe0zPUOcvD1vDfCLiVcCTmTnfec8fBT7UXvtD9Faf3tVRDceAczPz3yPiFcDfRcRLMvN7C9W8kGUui5fQWyV6wzxNxrksOv1OjFDPYmsFr87MoxFxNnBrRHwzM7+42GsPWwfDLduR3v+wNSzwe51r2ctiblkDpnXy2S+k0yDMzNctND8idgJvAi7MtoLPcOcmfxfYEBGntr8AA89fHqaGRexggR9AZj4yOxwRfwV8rqsaWq/rqTZ8MCIeBH4KmLsB+pGI2JiZxyJiI3B8geccaVlExGbgs8A7MvPBeZ57bMuCjr8TS6knIk4FfhF4xQLPcbTdH4+Iz9JbnVvSj3/Y5bLAsh3qnP7l1DDP73Xucyx7WcwxzPuabXOkfV7PBR5dxmuu6F7ji4D3AW/OzCf7Zu0DdrQ9QVuArcCX+h/bPoQ7gMvapJ3Agj2aEep7FnA5vW0S87XZ2Df6FqCzK+ZExFT0LmpLRJxHbzk8NKDpPnrvH8azHDYA+4H3Z+Y/L9BubMuC1f1OvA74ZmYeGTQzIn40In58dphej7nTKycNuWzHek7/Ar/X/jbjWBbDvK/+38BlwO3zBfXQutrbM8TeoBl66/WH2u2avnkfoLen6FvAG/um38IP92CeR+/HMAP8NXD6CDW8hd5fk6eAR4DP9817LXDXgMd8jLZHE/gU8DXg3vZhbOyqBnrb4+6jt5fsK8AvzFPDmcBtwAPt/owRP4/56vg94L/6PqdDwNkruSxW8jsxoKZPAL82Z9oLgFv6XvOr7XYfvdXIrn8rA5dtfx1t/GLgX9py6rQO5vm9rsSyGPS+gD+iF8oAz2mf90z7/M9b7mt6ip2k8jyzRFJ5BqGk8gxCSeUZhJLKMwgllWcQSirPIJRU3v8BYelZPjudf4MAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(np.log10(min_svs), bins=100);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Recompute Qs...screwd it up above somehow ?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fc86ba49ec4c4d168e0af24b0cfc79b1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=45), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fbresnet152\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f1995fc127d1485db02686a76f49f1bf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bninception\n",
      "skipping  bninception\n",
      "resnext101_32x4d\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "48b8e66727bb44cb8fff443aae6c91ba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "resnext101_64x4d\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f79589c859f14347acc334da4c7bb7c4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inceptionv4\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "528d944baa0240279f408e5152cb48d1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inceptionresnetv2\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "10ab2d1d58ad4d0eada470ff92d23135",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "alexnet\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f11a92c514354925b8cfc17a41f5d19b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "densenet121\n",
      "skipping  densenet121\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/torchvision/models/densenet.py:212: UserWarning: nn.init.kaiming_normal is now deprecated in favor of nn.init.kaiming_normal_.\n",
      "  nn.init.kaiming_normal(m.weight.data)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "densenet169\n",
      "skipping  densenet169\n",
      "densenet201\n",
      "skipping  densenet201\n",
      "densenet161\n",
      "skipping  densenet161\n",
      "resnet18\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "08fd5ef8072b45258980ebd8de78b986",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "resnet34\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "633b1fc48c6c4a44841a0ad01a60bef4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "resnet50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "099f7dffb3ca45fabd4d07da9b06d93b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "resnet101\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d721b94202ed4f5c9390d612c0e56817",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "resnet152\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a5a859c001b54bee9e68f382b2b7f948",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inceptionv3\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "94048e515d8a4420949eb131b915d973",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "squeezenet1_0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/torchvision/models/squeezenet.py:94: UserWarning: nn.init.kaiming_uniform is now deprecated in favor of nn.init.kaiming_uniform_.\n",
      "  init.kaiming_uniform(m.weight.data)\n",
      "/Users/charleshmartin/anaconda3/lib/python3.6/site-packages/torchvision/models/squeezenet.py:92: UserWarning: nn.init.normal is now deprecated in favor of nn.init.normal_.\n",
      "  init.normal(m.weight.data, mean=0.0, std=0.01)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4463d537769749c3a0c2f13ab47bc7be",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "squeezenet1_1\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6d15f57e98d34d9186874a853435908a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vgg11\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1560aaebf45e41b79ac779518c9a7be1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vgg11_bn\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3f82191f8bbd4deeaa89f75479010761",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vgg13\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "de20a255344d4285874a2d2c2eb405f2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vgg13_bn\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7c78fa01a385430cbf2a3b0f1292d79d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vgg16\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "32bb527b5d304b7d884c050a30792828",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vgg16_bn\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "04576639a1d3410b9eab4ef910f4005c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vgg19_bn\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "71c2d154aedc418595e32c061886f72c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vgg19\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5839de1fa96c4073bb4ae880e3e83ef5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nasnetamobile\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "caba8e77483d485ba21d5ae9f8d53eb6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nasnetalarge\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1611ec902d8640a6955e564dab0443bc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dpn68\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d72f23699eaf4583bcba875364d001b8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dpn68b\n",
      "skipping  dpn68b\n",
      "dpn92\n",
      "skipping  dpn92\n",
      "dpn98\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b65c9cfb7a3c4fe184350c827d228721",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dpn131\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "310d3bb1431c460f90d6c18dd7394747",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dpn107\n",
      "skipping  dpn107\n",
      "xception\n",
      "skipping  xception\n",
      "senet154\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "27337adbf84a40cfbe86cabe392e7319",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "se_resnet50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "468f1fc0a0c54abf9f811e89f518c094",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "se_resnet101\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0b6a069897834d2b87e89ad415bf0223",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "se_resnet152\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "447b22f335fb45bf8d599e8cda74431c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "se_resnext50_32x4d\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a53d7b1b25bf438a9dec69d4c527a46a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "se_resnext101_32x4d\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7e590f269426474fb4d87ad2621ba48b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cafferesnet101\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c18a7d353dd640e9a12aca8d981309a2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pnasnet5large\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "83b13d38d85848e19f90cf565fc0565f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "polynet\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1ce9953b89234d868f6193534a84c7f3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "Qs = []\n",
    "for model_name in tqdm(pretrainedmodels.model_names):\n",
    "    try:\n",
    "        print(model_name)\n",
    "\n",
    "        model = pretrainedmodels.__dict__[model_name](num_classes=1000, pretrained='imagenet')\n",
    "        model.eval()\n",
    "        \n",
    "        for W in tqdm(iter_pytotch_layers(model)):\n",
    "            M, N = np.min(W.shape), np.max(W.shape)\n",
    "            Q = N/M\n",
    "            Qs.append(Q)\n",
    "    except:\n",
    "        print(\"skipping \", model_name)\n",
    "    pass\n",
    "\n",
    "Qs = np.array(Qs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Rank Loss, Q>1.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "ids = np.where(Qs[Qs>1.1])\n",
    "MSV = np.array(min_svs)[ids]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUIAAAFiCAYAAACDAdGqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xm8nGV5//HP10ACsgXIYcvCCRqQLQgNiG2tCKgsSqiisiippE3FpW4IwQUslRbUFmtRNAolIBJWBQSrSIuIPwHDvgkECGRBEpCwyc71++O+hzyZzJwz58zMmTnn+b5fr/M68+zXbNfc97NcjyICM7Mye12nAzAz6zQnQjMrPSdCMys9J0IzKz0nwmFE0vckfaXV83YTST+XNGMItvNVST9q93YaJSkkvbHTcZTVGp0OwEDSQmALYIuIeKww/hZgJ2ByRCyMiI81us6BzDvUJH0R+AegB1gB/DYiPgQQEft2MrbBkPR9YO2IOLxq/FTg98DmEfGnjgRnDXGLsHs8CBxSGZC0I7B258Jpj9za+wiwd0SsC0wDrupsVAMjaVTVqDOB90lap2r84cDPnAS7nxNh9zib9MWpmAGcVZxB0pmSvpYf7yFpsaTPS1om6RFJH+1n3qML8x4oaT9J90r6U26lrbZscfnC8EJJX5B0m6RnJZ0uadPcrX1a0q8kbVjnee4K/CIi7geIiD9GxJzCuq+W9Pf58d9JulbSNyU9IelBSfsW5p0s6ZrCNr9T6e5Wx1yIe+9aQUm6QNIfJT2Z17l91etxmqQrJD0LvKO4bET8DlgCvL+wzCjgUGBuHt5N0u8krciv/6mSRteJ5bXXoPg6FIbfJOnK/L7dI+mDhWn7SborvyZLJB1Vaxu2KifC7nEdsL6kbfOX6ENAf/uwNgM2AMYDM4Hv9JGANgPWyvMeB/wA+DDwF8DbgOMkbTWAeN8PvBPYGngv8HPgi8A40ufqn+osdx1weE6k02q0rqq9Bbgnr/frwOmSlKf9GLgB2Bj4KqmlOVg/B6YAmwA3AedUTT8UOBFYD7iW1Z3Fqj9kewNr5vUCvAJ8Nj+PtwJ7AR8faJC51Xkl6blvQupFfLeQuE8H/jEi1gN2AP53oNsoIyfC7lJpFb4T+AOpldGXl4ATIuKliLgCeAbYpo95T4yIl4B5pC/kf0bE0xFxJ3AnMHUAsf5XRDwaEUuA3wDXR8TNEfEC8BNg51oLRcSPgE8B7wZ+DSyTNLuP7TwUET+IiFdIravNgU0lTSK1Lo+LiBcj4lrg0gHEXx3XGfm1eIGUVHeStEFhlksi4rcR8WpEPF9jFWcDb5c0IQ8fDvw4v95ExI0RcV1EvBwRC4HvA28fRKjvARZGxH/ndd0EXAQclKe/BGwnaf2IeCJPt344EXaXs0ktj7+jqltcx+MR8XJh+M/Aun3M+0p+/Fz+/2hh+nN9LFtL9bINrysizomIvYGxwMeAEyS9u87sfyws9+f8cF3SwaU/FcYBLGo8/JUkjZJ0kqT7JT0FLMyTxjW67oh4GLgG+LCkdYEDyd3ivI2tJf0sd7+fAv61av2N2hJ4S+5ir5C0AjiM1OKH1FLfD3hI0q8lvXUQ2ygdJ8IuEhEPkQ6a7Adc3MFQngVeXxjerN6Mzcgt2QuA20jduIF4BNhIUjHOiYXHqzyH3AXvqbOuQ4HppO7sBkBvZbFiuA3ENJfUEnw/8GBVa+w0Uit/SkSsT9qNoNVXsXrsrPr6LwJ+HRFjC3/rRsSRABHx+4iYTuo2/xQ4v4G4S8+JsPvMBPaMiGc7GMMtwH6SNpK0GfCZVq047/jfX9J6kl6XD35sD1w/kPXkH435wFcljc4tn/cWZrkXWCtva03gy8CYOqtbD3gBeJyUgP51YM/qNReRkvE/U2gNFrbxFPCMpDcBR/axnltIR6Ffr3Ru4czCtJ8BW0v6iKQ189+ued/yaEmHSdogd8mfIu2btH44EXaZiLg/IuZ3OIyzgVtJXcRfAue1cN1PkVpDD5POIfw6cGTexzdQh5EOPDwOfI0U5wsAEfEk6WDED0n7Wp8FFtdeDWcBD+X57iId0Bmw/ONVSYbVB1uOIrU8nyYdqOrrNT0FeJG0u2FucV0R8TTwLuBgYClp18HJrEzyHwEW5u73x0gHxKwfchkuGykknQf8ISKO73QsNry4RWjDVu4SviF3sfch7ef7aafjsuHHl9jZcLYZ6aDSxqRu75ERcXNnQ7LhyF1jMys9d43NrPScCM2s9JwIzaz0nAjNrPRKkQgl/Zukz1SNWyTpzZ2Oo43buqGqlNQqw3WW2UbSzbmEU73qMWY1SbpT0h6djmNQImJE/5GuL11CqiBcGbchqUrHmE7G0ebtfRC4qN5wnWVOB05pYQzPVP29QqpaU5l+NfB8Yfo9hWkbkarYPEu66uPQqnX3Ob1GLAtJxWA7/pmsiulRYJ3CuL8Hrh7gOuo+rzz9RWBc1fhbSNdP97ZiO8P9rwwtwr8DroiI5wrjdgTujVRyqZNxtNOlwDskbV5nuJYtSeW4BkRSzfNRIxUDWDdSJepNSVVpLqia7ZOF+YolxL5D+gJvSrqU7rSqFm1/04eLNYBPt3kbba1+Xu/9H05GRCLMF5r/P0nn5TJHi7SykvG+pLp3RVOB2/OyknSMpIdyWaPzi3Xo8lULx+V1LpX0Xkkv9lEAtZ7V4lCh4rKksZIeyyWcmhapZt6NpOtSVxuuJul/SZWXT5X0TC4bta1SteQVudtzQGH+hfl1uw14toEvw0HAMlLtwj7l4qPvB74SEc/EylqDH2lkegPrH1CFbUmzc4mup5WqP/9tYdouhd0JF+TPYLG69xaSLpK0PL/f1bscvgEcJWlsnVjrLi/pbGAScFl+z46u85QbqX7e13NcbTu13n9VVQCXNFHSxTn2xyWd2shrkte7JMdyj6S96jyv1ul0k7QVf8BJpNbGB0lVgY8iFfQEWA7sWjX/HOCL+fHXSF20zUkVnC8B/r0w7wmkOnPjSfXzfgcsGkSMteL4LfD3heHf0MLuB/Bt4D/qDdeY/+pKPPl1XEAqkDAa2JNUMGCbPH0hqXs1kQa6+6RKyV+tsb3lwGP5tdgjj98ZeK5q3qOAyxqZXmf7CyuvbX58Hak1OZ6UoG/K6x2TYz2+sOwHSPUPX0eqHP5s/ryMJnXLP51fr/eRWqlfy8u9jvTjc1yedyvgAeDdxZhIV8dUlnmta9zf8tXPq6/nTaryvS0wilTKa0sKXeN6z7Hedmq9/1Wv8ShS4Y5TgHVI362/buA12SbHt0Ue7gXe0O4cMiJahKSu7ikRcX6k8kNnAZMkrUVKXk/XmP82SZuSqiUfGhGPRGo1XUi6oRCSekjl1Y+IiCURsYJUjeV2SRvkAxDPSHqtlp6kEyX9RtKFWrVWXq04lpK+iBVrUb9G3SokfVzS1v3M9nTebr3hvuxOKoB6UqQK0P9LKgF1SGGeb0fEouinu69UTfrtrF6a6hjSF2E86cfpMklvyNt9smreJ0mlrGhgeiMarrAdERdExNJI1anPA+4DdiO9RmuQXoeXIuJi0q0DKnYFeiLihPwaPkCqPHNwVSzHAZ/KnzcGsXwj+qx+3sdz7Etf7/9upMT6hYh4NiKej9Ry7+85vUL6MdpO0pqR7t54/yCe74AM+759tiNQvIfvJsAzEfG8pCcofEEkiVQE9HbSvTpuj4ilhWXHkYp+QrqvxIKIWFCYvlFe9s/A/qSuTWXdO5B+vd4m6WPAEcCpefIqcWRLyIlQ0jRSUui36wgQEd9tYLb1SKWu6g33ZQtSy/fVwriHWDVxN1oR+nDg2oh4sDgyIoo1COdKOoRUlPZaYP2qdazPyh+SZ/qZ3oiGK2xLOhz4HCsLtq5L+pysDSyJ3HTJiq/JlsAWSlWkK0ZR9R5HxB2SfgbMBu4e6PINOpvUs5lMjernfTzHvvT1/k8k9cperhrf53OKiAVKZ1Z8Fdhe0i+Az1V9R1tu2LcI876ViaQuVsVBrLxpzm2kGwxVTAZejVTYs4fVWxbTWXlznnGkVltlW6NI+/puyy2A5VXLvq2w3Z+TugIV1XGQ1z1B0nqkD+oncqsUSTdJ+paku5WKcH5f6Y5z++fpv8n/b5D0H5JuqbH/aVtS96TecF+WAhMlFT8jk1i1JdHoheqHs3prsJYgtYjvBdaQNKUwbSdWHsjpb3rLSNqS1GL5JLBxRIwF7shxPgKMzz+uFcUq2YtIlaqL1aTXi4j9amzqeNK9nqt/aPpbvqH3IPqoft7Pc+xrO31texGpV1bd2Or3OUXEjyPir1nZfT+5kefYjGGfCEmtwVeAQ/MO2/1JBTm/mqdfwao3yXntQAnp5ttvVSrltK6kE0j7jc7I0+8G/lLSGyWtT9rH9obC8tU2ZGVifZLUeqyojgNSstmK1BX7UUTMA5A0jlQy/iukI6LHkn6tZwDvytOX5f8bk75E7yC1UMnrGEO6Q92VtYYbcD1pP9HRSlWQ9yBVgJ7X4PKVOP6S9OW+oGr8WEnvlrRWft8OA/6GdKvPZ0lf1hMkrSPpr0g/UGfDawVQ605vsXVIX8blOe6PsvK2Ar8jffY+mZ/DdFbtTt4APJV3/q+tdG+UHSTtWr2R3Os4j1Xv/tfI8o+SPkONqFf9vK/nOJjtVGJ/BDgpv0dr5fepz+ekdC7rnvnz+jypdd72KtsjJRGeQ6pU/ASpTPqBEXFXnn4Wqez82oX5bwOIVAn6RFILcDGpxbRn5BsCRcRVwLnAzaSy8LeQusR/qBPLE6QERv5fvLF3dRyQWlfbkO5BcWJh/FTg3EjViMcB5+cP7zhSZeep+TlMBebl+XpIO50rDiDtdF9aZ7hPEfFiXmZf0sGM7wKHR0S9517PDODiHGPRmqQDVZWDJZ8ivW/35OkfJ3U9l5HegyMj3W2PBqe3RP4c/Tsp6T1K+vz8Nk97kXSAZCZpl8OHSftRK1WyXyH9eLyZ1Bp7jFQxewNqO4GUlCrbbmT5fwO+rHRkv897GEed6ud9PcfBbKcq9jeSPrOLgQ818JzGkA5+Pkaqvr0J6YBdWw37MlySTiOdE3hKH/P8K7AsIr7V5LY+BuwfEe8tjDsT+Gbez7MjcGxEHCppFumE7f8aaBx5H8miiLhI0hdIVZcvk/RlUktte9L+ui0L830I2CgiTsvruB6YGRF31Bq29siv8/ci4r87HYs1biQcLNmRdMpLXRExqF8USbuTmveLSAdOTiC1kirTryD9sm0j6fsRcabS+Yi/IbVUiuduDSSOHYHL8uOdSC2eyvjKEbbLSPe4rcy3c+ExEfGWqm2vMmytIentpFNTHiPtxpgK/E9Hg7IBGwktwhXAmyPdNLvV6z6S1H1bk7SD/rhIN1I3AyC3/P+FdJT1flKP4PLORmUDNewToZlZs0bCwRIzs6Y4EZpZ6TkRmlnpdcVR43HjxkVvb2+nwzCzEebGG298LCKqr+FeTVckwt7eXubPX+08TzOzpkh6qJH53DU2s9JzIjSz0nMiNLPScyI0s9JzIjSz0nMiNLPScyI0s9JzIjSz0nMiNLPScyI0s9JzIjSz0nMiNLOG9M6+nN7ZI7P4thOhmZWeE6GZlZ4ToZmVXr+JUNIZkpZJuqNq/Kck3SPpTklfL4w/VtKCPO3d7QjazKyVGinMeiZwKnBWZYSkdwDTgakR8YKkTfL47Uj33N0e2AL4laSt893tzcy6Ur8twoi4BvhT1egjgZMi4oU8z7I8fjowLyJeiIgHgQXAbi2M18ys5Qa7j3Br4G2Srpf0a0m75vHjgUWF+RbncWZmXWuw9yxZA9gQ2B3YFThf0laAasxb8w7ykmYBswAmTZo0yDDMzJo32BbhYuDiSG4AXgXG5fETC/NNAJbWWkFEzImIaRExraen35tMmZm1zWAT4U+BPQEkbQ2MBh4DLgUOljRG0mRgCnBDKwI1M2uXfrvGks4F9gDGSVoMHA+cAZyRT6l5EZgREQHcKel84C7gZeATPmJsZt2u30QYEYfUmfThOvOfCJzYTFBmZkPJV5aYWek5EZpZ6TkRmlnpORGaWek5EZpZ6TkRmlnpORGaWek5EZpZ6TkRmlnpORGaWek5EZpZ6TkRmlnpORGaWek5EZpZ6Q22VL+ZtVnv7Mtfe7zwpP07GMnI5xah2TDQO/vyVRKjtZYToZmVnhOhmZWeE6GZlZ4ToZmVno8am3UZHxQZem4Rmlnp9ZsIJZ0haVm+h3H1tKMkhaRxeViSvi1pgaTbJO3SjqDNzFqpkRbhmcA+1SMlTQTeCTxcGL0vMCX/zQJOaz5EM7P26jcRRsQ1wJ9qTDoFOBqIwrjpwFmRXAeMlbR5SyI1M2uTQe0jlHQAsCQibq2aNB5YVBhenMeZmXWtAR81lvR64EvAu2pNrjEuaoxD0ixS95lJkyYNNAyzEcVHijtrMC3CNwCTgVslLQQmADdJ2ozUApxYmHcCsLTWSiJiTkRMi4hpPT09gwjDzKw1BpwII+L2iNgkInojopeU/HaJiD8ClwKH56PHuwNPRsQjrQ3ZzKy1Gjl95lzgd8A2khZLmtnH7FcADwALgB8AH29JlGZmbdTvPsKIOKSf6b2FxwF8ovmwzMyGjq8sMbPScyI0s9JzIjSz0nMiNLPScyI0s9JzIjSz0nMiNLPScyI0s9JzIjSz0nMiNLPScyI0s9JzIjSz0nMiNLPScyI0s9JzIjSz0nMiNLPScyI0s9JzIjSz0nMiNLPScyI0s9JzIjSz0nMiNLPScyI0s9Lr977Gks4A3gMsi4gd8rhvAO8FXgTuBz4aESvytGOBmcArwD9FxC/aFLvZsNY7+/JOh2BZIy3CM4F9qsZdCewQEVOBe4FjASRtBxwMbJ+X+a6kUS2L1sysDfptEUbENZJ6q8b9sjB4HXBQfjwdmBcRLwAPSloA7Ab8riXRmtmQKkurtRX7CI8Afp4fjwcWFaYtzuPMzLpWvy3Cvkj6EvAycE5lVI3Zos6ys4BZAJMmTWomDLPSKLbQFp60fwcjGVkG3SKUNIN0EOWwiKgku8XAxMJsE4CltZaPiDkRMS0ipvX09Aw2DDOzpg0qEUraBzgGOCAi/lyYdClwsKQxkiYDU4Abmg/TzKx9Gjl95lxgD2CcpMXA8aSjxGOAKyUBXBcRH4uIOyWdD9xF6jJ/IiJeaVfwZmat0MhR40NqjD69j/lPBE5sJigzs6HkK0vMrPScCM2s9JwIzaz0nAjNrPScCM2s9JwIzaz0nAjNrPScCM2s9JwIzaz0nAjNrPScCM2s9JwIzaz0nAjNrPScCM2s9JwIzaz0nAjNrPScCM2s9JwIzaz0nAjNrPScCM2s9Jq6wbuZWT3D6Wb0bhGaWen1mwglnSFpmaQ7CuM2knSlpPvy/w3zeEn6tqQFkm6TtEs7gzcza4VGWoRnAvtUjZsNXBURU4Cr8jDAvsCU/DcLOK01YZqZtU+/iTAirgH+VDV6OjA3P54LHFgYf1Yk1wFjJW3eqmDNzNphsPsIN42IRwDy/03y+PHAosJ8i/M4M7Ou1eqDJaoxLmrOKM2SNF/S/OXLl7c4DDOzxg02ET5a6fLm/8vy+MXAxMJ8E4CltVYQEXMiYlpETOvp6RlkGGZmzRtsIrwUmJEfzwAuKYw/PB893h14stKFNjPrVv2eUC3pXGAPYJykxcDxwEnA+ZJmAg8DH8izXwHsBywA/gx8tA0xm5m1VL+JMCIOqTNprxrzBvCJZoMyMxtKvrLEzErP1xqbWdOG03XFtbhFaGal50RoZm3XO/vyVVqN3caJ0MxKz4nQzErPB0vMhlg3dxHLyi1CMys9J0IzKz0nQjMrPSdCMys9J0IzKz0nQjMrPSdCMys9J0IzKz0nQjMrPSdCMys9J0IzKz1fa2w2BNpxfXFlncOxEGq3cYvQzErPidDMSs+J0MxKr6lEKOmzku6UdIekcyWtJWmypOsl3SfpPEmjWxWsmVk7DDoRShoP/BMwLSJ2AEYBBwMnA6dExBTgCWBmKwI1M2uXZrvGawBrS1oDeD3wCLAncGGePhc4sMltmJm11aATYUQsAb4JPExKgE8CNwIrIuLlPNtiYHyzQZqZtVMzXeMNgenAZGALYB1g3xqzRp3lZ0maL2n+8uXLBxuGmVnTmuka7w08GBHLI+Il4GLgL4GxuasMMAFYWmvhiJgTEdMiYlpPT08TYZiZNaeZK0seBnaX9HrgOWAvYD7wf8BBwDxgBnBJs0Ga2dAp4132mtlHeD3poMhNwO15XXOAY4DPSVoAbAyc3oI4zczapqlrjSPieOD4qtEPALs1s14zs6HkK0vMrPScCM2s9JwIzaz0nAjNrPScCM2s9JwIzaz0nAjNrPScCM2s9JwIzaz0nAjNrPScCM2s9JwIzaz0nAjNrPScCM2s9JwIzaz0nAjNrPScCM2s9JqqUG1mnVe8x8jCk/bv2LaHM7cIzaz0nAjNrPTcNTZro5HSdRzp3CI0s9JzIjSz0msqEUoaK+lCSX+QdLekt0raSNKVku7L/zdsVbBmZu3QbIvwP4H/iYg3ATsBdwOzgasiYgpwVR42M+tag06EktYH/gY4HSAiXoyIFcB0YG6ebS5wYLNBmpm1UzMtwq2A5cB/S7pZ0g8lrQNsGhGPAOT/m7QgTjOztmkmEa4B7AKcFhE7A88ygG6wpFmS5kuav3z58ibCMDNrTjOJcDGwOCKuz8MXkhLjo5I2B8j/l9VaOCLmRMS0iJjW09PTRBhmZs0ZdCKMiD8CiyRtk0ftBdwFXArMyONmAJc0FaGZWZs1e2XJp4BzJI0GHgA+Skqu50uaCTwMfKDJbZiZtVVTiTAibgGm1Zi0VzPrNbPhazheVugrS8ys9JwIzaz0nAjNrPScCM1GkN7Zlw/LfXSd5kRoZqXnRGhmpecK1WY2IM10vWstO9Q3nKrFLUIzKz0nQjMrPSdCMys9J0IzKz0nQjMrPSdCMys9nz5TYsVTGbrhFAazTnGL0MxKz4nQzErPidDMSs/7CEvI1Unao/K6en/r8OMWoZmVnhOhmZWeu8ZmBpR7l4lbhGZWek0nQkmjJN0s6Wd5eLKk6yXdJ+m8fM9jM7Ou1Yqu8aeBu4H18/DJwCkRMU/S94CZwGkt2I7ZsFDmLuZw1VSLUNIEYH/gh3lYwJ7AhXmWucCBzWzDzFqjcmMnJ+rVNds1/hZwNPBqHt4YWBERL+fhxcD4JrdhZtZWg+4aS3oPsCwibpS0R2V0jVmjzvKzgFkAkyZNGmwYZlaDC2oMTDMtwr8CDpC0EJhH6hJ/CxgrqZJgJwBLay0cEXMiYlpETOvp6WkiDDOz5gy6RRgRxwLHAuQW4VERcZikC4CDSMlxBnBJC+I060re3zYytOOE6mOAeZK+BtwMnN6GbdgA+QtrVl9LEmFEXA1cnR8/AOzWivWamQ0FX1liZl2jU6f3+FrjEczdYQN/DhrhFqGZlZ5bhGbWUd3QYnWL0MxKz4nQzErPXeNhzvfJGDq+bG3kcovQzErPidDMSs9d4xGiG468tYq7oDbU3CI0s9Jzi9BsEEZSC9zcIjQzcyI0M3MiNLPS8z7CYagd+6d8YnZ93h848rlFaGal50RoZqXnRGhmped9hLaKob6qw1eRWDdwi9DMSs+J0MxKz4nQzEpv0PsIJU0EzgI2A14F5kTEf0raCDgP6AUWAh+MiCeaD9V8PptZezTTInwZ+HxEbAvsDnxC0nbAbOCqiJgCXJWHzcy61qBbhBHxCPBIfvy0pLuB8cB0YI8821zgauCYpqLsciP1yKevNrGyaMk+Qkm9wM7A9cCmOUlWkuUmdZaZJWm+pPnLly9vRRhmZoPSdCKUtC5wEfCZiHiq0eUiYk5ETIuIaT09Pc2GYWY2aE0lQklrkpLgORFxcR79qKTN8/TNgWXNhWhm1l6DToSSBJwO3B0R/1GYdCkwIz+eAVwy+PDMzNqvmUvs/gr4CHC7pFvyuC8CJwHnS5oJPAx8oLkQzYaOT1Eqp2aOGl8LqM7kvQa7XjOzoeYrS8ys9Fx9xoacu5+d5/dgVU6EHTJST8JuRqe+nE4K5q6xmZWeE6GZlZ67xi1Wq5vlrq/ZwAz1riO3CM2s9JwIzaz0nAjNrPScCG1Aemdf7tNNbMTxwRLr10hMfCPxOdnguUVoZqXnFmENnSpR76tNzDrDidBGFP+Y2GC4a2xmpecWoQ1Ko1fQuIVmw4ETYRMaPfI4mCOUPqppNnTcNTaz0nOLsA+1unVuqQ0fLoBhjXKL0MxKzy3CBrkl2Blu1dlQKFUi7OtE6U4mOifZwfFBqHIYijMP2tY1lrSPpHskLZA0u13bMTNrVltahJJGAd8B3gksBn4v6dKIuKtV22j0Mrj+WgBuIbROf7/crXqt/Z5Zq7Wra7wbsCAiHgCQNA+YDrQsEVa4e9Sd/BrbcNKurvF4YFFheHEeZ2bWddrVIlSNcbHKDNIsYFYefEbSPW2KpT/jgMc6tO3+dHNs4Pia1c3xdWVsOvm1h43Gt2Uj621XIlwMTCwMTwCWFmeIiDnAnDZtv2GS5kfEtE7HUUs3xwaOr1ndHF83xwatj69dXePfA1MkTZY0GjgYuLRN2zIza0pbWoQR8bKkTwK/AEYBZ0TEne3YlplZs9p2QnVEXAFc0a71t1DHu+d96ObYwPE1q5vj6+bYoMXxKSL6n8vMbARz0QUzK71SJkJJH5B0p6RXJU0rjH+npBsl3Z7/79lN8eVpx+bLFu+R9O5OxFcVz5slXSfpFknzJe3W6ZiqSfpUfr3ulPT1TsdTTdJRkkLSuE7HUiTpG5L+IOk2ST+RNLYLYmrPpbsRUbo/YFtgG+BqYFph/M7AFvnxDsCSLotvO+BWYAwwGbgfGNXh1/KXwL758X7A1Z1+f6viewfwK2BMHt6k0zFVxTeRdFDxIWBcp+Opiu1dwBr58cnAyR2OZ1T+zG8FjM7fhe1ase5Stggj4u6IWO0E7oi4OSIq5zveCawlaczQRlc/PtJlivMi4oWIeBBYQLqcsZMCWD8/3oCq80W7wJHASRHxAkBELOtwPNVOAY6m6oKDbhARv4yIl/PgdaTzgTvptUt3I+JFoHLpbtNKmQgb9H7g5soXqEt046WS3AYPAAAFHElEQVSLnwG+IWkR8E3g2A7HU21r4G2Srpf0a0m7djqgCkkHkHodt3Y6lgYcAfy8wzG07fM/YusRSvoVsFmNSV+KiEv6WXZ7UlfgXe2ILW9jMPH1e+liO/QVK7AX8NmIuEjSB4HTgb3bHdMA4lsD2BDYHdgVOF/SVpH7Wh2O7Yu08TPWiEY+h5K+BLwMnDOUsdXQts//iE2EETGoL6OkCcBPgMMj4v7WRrXSIOPr99LFdugrVklnAZ/OgxcAP2x3PNX6ie9I4OKc+G6Q9CrpOtXlnYxN0o6k/by3SoL0Xt4kabeI+ONQxNZXfBWSZgDvAfYaqh+PPrTt8++ucUE+KnY5cGxE/LbT8dRwKXCwpDGSJgNTgBs6HNNS4O358Z7AfR2MpZafkuJC0taknewdLyYQEbdHxCYR0RsRvaQv+S5DmQT7I2kf4BjggIj4c6fjoZ2X7nb6yFSHjj79LemD9wLwKPCLPP7LwLPALYW/IT/KWC++PO1LpCNn95CP1nb4tfxr4EbSEbzrgb/odExV8Y0GfgTcAdwE7NnpmOrEuZDuO2q8gLRPrvJd+F4XxLQfcG/+DnypVev1lSVmVnruGptZ6TkRmlnpORGaWek5EZpZ6TkRmlnpORGaWek5EZpZ6TkRWstIWjsXNhiVh/eWdPZQba+F6x0t6RpJa9QatpHHidBa6QjSdb2v5OGdSFecDNX2WiJSiaergA/VGraRx4nQBkzSPEnn5dJWD0naP086DChWztmJdGkWkt6UW1V3SvpVpRqzpG3z+NskfUHSggGE8tr2JI2V9Np1urnC+AZNPM2f5vXXG7YRxInQBmMn4IGIeAspORyfL4LfKiIWVs13ay5uexHw6YjYHrgS+Gzuap6Tx08lVR6+o5EAqrcXESuAdSStmWe5FZjaxHO8g1S2q96wjSBOhDYgktYmlbH65zzqLlK9v3HAisJ8awLrR8Ry4EDg2oi4ubDMJsD7gFurxt8qaStJp0u6MK9rHUlzJf1AUqVVtsr2skdZWVvvTXm4v+fzL7XG5+72i5LWqzVsI4sToQ3UDsB9EfF8Ht6F1Pp6DlirMN92wN2Fx7cXpu1ISnpTyV3nwrpviVSKfWZh/PuACyPiH4AD8rjq7UEqCbaFpA8Bj0XEvZI2ywdUjs7J9B8lXShpB0mbARMk/Z+kz0o6r2p9Y4Dn+xi2EcKJ0AZqJ2CSpLUkrUNqGZ4SEU8AoyStVZivkuSWkJIhkrYCPgKcBTxOKqWPpDcDH6b2wZUJrCzR/gpAje1BSoQHArNJB1Ig3ZDr4oj4OumeKj8gFZDdMk+7GvhpRJxCqsJMjmdjYHlEvFRr2EYWJ0IbqJ1I+/WuJhXKPC1WFrH9Jak+YWW+SlI7m9RSu510w50jIuLxPH6apN+TEtfCiHigxjYXs/LGQcXPbHF7kBLuQaRCopXiq28GfpG76o9HxKuklufteVov6S5ysGrZ93cAV/QxbCOIz4uygdoJ+IeIOKbGtFOBzwG/iojPV0ZGxHOkllq15/MBFyR9gXSLhErr60RgZ0nHAt8GTs1Hpy+rtb28nc8Dn2dVbyQV8pzKyq56b0Q8LOmNpPtg3JuPYherQx/Kqjeiqh62EcSFWW1AJC0BJuaWVa3pRwBzGzm3T9JXSOXWXwJ+C3wuBnjXwIFsbwDrHA0cHBFn1Rq2kceJ0MxKz/sIzaz0nAjNrPScCM2s9JwIzaz0nAjNrPScCM2s9JwIzaz0nAjNrPT+P5Gu/b3oQtR5AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(np.log10(MSV), bins=100);\n",
    "plt.title(\"Minimum Singular Values\\n\"+r'$p(log_{10}(\\nu_{min}))$ for 7500 ImageNet Matrices')\n",
    "plt.xlabel(r'$p(log_{10}(\\nu_{min}))$')\n",
    "plt.savefig(\"img/even-more-rank-loss.png\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.15912366676275583"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(np.log10(MSV)[np.log10(MSV)<-8])/len(MSV)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "widgets": {
   "state": {
    "52ecdae62c394a67868cd3ee83e19861": {
     "views": [
      {
       "cell_index": 14
      }
     ]
    },
    "e232317699e24554b5ed09fa6c78c662": {
     "views": [
      {
       "cell_index": 8
      }
     ]
    }
   },
   "version": "1.2.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}