All-pytorch-models.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Power Laws in Pytorch\n",
    "\n",
    "We compute the Empirical Spectral Density of the Fully Connected (FC) layers for all Pytorch pretrained models\n",
    "\n",
    "\n",
    "http://pytorch.org/docs/master/torchvision/models.html\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-01T02:55:17.341813Z",
     "start_time": "2018-09-01T02:55:15.428817Z"
    }
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision.models as models\n",
    "import torchvision.transforms as transforms\n",
    "import torch.nn as nn\n",
    "\n",
    "import numpy as np\n",
    "import powerlaw\n",
    "\n",
    "import sklearn\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "\n",
    "import matplotlib\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-01T02:55:19.471884Z",
     "start_time": "2018-09-01T02:55:17.343557Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "importing notebook from RMT_Util.ipynb\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/charlesmartin14/anaconda3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6\n",
      "  return f(*args, **kwds)\n",
      "/Users/charlesmartin14/.local/lib/python3.6/site-packages/h5py/__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.6.5 |Anaconda custom (64-bit)| (default, Apr 26 2018, 08:42:37) \n",
      "[GCC 4.2.1 Compatible Clang 4.0.1 (tags/RELEASE_401/final)]\n",
      "numpy version 1.14.2\n",
      "scipy version 1.0.1\n",
      "tensforflow version 1.5.0-dev20171116\n",
      "keras version 2.1.5\n",
      "sklearn version 0.19.1\n",
      "powerlaw version 1.4.3\n"
     ]
    }
   ],
   "source": [
    "import import_ipynb\n",
    "import RMT_Util"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-01T02:56:18.790173Z",
     "start_time": "2018-09-01T02:56:18.786159Z"
    }
   },
   "outputs": [],
   "source": [
    "plt.rcParams['figure.figsize'] = [5,5]\n",
    "matplotlib.rcParams.update({'font.size': 16})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-01T04:28:11.312838Z",
     "start_time": "2018-09-01T04:28:11.266897Z"
    }
   },
   "outputs": [],
   "source": [
    "def analyze_model(model, plot=True):\n",
    "    all_alphas = []\n",
    "    pretrained_model = model(pretrained=True)\n",
    "    \n",
    "    model_name = model.__name__\n",
    "    model_name = model_name.replace(\"_\",' ')\n",
    "    \n",
    "    for im, m in enumerate(pretrained_model.modules()):\n",
    "        if isinstance(m, nn.Linear):\n",
    "                \n",
    "            W = np.array(m.weight.data.clone().cpu())\n",
    "            M, N = np.min(W.shape), np.max(W.shape)\n",
    "            Q=N/M \n",
    "            \n",
    "            sv, _ = RMT_Util.singular_spectrum(W)\n",
    "            evals =(1/N)*sv*sv\n",
    "            \n",
    "            # RMT_Util.fit_power_law\n",
    "            fit = powerlaw.Fit(evals, xmax=np.max(evals), verbose=False)  \n",
    "            alpha = fit.alpha\n",
    "            D = fit.D\n",
    "            \n",
    "            all_alphas.append(alpha)\n",
    "            \n",
    "            dist = RMT_Util.best_dist(fit)\n",
    "\n",
    "            if plot:\n",
    "                RMT_Util.plot_ESD_and_fit(model=None,eigenvalues=evals, weightfile=None, layer=None, \n",
    "                                Q=Q, num_spikes=0, sigma=None, color='blue')\n",
    "                plt.show()\n",
    "            \n",
    "            line = \"{} & {}& {:.3g} & $({}\\\\times {})$ & {:.3g} & {:.3g} & {} \\\\\\\\ \".format(model_name,im, Q, np.min(W.shape),np.max(W.shape),alpha,D,dist)\n",
    "            print(line)\n",
    "            model_name = \" \"\n",
    "    print(\"\\\\hline\")\n",
    "    return all_alphas\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-01T04:36:17.918114Z",
     "start_time": "2018-09-01T04:28:12.412557Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/charlesmartin14/anaconda3/lib/python3.6/site-packages/powerlaw.py:697: RuntimeWarning: divide by zero encountered in true_divide\n",
      "  (Theoretical_CDF * (1 - Theoretical_CDF))\n",
      "/Users/charlesmartin14/anaconda3/lib/python3.6/site-packages/powerlaw.py:697: RuntimeWarning: invalid value encountered in true_divide\n",
      "  (Theoretical_CDF * (1 - Theoretical_CDF))\n",
      "Assuming nested distributions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "alexnet & 17& 2.25 & $(4096\\times 9216)$ & 2.29 & 0.0527 & PL \\\\ \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  & 20& 1 & $(4096\\times 4096)$ & 2.25 & 0.0372 & PL \\\\ \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  & 22& 4.1 & $(1000\\times 4096)$ & 3.02 & 0.0186 & PL \\\\ \n",
      "\\hline\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/charlesmartin14/anaconda3/lib/python3.6/site-packages/powerlaw.py:1602: RuntimeWarning: invalid value encountered in true_divide\n",
      "  CDF = CDF/norm\n",
      "/Users/charlesmartin14/anaconda3/lib/python3.6/site-packages/numpy/core/_methods.py:29: RuntimeWarning: invalid value encountered in reduce\n",
      "  return umr_minimum(a, axis, None, out, keepdims)\n",
      "'nan' in fit cumulative distribution values.\n",
      "Likely underflow or overflow error: the optimal fit for this distribution gives values that are so extreme that we lack the numerical precision to calculate them.\n",
      "/Users/charlesmartin14/anaconda3/lib/python3.6/site-packages/numpy/core/_methods.py:26: RuntimeWarning: invalid value encountered in reduce\n",
      "  return umr_maximum(a, axis, None, out, keepdims)\n"
     ]
    }
   ],
   "source": [
    "for model in [models.alexnet]:\n",
    "    analyze_model(model, plot=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate data for Table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-01T05:59:42.774681Z",
     "start_time": "2018-09-01T04:36:17.919948Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/charlesmartin14/anaconda3/lib/python3.6/site-packages/powerlaw.py:697: RuntimeWarning: divide by zero encountered in true_divide\n",
      "  (Theoretical_CDF * (1 - Theoretical_CDF))\n",
      "/Users/charlesmartin14/anaconda3/lib/python3.6/site-packages/powerlaw.py:697: RuntimeWarning: invalid value encountered in true_divide\n",
      "  (Theoretical_CDF * (1 - Theoretical_CDF))\n",
      "Assuming nested distributions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "alexnet & 17& 2.25 & $(4096\\times 9216)$ & 2.29 & 0.0527 & PL \\\\ \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  & 20& 1 & $(4096\\times 4096)$ & 2.25 & 0.0372 & PL \\\\ \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n",
      "/Users/charlesmartin14/anaconda3/lib/python3.6/site-packages/powerlaw.py:1602: RuntimeWarning: invalid value encountered in true_divide\n",
      "  CDF = CDF/norm\n",
      "/Users/charlesmartin14/anaconda3/lib/python3.6/site-packages/numpy/core/_methods.py:29: RuntimeWarning: invalid value encountered in reduce\n",
      "  return umr_minimum(a, axis, None, out, keepdims)\n",
      "'nan' in fit cumulative distribution values.\n",
      "Likely underflow or overflow error: the optimal fit for this distribution gives values that are so extreme that we lack the numerical precision to calculate them.\n",
      "/Users/charlesmartin14/anaconda3/lib/python3.6/site-packages/numpy/core/_methods.py:26: RuntimeWarning: invalid value encountered in reduce\n",
      "  return umr_maximum(a, axis, None, out, keepdims)\n",
      "/Users/charlesmartin14/anaconda3/lib/python3.6/site-packages/torchvision-0.2.1-py3.6.egg/torchvision/models/densenet.py:212: UserWarning: nn.init.kaiming_normal is now deprecated in favor of nn.init.kaiming_normal_.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  & 22& 4.1 & $(1000\\times 4096)$ & 3.02 & 0.0186 & PL \\\\ \n",
      "\\hline\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "densenet121 & 432& 1.02 & $(1000\\times 1024)$ & 3.32 & 0.0383 & PL \\\\ \n",
      "\\hline\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "densenet161 & 572& 2.21 & $(1000\\times 2208)$ & 3.45 & 0.0322 & PL \\\\ \n",
      "\\hline\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "densenet169 & 600& 1.66 & $(1000\\times 1664)$ & 3.38 & 0.0396 & PL \\\\ \n",
      "\\hline\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "densenet201 & 712& 1.92 & $(1000\\times 1920)$ & 3.41 & 0.0332 & PL \\\\ \n",
      "\\hline\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inception v3 & 226& 1.3 & $(768\\times 1000)$ & 5.26 & 0.0421 & PL \\\\ \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  & 302& 2.05 & $(1000\\times 2048)$ & 4.48 & 0.0275 & PL \\\\ \n",
      "\\hline\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "resnet101 & 286& 2.05 & $(1000\\times 2048)$ & 3.57 & 0.0278 & PL \\\\ \n",
      "\\hline\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "resnet152 & 422& 2.05 & $(1000\\times 2048)$ & 3.52 & 0.0298 & PL \\\\ \n",
      "\\hline\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "resnet18 & 67& 1.95 & $(512\\times 1000)$ & 3.34 & 0.0342 & PL \\\\ \n",
      "\\hline\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "resnet34 & 115& 1.95 & $(512\\times 1000)$ & 3.39 & 0.0257 & PL \\\\ \n",
      "\\hline\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n",
      "/Users/charlesmartin14/anaconda3/lib/python3.6/site-packages/torchvision-0.2.1-py3.6.egg/torchvision/models/squeezenet.py:94: UserWarning: nn.init.kaiming_uniform is now deprecated in favor of nn.init.kaiming_uniform_.\n",
      "/Users/charlesmartin14/anaconda3/lib/python3.6/site-packages/torchvision-0.2.1-py3.6.egg/torchvision/models/squeezenet.py:92: UserWarning: nn.init.normal is now deprecated in favor of nn.init.normal_.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "resnet50 & 150& 2.05 & $(1000\\times 2048)$ & 3.54 & 0.027 & PL \\\\ \n",
      "\\hline\n",
      "\\hline\n",
      "\\hline\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vgg11 & 24& 6.12 & $(4096\\times 25088)$ & 2.32 & 0.0327 & PL \\\\ \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  & 27& 1 & $(4096\\times 4096)$ & 2.17 & 0.0309 & TPL \\\\ \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  & 30& 4.1 & $(1000\\times 4096)$ & 2.83 & 0.0398 & PL \\\\ \n",
      "\\hline\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vgg11 bn & 32& 6.12 & $(4096\\times 25088)$ & 2.07 & 0.0311 & TPL \\\\ \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  & 35& 1 & $(4096\\times 4096)$ & 1.95 & 0.0336 & TPL \\\\ \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  & 38& 4.1 & $(1000\\times 4096)$ & 2.99 & 0.0339 & PL \\\\ \n",
      "\\hline\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vgg16 & 34& 6.12 & $(4096\\times 25088)$ & 2.3 & 0.0277 & PL \\\\ \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  & 37& 1 & $(4096\\times 4096)$ & 2.18 & 0.0321 & TPL \\\\ \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  & 40& 4.1 & $(1000\\times 4096)$ & 2.09 & 0.0403 & TPL \\\\ \n",
      "\\hline\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vgg16 bn & 47& 6.12 & $(4096\\times 25088)$ & 2.05 & 0.0285 & TPL \\\\ \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  & 50& 1 & $(4096\\times 4096)$ & 1.97 & 0.0363 & TPL \\\\ \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  & 53& 4.1 & $(1000\\times 4096)$ & 3.03 & 0.0358 & PL \\\\ \n",
      "\\hline\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vgg19 & 40& 6.12 & $(4096\\times 25088)$ & 2.27 & 0.0247 & PL \\\\ \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  & 43& 1 & $(4096\\times 4096)$ & 2.19 & 0.0313 & PL \\\\ \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  & 46& 4.1 & $(1000\\times 4096)$ & 2.07 & 0.0368 & TPL \\\\ \n",
      "\\hline\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vgg19 bn & 56& 6.12 & $(4096\\times 25088)$ & 2.04 & 0.0295 & TPL \\\\ \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  & 59& 1 & $(4096\\times 4096)$ & 1.98 & 0.0373 & TPL \\\\ \n",
      "  & 62& 4.1 & $(1000\\times 4096)$ & 3.03 & 0.035 & PL \\\\ \n",
      "\\hline\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assuming nested distributions\n"
     ]
    }
   ],
   "source": [
    "all_alphas = []\n",
    "for model in [models.alexnet, \n",
    "              models.densenet121, models.densenet161, models.densenet169, models.densenet201, \n",
    "              models.inception_v3,\n",
    "              models.resnet101, models.resnet152, models.resnet18, models.resnet34, models.resnet50, \n",
    "              models.squeezenet1_0, models.squeezenet1_1,\n",
    "              models.vgg11, models.vgg11_bn,\n",
    "              models.vgg16, models.vgg16_bn,\n",
    "              models.vgg19, models.vgg19_bn ]:\n",
    "    #print(model)\n",
    "    alphas = analyze_model(model, plot=False)\n",
    "    all_alphas.extend(alphas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-01T16:15:51.511344Z",
     "start_time": "2018-09-01T16:15:51.253486Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUYAAAF7CAYAAACuMyKDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xm8HFWd/vHPAyTIqkQCCBIDyCLq\nABIQ+CEEl1EHt1HQ0VEBRzPu26CCG9EBwX0cASUMM1EHEBRZRkRlSxBkkV1W2YKCoCyByE7g+/vj\nnCaVk+q+1fd2p/tenvfr1a++91R19amuqqdr63MUEZiZ2RIrDLoCZmbDxsFoZlZwMJqZFRyMZmYF\nB6OZWcHBaGZWcDCamRUcjGZmBQejmVnBwWhmVnAwTkCSZkoKSbMHXRez8WhCBqOk6TkYqo9HJS2Q\n9N+SNhl0Hc3Gyl+A/bPSoCvQZ9cDP85/rwnMBPYB/lHS9hFxw6AqZmbDa6IH43URMbv1jyQB/wPs\nBXwO2Hsw1TKzYTYhD6XbidTG2uH53xmtckmTJO0r6feSHpa0UNIvJb2s+npJUyQ9KenYovx1+ZDm\nQUmTK+UrSrpf0m/Kukh6i6R5efjDki6TNKsY56lDJUm7SDpL0iJJt4xm/iVNlvRRSadLul3SY5Lu\nkHSMpE37MZ8d6tJk/pXrGpJ2L4atnJfXY5Jeksuqn9duks7Ldb1L0lGS1m5Tl6bLvzr97SSdIekB\nSfdKOlrS1NHOa7fTz4fPZ+d/D6ieNiqm+VZJ50q6O7/3rZJOkrRz24WzbL2eKekrkm7K07hG0pvz\nsFmSFmuinZ6KiAn3AKYDAZxUM+yledhV+X8BJ+eyq4GvAUcCi4DFwJ7F668A7ijKvpFfH8DOlfLt\nctm/F+N/PZcvAOYA/5HfO4BvV8abmctOBx4Dfg58FTh8hPlvvW52Ub5enqezgO/naZ2Uy+4FNurl\nfHaoX6P5z+NuANwD/AVYp1L+nTz+Z2rm+1fAo8BPgK+QAqS1fFcrpt94+VemfyrwEHBK/kzOz+Xn\nAxrDvDaefh53bi6fB8xuPSrT+1AefiNwKHAI8EPgVuDzXWxLNwKPkI62vgPcDzwMbAjcBMwd9Dbf\n8wwZdAX6MlNtgjFvBK2V6X9y2V6VjWmlyrgvAB4E7gPWqJS3NsgtKmUXA7/JG+PnK+WfyuO+olL2\n6lbdgGdUyiflsgC2y2WtDSWAd3Qx/63XzS7KVwbWrxl/V1II/Fev5rND3RrPf2XYW3L5zyvTeDIH\nwgo18x3Au4tp/Gcu/3JR3nj5F9N/S2XcFYAzc/mOo53XUUy/djlXhl8K3AasWrMdTGmwrFbM03gC\n2LXmMzsurzfPH+T23o/HwCvQl5laEozXseSb9FvAJbn8XmDTPO5ZuezvaqbzH3nYuyplb85lH8j/\nr5lXjv2Bc4EzK+P+ghQiq1TKTskb9To17/eiPO1v5P9bK/7vupz/jhtMm9dcCSzo1Xx2eJ/G818M\n++887IvAn4GFwIZt5vtalt1zezZpL+zmorzx8q9Mf17NuK2w+Mho53UU0++4nEmhdjMweZTb0Z55\n+nOK8heyJMDnjnV7HcbHRL/4sjlwQP77cdIG9d/AgRFxSy7fGlgYEVfWvH4e8LE8zo9y2XzSCjET\n+B6wC+mb9WxgNeCT+fzbE8DOwEUR8XBlmi8lHaZ9UFL5fpPy8xZF+cUjz2ozkrYFPg38P2CdyntC\nOlxvGet8tjOa+Qf4aK7Dl/L//xQRf2rzHudF3oJbIuIeSdcB20haIyL+lgd1u/wBLqsZ9/b8/KxK\n2Wjnten0R3Ic6fD5KknHkZbp+RHxYMPXvz0/H1qUP5qfnwAO6qI+48ZED8aTI+JNI4yzJtDutp07\nK+MAT21gV5EOPyEFxwOk8FqddLX7paSVZw3SxlU1hfS5H0B7qxX//7XTDDSVT7ifSdqL+RXp3NGD\npADcG3hea9wezGc7o5l/IuIBSWcCmwB3ACd2eP1dbcr/kp/XBP5W+bvx8s/urxl3cX5esVI2qnnt\nYvoj+Rppz/oDwOfz4xFJPwY+GRELR3j9LsDtbb40AI6OCXrL20QPxiYWAeu2GbZuZZyq+cCHJW1J\nCoxzI2KxpN+S9rp2Y8m36rya93s4Ip7bRR1j5FEa2R+YDOwUEedXB0h6W834Y5nPdkYz/0h6JfA+\n0oWY5wAHkvZ869ReHaZ+eY5m+Tc1qnntlbzXPAeYI2ld0pfce0hfglOAN7Z7raRnk04/nFkz+LX5\n+Re9rO8weVrdrtPG5cBakl5UM2zXyjhV8/LzG0iHWWcDRMRDwO9IITKTFB6/LV57EbCBpA3HWO/R\n2AS4pyYU183DSvPy82jms52u5z9vpD8gBc12pHOc+0p6eZuX7KTi2DVPYwvglsphNIxu+TfV72X9\nRH4ecS8yIv4SEccD/0DaQ36tpE47Rqu2XlotlDSFJXvAT3ZX3fHDwZhuXwA4WNJTK5ikzYBZpMOa\nk4vXtM6/fZIl591a5gE70v6823fz81GSnllWRtJGkqaPZkYa+CMwRdILKu83mXQOaVLN+GOZz3ZG\nM/9HAusDH8znht9FOhT+gaS1at7jBXmcqgOAVYD/LcpHs/yb6veyvjc/b1A3UNLfV+cpW5V0KuQx\nOgfbnXmc7SWtn6e3AnAEaU8S2u9pj3s+lE4bxh7A64DLJJ0GrAW8jbQh/XNELHUoFRF3S7qGdHVu\nEenqX8s80vm31t9LiYhfSDqYdFh7o6RfkW6pmEraoHcA3kG6763XDgVeBZyXT8YvBl5JCsUrgK2K\nuo56Ptvpdv4lvRf4R+DYiDgmT2OBpA+RLojMIV09rfo1cKSk15P2jnYk7dleQ7p3s6rr5d+veR2F\n60nnW/9J0oPkCzQRcUgefjzwgKRzSfcurgrsTj4VERFtgzEiHle6wX8v4FxJpwDbkz7L/YCDgU9J\nemZETLwLMIO+LN6PBx1u8G4z/iTgM6Qbbx8h3bv2Kyr3btW85lAq99ZVylclnXcLOtzXRzpPcypw\nN+mb+XbSHtq/AWvncWbS5W03I70OeCvpqudDpL2CuaRv/nnk01K9nM8xzv/zSRd8/gg8q2Yax+T3\n36ecb+DlwHmki0t3k+5GmDqW5T/C59pp2IjzOtrpAzuR7i19IA+PyrAPAP9HCsVHSBfxzgHe1nAZ\nrUG6I+Evud43A+/Mw/Yj7U1f1q/teJCP1l30ZuOepJmkw/0vReU38mbd8jlGM7OCg9HMrOBgNDMr\n+ByjmVnBe4xmZgUHo5lZwcFoZlZwMJqZFRoHo1IfG61+Hx7P/Uts3c/KPV3lPj8i37A8cMNWn9HI\n9Z836GnUTHNunu70Xk736awXy6mbPcZPkX7zeSvp96ZfYkl7dROeUmdGvoRvlk3kUO+mEYnXkn6P\n+eqIeLxP9bHkUFJ/2H8cdEWs7/YntbJ9+0gj2vLTTTA+h9SWn0OxzyLiblKDAzbBRcQdpBZybJg0\naGFjNks6vqk+5lXGWZ3UovIfSC2u3AWcALy4Znrz8utXIX1TLiA1f7X3CPWYm1+3CamJ9ptJLYZc\nS+6wqU29ZwLvJXX29AiVzntIveZ9itTk1kOk1kLOYNlWVerm/6mWTli6VZddSB0sLSI1igqp1eyP\nkrpBvZ3UUskdpNZhNu1U97rWVUiNtZ5B2oO/Fzia9i3H7EZqafmePP/XkFpGWalm3KnAUXn5PUhq\nEHbXuvp0WE7T87hzgS3zey8iBf2R5O5LSQ2mXpA/99uB/dtMbyqpXcNbK5/bXGB6m/H3JLUe9Eie\n7jdJ61q7TqbWI/UgeDNp3f0Lqc3GjWrGXWYapLYQDyN1E/Fwns9Lga83acWFJev19ErZ3izpbuLV\npG5THyK1jnM4Ra9/DZfF1qTm2BaRWg/6afGeq+Vhl7eZ1hp5fbsy/7+A+m1ibvG6fyE1avxgnv45\nwBtHub0+g7S9Xprrsoi07R4ITCqXE6nVqB/mZfJQLntJo8+uwYc7M1f6vvyYnR97Vyp7Ua7MBaR2\n2n6UV7IHqfQ/XATjafnDPYLUG9trG65APyed2/xP4Nuk9u2C1KJK3Qd9GqlR06NJ50Y/Uan3OXmc\ni3Idjswr32LgzcW0WivC7MpjZhFatf0/00V/zg2CsZs+jT9Maoz0r6Rmt74JXJjH/1kx7uqk0Gyt\nVF8hbTyPkprg6jYY55P6Gzkt17O1jhxHav/wAVIA/Qcp9ALYqyYUb8rDfk1at07I83QXsFkx/ntY\n0gvk4Xl+b8nLoy7UNiWF5xN5nK+TTmE8lqe/SadgJIXJgjz+z0hf9Ifm9eDhHgTjiaRwOC7Py+9z\n+Y+7DMZzSF/6v8yf4Sm5/M/ABpXxv5/Lt62Z1nvzsI/m/z9Oatk88jKcnR9vqrzmu3n4AlIvnYeS\nvniC1OdMN9vrqixZ168mbfvfIn3xPkalWbo8zuWkL6sL83gn5PKFwHpjDsbKmy2g0r1mpfyA/IZH\nUdkwSXsaT5IaCq32/Tsvj/874JldvH9rBfozsG6lfApp5V9MZe+r8kHfT6Vv5Mrwg/Pw/Wo2xgWk\nDWOVst5t6jaTJd+Yy/T/TBf9ORd1rwvGoFmfwy8k9Yx4QfVzJvUp3GpjcY9K+Zdz2X+2CZtugzFI\nLW63ylci7cm1gnqbyrANSAFwZTGt/8nT+UJR/u5cflal7JmkPYj7qXzRkPZ0Wh3cl8F4Pin4dynK\nd8yfXdkGZRmMb6ASFsW4z+5yva4LxseAHSrlzyAdIT1JJdAaLosvFsM+mct/WCnbNpcdXjOt8/My\nmtKp7sW6HaQ9utUr5euRvoweBzbuYnv9Vh7+PZbdAViXpfsEb83zd1g6k1pZVXt0stQ0myy8PNEF\n1AfjzaRDiHVrhrU6FX9ZpWxeLtu96XsXC2G/mmEfLBd+5YNe5pCGFCYLgavavNeH82tfV9a7zfgz\n8/hd9f+cX7tUf85F3WfWvMe8mmnslYdV+xxudTA/o2b8NUkb10+L5fgQxQZNCtLryvp0mJ/pedwb\na1bgz+dhR9W87gzSl8RK+f/Jeb26A1i5ZvzL8rQ2zP+3wvJrNeO+o/zsgJfkskPbzMdPSXuS1S+V\ndsH4vm6Xe816Pb1Stjc1h6V5WGvjfn0Xy+IeisNvUuO8t+XPeHKl/NK8bVR3Cl5AzZ5qXd0rw1r9\ngL+hZtjHKb7w6Ly9rkT60rsLWKPBfAfpiGS1ovx5edgJI01jTF0bSFoT2IjUiu9fakaZR+qJbGtS\nK8NVo+0r+dwOZVvVDKt7n81J/fPeKml2zfBN8/MWpEOsptrOUxf9OY+kmz6NA3iDpNfVvOZhcp/G\nktYgLcdLI+Ke6kgREblXwM27qCOkvb8oylq3d11RM/6dpH5l1iXNzxakPaTzI+LRmvHnk9arrYE/\nsWTZl+sZ1K8zL83Pz22zDjyH9AW6Ke2X6/xc78MkvYp0qHpuRPyhzfjd6lX/0pdF6sDsKZG6LriI\n1G3E5qTDdEinkw4H3kw6nIV0nhDSUWFTrXuc59UMm1eMU1X3WW9B7qI3lu7IrJMbYtn+sxt/dmPt\n86XV325dKEL7fnlh9H0l1/UZXO0vuMn7TMnPW1Efpi11ff52UjtP3fTn3EA3fRoL+EKHabXmr9VR\nU7tl0m75dlLXT8riBsNaXxjdrlud5qFuGq114I106EaUDutARNwvaSfg30l9xuwJIOk64LMR0anv\n6yZ61b90k362W44mnRP+F+BoSZNIHYvdStqrb2pN4JGo7y+n21xoLds/d/H+y3x2kbr+hQaf3ViD\nsTXTXffLW7M30dRUUidAjd6HFD6l1nhHR8Q7R1mPOu3mqdv+nHthEelQcLU2e1zluJD2ZOsMoje4\nbtet1oZQNw9102i97n0R8V/dVy+J1GvhO3NXpNuQ7vf9GPATSTtExGiPjHqpcT/bEbFI0vHAXpI2\nJu04rAMc0OU2uwjYRNKaNeHY7fZ6X35ev4v3H5Mx/VY6z/AtwAsk1X34u+bn0fbLW2fnDmV1h2h1\nriVd+dqupnvJdp4A6GL8qm77c+6Fi0jfjNuNNGKxHJ9dHZb7Z96pLzXs7HrSyf4dchevpV3y8xXF\n88tqxq1bZy7KzzuMuoYVEbE4In4XEV8mBeOKpB75hsE2klatFuQ9we1Jn3G5o3Ek6WhjH9Ke45Ok\n84mlTv1at7b5XWqGdZsL15O21x0lrd7wNWPSi0Ykfkg6F/TlamE+fHwT6XaL83rwPi0fzYHSep8p\npHubngCObTKBiFhMujVhM+DAurCT9NJiZerYh+8Iuu3PuRcOJ30mh0l6TjlQ0rrV+pBunVmFJZ2p\nt+xD9+cXxyzv5R5HOtf3ieowSf9MungyLyJavw46hbTxzJK0UWXc1VnSzWt1+heSwnGf3M3qUiRN\nyutwW5JeJGnDmkGt9bNpX9v9NgXYtyj7CGld/klELHWOOyJ+S7qSPwt4DXB65XOu6rRNtPrr/lJ1\nO5K0Dulc+2LSfbwjytvrkcDawNfzl/VTJK2T99h7phcT+yrp/Mr7Jf0d6Z6pDUj98j4CvCc69F87\nCpcCV+Td/SdI53U2AL4cETd0MZ0vAjNINzv/o6TfkBb0c3P5ZqSNsnXS+mzS/XfHSfol6TaPcyOi\n7sR+qav+nHshIn4v6SP5vf8g6RekOwvWIl1Q2Jl0/vHa/JKvkebvI3k5/pb0GbyedA/h3/e6jg18\nmrR3cYik3UjLfjPSBYN7SN2DAhAR90n6OOkCwSWSfkwKpjeTNvIta6b/DtJyPSUv/8tJy+Z5pD3P\ne8kXqNp4JfBNSeeQ9moWkq7gvo50Xu9Ho5vtnjsX+DdJO5Iu6LyQdEX9DtJpnjpHku5PBGh3quFs\nUuB+X9JPSdvK7yPi1IiYJ+l7pGV0laQTSaeT3ko6NP90RNzUxTx8gXTh8v3AyyT9mrQnuxnpJvh1\nWXLIPXZd3FawgJrbdfKwNUg3BN9IusJ6D+mG161qxp1Hm9teRnj/uaTzD5vkD+kWUjhdR+V+uZrL\n/zM7THMl4EOke/0WkTakm0m3Gb2bpe+NmkS6yfZPpI0nqPnlS4f3atyfc13dO73HCMN2BH5C2gge\ny+99AemLYVox7lTSbRZ3ky4OnceSG/y7vV1nbs2wvfOwvTss3+lF+TqkcP9jpf4/pOaXKZXP+XKa\n//Ll2aR7Wq/Jy38R6cviKIr+sstpkELwO3m53puX7Q2kG5s37HK9nt7wc2o7rNOyYOlfvtxPuh2p\n9jOsfC6tG+kndxhvf9JR4ePlcicdjr8PuCR/Ng+Q7hp4c810RlzH8nLcn3QF/eE8H5eTjlaX+eVL\nm2m0HVZ9jJs+XyTNJd2vt1FELBhsbcyGX2715hbgBxGxd5evfTnpTopvR8Qne165IeeGas2sTusm\n7DmDrsgg9PSEpZmNX5Kmkc67bkU6t3xcRFw32FoNhoPRzFo2Jp1v/Rup0YUPDbY6g9P3c4xrr712\nTJ8+va/vYWZPP5dccsndEdHu5vUx6fse4/Tp07n44mG4+d/MJhJJt/Zr2r74YmZWcDCamRUcjGZm\nBQejmVnBwWhmVnAwmpkVGgWjpJmSoubRu9YszMyGRLf3MX6U1Ltfy+J2I5qZjVfdBuO1EXFBX2pi\nZjYkfI7RzKzQbTAeLekJSfdIOia3xmFmNqE0PZS+n9QS8nxSC8DbAJ8Fzpe0TUQs1eWhpFmk/iKY\nNq377Jy+36kdhy84ZFj6GDKziahRMEbEZSzd+ff83M/FRaQLMp8vxp9DbuByxowZ46OJcDOzbNTn\nGCPiUuAPNOie08xsPBnrxRfRvpN5M7NxadTBKKnVxeiFvauOmdngNTrHKOloUm9jl5L6bt2G1I3h\n7aSuIs3MJoymV6WvAt4OfARYldS378+AAyLi7j7VzcxsIJpelT6Y1EmOmdmE51++mJkVHIxmZgUH\no5lZwcFoZlZwMJqZFRyMZmYFB6OZWcHBaGZWcDCamRUcjGZmBQejmVnBwWhmVnAwmpkVHIxmZgUH\no5lZwcFoZlZwMJqZFRyMZmYFB6OZWcHBaGZWcDCamRUcjGZmBQejmVnBwWhmVnAwmpkVHIxmZgUH\no5lZwcFoZlZwMJqZFRyMZmYFB6OZWcHBaGZWcDCamRUcjGZmBQejmVnBwWhmVnAwmpkVHIxmZoVR\nBaOkX0oKSQf2ukJmZoPWdTBKejuwVR/qYmY2FLoKRknPAr4NfLI/1TEzG7xu9xi/BlwdEcf2ozJm\nZsNgpaYjStoZeDc+jDazCa7RHqOkScARwDci4vr+VsnMbLCa7jF+BlgFOKjJyJJmAbMApk2bNrqa\njcH0/U4d0+sXHLJ7j2qy/I0072Odt35P32wYjLjHKGka8DngC8DKkp6VL8JQ+X/F6msiYk5EzIiI\nGVOnTu19rc3M+qjJofTGwDOA/wUWVh4A++a/X9yX2pmZDUCTQ+nLgd1qys8mheVRwI29rJSZ2SCN\nGIwRcR8wryyXBHBrRCwzzMxsPPNvpc3MCo3vYyxFhHpZETOzYeE9RjOzgoPRzKzgYDQzKzgYzcwK\nDkYzs4KD0cys4GA0Mys4GM3MCg5GM7OCg9HMrOBgNDMrOBjNzAoORjOzgoPRzKzgYDQzKzgYzcwK\nDkYzs4KD0cys4GA0Mys4GM3MCg5GM7OCg9HMrOBgNDMrOBjNzAoORjOzgoPRzKzgYDQzKzgYzcwK\nDkYzs4KD0cys4GA0Mys4GM3MCg5GM7OCg9HMrOBgNDMrOBjNzAoORjOzgoPRzKzgYDQzKzQKRkmv\nlnSWpDslPSrpNknHS9qy3xU0M1veVmo43hTgEuBw4C5gGrAfcIGkF0fErX2qn5nZctcoGCPiWODY\napmki4DrgD2Ab/a+amZmgzGWc4z35OfHe1ERM7Nh0VUwSlpR0mRJmwJHAHcCP+5LzczMBqTpOcaW\nC4Ft8983Ai+PiL/2tkpmZoPVbTC+C1gT2BjYFzhd0s4RsaA6kqRZwCyAadOm9aCaS5u+36k9n6Yt\nHyMtuwWH7L6camLWXleH0hFxbURcmC/GvAJYnXR1uhxvTkTMiIgZU6dO7VFVzcyWj1FffImI+0iH\n08/vXXXMzAZv1MEoaV1gC+Cm3lXHzGzwGp1jlHQicClwJbAI2Az4BLAY38NoZhNM04svFwBvBf4N\nmAz8CZgHHFxeeDEzG++a/vLlq8BX+1wXM7Oh4NZ1zMwKDkYzs4KD0cys4GA0Mys4GM3MCg5GM7OC\ng9HMrOBgNDMrOBjNzAoORjOzgoPRzKzgYDQzKzgYzcwKDkYzs4KD0cys4GA0Mys4GM3MCg5GM7OC\ng9HMrOBgNDMrOBjNzAoORjOzgoPRzKzgYDQzKzgYzcwKDkYzs4KD0cys4GA0Mys4GM3MCg5GM7OC\ng9HMrOBgNDMrOBjNzAoORjOzgoPRzKzgYDQzKzgYzcwKDkYzs4KD0cysMGIwStpD0gmSbpX0sKTr\nJR0saY3lUUEzs+WtyR7jvsATwGeB1wDfAz4AnC7Je5xmNuGs1GCc10fEXZX/50u6F/gBMBM4qx8V\nMzMblBH3+IpQbPldft6gt9UxMxu80R4K75qfr+1VRczMhkXXwShpA+DLwBkRcXHvq2RmNlhNzjE+\nRdLqwMnAYmCfDuPNAmYBTJs2bSz1s3Fm+n6nDroKZmPWeI9R0jOAU4CNgVdHxG3txo2IORExIyJm\nTJ06tQfVNDNbfhrtMUqaBJwAbA+8MiJ+39damZkN0IjBmO9VPBp4BbB7RFzQ91qZmQ1Qkz3Gw4A9\ngYOAByXtUBl2W6dDajOz8ajJOcbX5ufPAecXj/f2qV5mZgMz4h5jRExfDvUwMxsa/q2zmVnBwWhm\nVnAwmpkVHIxmZgUHo5lZwcFoZlZwMJqZFRyMZmYFB6OZWcHBaGZWcDCamRUcjGZmBQejmVnBwWhm\nVnAwmpkVHIxmZgUHo5lZwcFoZlZwMJqZFRyMZmYFB6OZWcHBaGZWcDCamRUcjGZmBQejmVnBwWhm\nVnAwmpkVHIxmZgUHo5lZwcFoZlZwMJqZFRyMZmYFB6OZWcHBaGZWcDCamRUcjGZmBQejmVnBwWhm\nVnAwmpkVGgWjpOdK+q6k8yU9JCkkTe9v1czMBqPpHuPzgbcCC4Hf9K86ZmaD1zQYz4mIdSPiH4Cf\n9LNCZmaD1igYI+LJflfEzGxY+OKLmVlhpX5MVNIsYBbAtGnT+vEWE9r0/U5tO2zBIbsvx5rY8tRp\nucP4X/bjaf76sscYEXMiYkZEzJg6dWo/3sLMrG98KG1mVnAwmpkVHIxmZoXGF18k7ZH/3DY/v1bS\nXcBdETG/5zUzMxuQbq5Klzd2H56f5wMze1IbM7Mh0DgYI0L9rIiZ2bDwOUYzs4KD0cys4GA0Mys4\nGM3MCg5GM7OCg9HMrOBgNDMrOBjNzAoORjOzgoPRzKzgYDQzKzgYzcwKDkYzs4KD0cys4GA0Mys4\nGM3MCg5GM7OCg9HMrOBgNDMrOBjNzAoORjOzgoPRzKzgYDQzKzgYzcwKDkYzs4KD0cys4GA0Mys4\nGM3MCg5GM7OCg9HMrOBgNDMrOBjNzAoORjOzgoPRzKzgYDQzKzgYzcwKDkYzs4KD0cys0CgYJW0o\n6aeS7pe0SNLPJE3rd+XMzAZhxGCUtCpwFrAFsBfwLmBT4GxJq/W3emZmy99KDcZ5H7AxsHlE3Agg\n6UrgBuBfgW/1r3pmZstfk0PpNwAXtEIRICJuAc4D3tivipmZDUqTYHwhcFVN+dXAlr2tjpnZ4DUJ\nxinAwprye4G1elsdM7PBU0R0HkF6DPhmROxflB8EfCYiljlPKWkWMCv/uzlw/RjruTZw9xin8XTj\nz6w7/ry6N+jP7HkRMbUfE25y8WUhaa+xtBb1e5JExBxgzhjqtRRJF0fEjF5N7+nAn1l3/Hl1byJ/\nZk0Opa8mnWcsbQlc09vqmJkNXpNgPAXYQdLGrQJJ04H/l4eZmU0oTYLxSGABcLKkN0p6A3Ay8Cfg\niD7Wrapnh+VPI/7MuuPPq3sT9jMb8eILQP7537eBVwECzgQ+HhEL+lo7M7MBaBSMZmZPJ0PZuo6k\nPSSdIOlWSQ9Lul7SwZLWGHTdhpWkV0s6S9Kdkh6VdJuk4yX5JvyGJP1SUkg6cNB1GUaSZubPp3zc\nN+i69VqT23UGYV/gj8BngdvEbnBHAAAIQElEQVSAbYDZwG6SdoqIJwdYt2E1BbgEOBy4C5gG7Adc\nIOnFEXHrICs37CS9Hdhq0PUYJz4K/K7y/+JBVaRfhjUYXx8Rd1X+ny/pXuAHwExSaz9WERHHAsdW\nyyRdBFwH7AF8cxD1Gg8kPYt0Dv0TwDEDrs54cG1EXDDoSvTTUB5KF6HY0vqG2mB51mWcuyc/Pz7Q\nWgy/rwFX5y8Xs+EMxjZ2zc/XDrQWQ07SipImS9qUdDvVncCPB1ytoSVpZ+DdwAcHXZdx5GhJT0i6\nR9IxE7HR6mE9lF6KpA2ALwNnRMTFg67PkLsQ2Db/fSPw8oj46wDrM7QkTSJ9eXwjIsb6e/6ng/tJ\np2TmA4tI5/4/C5wvaZuJtJ4NfTBKWp10Q/liYJ8BV2c8eBewJqlx4X2B0yXt7HtOa30GWAU4aNAV\nGQ8i4jLgskrRfEnnABeRLsh8fiAV64OhDkZJzyD97HBjYNeIuG3AVRp6EdE61XChpNNIv1raD3j/\nwCo1hPLh3+eA9wIrS1q5MnjlfEHmbxHxxEAqOE5ExKWS/gBsN+i69NLQnmPMhzknANsD/xARvx9w\nlcadiLiPdDj9/EHXZQhtDDwD+F9SK1GtB6Q97YXAiwdTtXFHwIT6pchQ7jFKWgE4GngFsPtEvzWg\nXyStS+rE7OhB12UIXQ7sVlN+NiksjyJ9qVgHkmYAmwHHD7ouvTSUwQgcBuxJOvfzoKQdKsNu8yH1\nsiSdCFwKXEk6Mb4Z6b68xfgexmXkvel5ZbkkgFsjYplhT3eSjgZuIa1n95EuvuwP3A58d4BV67mh\n/K20pAXA89oM/lJEzF5+tRkfJH0GeCuwCTCZ1PrRPOBgX3hpTlIAB0XEhLmQ0CuS9gfeTto2VyXd\nCnYacEBE3DHIuvXaUAajmdkgDe3FFzOzQXEwmpkVHIxmZgUHo5lZwcFoZlZwMJqZFRyMZmYFB6OZ\nWcHBOMFJeo+kGyQ9NkydFkn6rqT/qynfKHew9FdJ/1Iz/BOSrsy/p+93HTvWpctprZsbdb1U0tWS\nXiTpYkmr9aq+1jsOxglM0vqkTtF/C7wceOVga5RI2gT4V+BLNYPvJvVfvpDUOHHp+8A6wF59q2Dz\nunTjRGB+RLwEOInUxuiciHhwjNO1PnAwTmybAisCP4iIc3vR+nnRbuFofRy4oq4+EfG3iDiD1LrN\n+pLWKYY/DPyQ1DRYX41Ul5a8F3h3m8eGkrYD1oqII/JL7gAezdO1IeRgHAckTZL0eUk35362z5e0\nuaQZkh7Ke4bla+aypPWYM/Mh4dzK8Nfk6Tws6X5JJ0navJjG7Py6F0n6laQHGKF5KUkrSdo31/Ux\nSXdJml0ZvjLwTkbuja/V1UBdl6Y/BraUtNMI0+iVTnUhIl4SEWu3efyJ1Kbo+ZWXbA3MdSO4w8vB\nOORyg72nkZqOP4jUHNsGwIHAAcBhEfHnmpf+e34NwIeAHXMZkl4DnAo8ALwN+ADwIuDc3L9O6WRS\nPx9vIHUz2qmu/0faIzwkjz8POEDS2/JoOwDPAn7TYToCPpn//buaUS4nNa32mnbT6JUGdWnicVI/\n30iaCfwTqQUkG1YR4ccQP4CPAU8CO1XKvkRqD+9+4NkdXvtKUsvKM4vyi4EbgJUqZRuRNuBvVcpm\n59d/rGFd98t1mlYpm0QKsWPy/5/J8zO5w3Q+BDxGCu4fthnnN8CvR6jPCsB67d6LdHg70jyNWJcG\n01gLOIPUx/fZpPO9twGvGvT65Uf9Y1gbqrUl3k8KgN9Wyu4DngnMjoh76l9WL18FfQnwlYhY3CqP\niFsknceSbmqrTmww3RVIe1aHRcQfK9N9XNKtwOq5aH1gUUQ81mY600l7mwcBu9B+L+0uUmO8ddOY\nTDoH+THShZrHJZ1BOgSfT9pbeyfp0PhNHeapaV06ioiFLHvh67mjmZYtHz6UHmKS1iN1TfCLYtBk\n4F7gW6OY7FqkPjrqGha9E5hSU96kEdLtgamkQ/Sn5EPR9UktP0PqZ+XRDtM5ktSlwFdIh8wvyIfo\npYdJPfzVeRXwatIpgq1JDfjeTgq5BcAVub4HjjBPTetiE4z3GIdbqxOrVqggaUVSB/E3RcTfRjHN\nhaTD4/Vqhq0H1O2BNmnNuNX9RNntxE6ksD0p/38PKZyXIWkWaY91u7yneTnpS2ALoOwMbQrpdpo6\n5wC/iHwcSwrCk/J7PJPU+9+TnWamy7rYBOM9xuHW2nire3EfALYk3YbTtUj3zV0C7JlDFgBJzyOF\n2PzRVZVt8/MmlWmuABxM6oemNd3rgEmSljqUzP9/HTgwIq7IxZfn57pD2I1YcrV4KZFus6kN84i4\nv0EodlsXm2AcjMPtCtLFjC9KepOkDwNfI3Ur+yJJr819b3frC6R7HH8u6fWS3g6cnt9rtB1nbUva\nWzxc0lskvRk4k9Rh0tsrYXROft6+eP0c4CZSkLZcSzrsXuo2mdzn82aVafVa47rYxORD6SEWEQ9K\n2pN0i8zxpAsO7wN+Rbr94+fAmqOY7i8l7U663ed40lXXecCno/7Wn47yBZ3NSVdwpwNHkM7/zQd2\njIhrKu+9QNJFwOuBn+XX7026OLFdRDxeGXexpKtYdi9t91znES8KjWJeuq2LTUDuDMvGTNLOpNtn\nZkTEJQ3G3xv4DvCciHhoFO93GnB3RLyr29eaNeFDaeuFbUl7cE0vSvyIdJX4g92+kaStgd2o/521\nWU84GK0XtgWuandvYinST+HeA3S9t0i6cr5PRNw4iteaNeJDaTOzgvcYzcwKDkYzs4KD0cys4GA0\nMys4GM3MCg5GM7OCg9HMrOBgNDMrOBjNzAr/HwWYMGAPgRixAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(all_alphas,bins=32);\n",
    "plt.title(r\"Power law exponents $\\alpha$ \"+\"\\nfor pretrained models in pytorch\")\n",
    "plt.xlabel(r\"$\\alpha$ for $\\rho(\\lambda)\\sim\\lambda^{-\\alpha}$\")\n",
    "plt.savefig(\"img/power-law-exponents\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}