All-pytorch-models-wCNNs-Slices.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preliminary analysis of extracted convolutional layers\n",
    "\n",
    "### includes auto MP fits and mp soft ranks\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-27T08:06:08.823252Z",
     "start_time": "2018-08-27T08:06:06.545222Z"
    }
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision.models as models\n",
    "import torchvision.transforms as transforms\n",
    "import torch.nn as nn\n",
    "\n",
    "import numpy as np\n",
    "import powerlaw\n",
    "\n",
    "import sklearn\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "\n",
    "import matplotlib\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-27T08:06:11.499813Z",
     "start_time": "2018-08-27T08:06:08.826030Z"
    }
   },
   "outputs": [],
   "source": [
    "import import_ipynb\n",
    "import RMT_Util"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-27T08:06:11.506733Z",
     "start_time": "2018-08-27T08:06:11.501899Z"
    }
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['figure.figsize'] = [5,5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### All Linear Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-27T13:12:09.422260Z",
     "start_time": "2018-08-27T13:12:09.341227Z"
    }
   },
   "outputs": [],
   "source": [
    "def list_layers(model):\n",
    "    pretrained_model = model(pretrained=True)\n",
    "    \n",
    "    model_name = model.__name__\n",
    "    model_name = model_name.replace(\"_\",' ')\n",
    "    alphas = []\n",
    "    soft_ranks = []\n",
    "    \n",
    "    for im, m in enumerate(pretrained_model.modules()):\n",
    "        #print(im,m)\n",
    "        if m isinstance(m, nn.Linear):     \n",
    "            W = np.array(m.weight.data.clone().cpu())\n",
    "            if W is not None:\n",
    "                print(im,m, W.shape)\n",
    "        elif m isinstance(m, nn.Conv2d):\n",
    "            #print(\"conv2d\")\n",
    "            Wtensor = np.array(m.weight.data.clone().cpu())\n",
    "            Wmats = get_conv2D_Wmats(Wtensor)\n",
    "            print(im,m)\n",
    "            for W in Wmats:\n",
    "                print(W.shape)\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_conv2D_Wmats(Wtensor):\n",
    "    \"\"\"Extract W slices from a 4 index conv2D tensor of shape: (N,M,i,j) or (M,N,i,j).  Return ij (N x M) matrices\"\"\"\n",
    "    Wmats = []\n",
    "    s = Wtensor.shape\n",
    "    N, M, imax, jmax = s[0],s[1],s[2],s[3]\n",
    "    print(\"tensor shape\", N,M,imax,jmax)\n",
    "    for i in range(imax):\n",
    "        for j in range(jmax):\n",
    "            W = Wtensor[:,:,i,j]\n",
    "            if N < M:\n",
    "                W = W.T\n",
    "            Wmats.append(W)\n",
    "    return Wmats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-27T13:12:09.422260Z",
     "start_time": "2018-08-27T13:12:09.341227Z"
    }
   },
   "outputs": [],
   "source": [
    "def analyze_model(model, linear=False, conv2d=True, plot=True):\n",
    "    pretrained_model = model(pretrained=True)\n",
    "    \n",
    "    model_name = model.__name__\n",
    "    model_name = model_name.replace(\"_\",' ')\n",
    "    alphas = []\n",
    "    soft_ranks = []\n",
    "    min_svs = []\n",
    "    Qs = []\n",
    "    \n",
    "    for im, m in enumerate(pretrained_model.modules()):\n",
    "        Wmats = []\n",
    "        #print(im, isinstance(m, nn.Conv2d))\n",
    "        if linear and isinstance(m, nn.Linear):     \n",
    "            W = np.array(m.weight.data.clone().cpu())\n",
    "            Wmats = [W]\n",
    "        elif conv2d and isinstance(m, nn.Conv2d):\n",
    "            #print(\"conv2d\")\n",
    "            Wtensor = np.array(m.weight.data.clone().cpu())\n",
    "            Wmats = get_conv2D_Wmats(Wtensor)\n",
    "            \n",
    "        #print(\"num mats \",len(Wmats))\n",
    "        if Wmats is not None and len(Wmats)>0:\n",
    "            for W in Wmats:\n",
    "                #print(W.shape)\n",
    "\n",
    "                M, N = np.min(W.shape), np.max(W.shape)\n",
    "                Q=N/M \n",
    "                if M > 49:\n",
    "                    sv, _ = RMT_Util.singular_spectrum(W)\n",
    "                    evals = sv*sv\n",
    "                    #scaled_evals = (1/N)*evals\n",
    "                    # RMT_Util.fit_power_law\n",
    "                    fit = powerlaw.Fit(evals, xmax=np.max(evals), verbose=False)  \n",
    "                    alpha = fit.alpha\n",
    "                    D = fit.D\n",
    "\n",
    "                    dist = RMT_Util.best_dist(fit)\n",
    "                    sigma = RMT_Util.fit_mp(evals, Q)\n",
    "                    mp_soft_rank = RMT_Util.calc_mp_soft_rank(evals,Q,sigma)\n",
    "\n",
    "                    min_svs.append(np.min(sv))\n",
    "\n",
    "                    alphas.append(alpha)\n",
    "                    soft_ranks.append(mp_soft_rank)\n",
    "                    Qs.append(Q)\n",
    "    return alphas, soft_ranks, min_svs, Qs\n"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-27T08:14:29.594092Z",
     "start_time": "2018-08-27T08:06:11.627775Z"
    }
   },
   "source": [
    "for model in [models.vgg11]:\n",
    "    list_layers(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PoweLaw fit for all FC and CNN layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-27T09:07:33.367507Z",
     "start_time": "2018-08-27T08:14:29.596376Z"
    }
   },
   "outputs": [],
   "source": [
    "all_alphas, all_ranks, all_min_svs, all_Qs = [], [], [], []\n",
    "for model in tqdm([models.alexnet, \n",
    "              models.densenet121, models.densenet161, models.densenet169, models.densenet201, \n",
    "              models.inception_v3,\n",
    "              models.resnet101, models.resnet152, models.resnet18, models.resnet34, models.resnet50, \n",
    "              models.squeezenet1_0, models.squeezenet1_1,\n",
    "              models.vgg11, models.vgg11_bn,\n",
    "              models.vgg16, models.vgg16_bn,\n",
    "              models.vgg19, models.vgg19_bn ]):\n",
    "    #print(model)\n",
    "    #analyze_model(model, conv2d=True, plot=False)\n",
    "    alphas, soft_ranks, min_svs, Qs = analyze_model(model)\n",
    "    all_alphas.extend(alphas)\n",
    "    all_ranks.extend(soft_ranks)\n",
    "    all_min_svs.extend(min_svs)\n",
    "    all_Qs.extend(Qs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-27T13:37:07.163339Z",
     "start_time": "2018-08-27T13:37:07.027236Z"
    }
   },
   "outputs": [],
   "source": [
    "plt.hist(all_alphas,bins=100);\n",
    "plt.title(r\"Power Law Exponents ($\\alpha$) for PyTorch Conv2D Layers\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-27T13:37:37.568004Z",
     "start_time": "2018-08-27T13:37:37.440336Z"
    }
   },
   "outputs": [],
   "source": [
    "plt.hist(all_ranks,bins=100);\n",
    "plt.title(r\"MP Soft Rank $\\mathcal{R}_{mp}$ for PyTorch Conv2D Layers\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "aa = np.array(all_alphas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(aa), len(aa[aa<5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "100.0*len(aa[aa<5])/len(aa), 100.0*len(aa[aa<4])/len(aa),  100.0*len(aa[aa<1.5])/len(aa)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Rank Loss\n",
    "\n",
    "...only ~ 300/3000 ~ %13\n",
    "\n",
    "This is hard to see in the plot\n",
    "\n",
    "It may be easier to see on a log plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ms = np.array(all_min_svs)\n",
    "len(ms[ms<0.00001]), len(ms), 100.0*len(ms[ms<0.00001])/len(ms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.hist(all_min_svs,bins=100);\n",
    "plt.title(r\"Min Singular Value PyTorch Conv2D Layers\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Log Scale on Min Values\n",
    "\n",
    "#TODO:  remove Q=1 cases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lm = np.log10(all_min_svs)\n",
    "plt.hist(lm,bins=100);\n",
    "plt.title(r\"$Log_{10}(\\nu_{min})$ PyTorch Conv2D Layers\")\n",
    "plt.xlabel(\"Log10 Min Singular Value\")\n",
    "plt.savefig(\"img/log-min-nu-pytorch-conv2d\")\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(lm[lm<-6]), len(lm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "widgets": {
   "state": {
    "52ecdae62c394a67868cd3ee83e19861": {
     "views": [
      {
       "cell_index": 14
      }
     ]
    },
    "e232317699e24554b5ed09fa6c78c662": {
     "views": [
      {
       "cell_index": 8
      }
     ]
    }
   },
   "version": "1.2.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}