PTH2TF.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import dilated_model as dm\n",
    "import torch as th\n",
    "from tensorflow.python.framework import ops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pthnet = dm.EncodeWideResNetFIXED(in_channel=1, init_channel=32, num_enc_layer=4, N_res_in_block=1, use_selu=True, num_classes=3)\n",
    "pthnet = dm.EncodeWideResNet(in_channel=1, init_channel=32, num_enc_layer=4, N_res_in_block=1, use_selu=True, num_classes=3)\n",
    "sd = th.load('saved/ENCODE-selu-adam/0002/state_dict_highscore')\n",
    "#sd = th.load('saved/fixed-noise/0001/state_dict_highscore')\n",
    "pthnet.load_state_dict(sd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def selu(x):\n",
    "    with ops.name_scope('elu') as scope:\n",
    "        alpha = 1.6732632423543772848170429916717\n",
    "        scale = 1.0507009873554804934193349852946\n",
    "        return scale*tf.where(x>=0.0, x, alpha*tf.nn.elu(x))\n",
    "    \n",
    "#def init(*shape):\n",
    "#    return np.zeros(shape, dtype='float32')\n",
    "\n",
    "def gener():\n",
    "    for p in sd.values():\n",
    "        yield p.cpu().transpose(0, -1).numpy()    \n",
    "paramgen = gener()\n",
    "def init(*args, do_assert=True):\n",
    "    p = next(paramgen)\n",
    "    if do_assert:\n",
    "        assert p.shape == args, (p.shape, args)\n",
    "    return p\n",
    "    \n",
    "def BatchNorm(input, channel=8):\n",
    "    with tf.variable_scope('BatchNorm'):\n",
    "        weight = tf.Variable(init(channel), name='weight')\n",
    "        bias = tf.Variable(init(channel), name='bias')\n",
    "        mean = tf.Variable(init(channel), name='running_mean')\n",
    "        var = tf.Variable(init(channel), name='running_var')\n",
    "        return tf.nn.batch_normalization(input, mean, var, bias, weight, 1e-05)\n",
    "\n",
    "def Conv1d(input, in_channel, out_channel, kernel_size, dilation=1, bias=False):\n",
    "    with tf.variable_scope('Conv1d'):\n",
    "        w = tf.Variable(init(kernel_size, in_channel, out_channel), name='weight')\n",
    "        if dilation > 1:\n",
    "            w = tf.expand_dims(w, 0)\n",
    "            x = tf.expand_dims(input, 1)\n",
    "            p = kernel_size - 1 #// 2\n",
    "            x = tf.pad(x, [[0, 0], [0, 0], [p, p], [0, 0]], \"CONSTANT\")\n",
    "            x = tf.nn.atrous_conv2d(x, w, dilation, 'VALID')\n",
    "            x = tf.squeeze(x, 1)\n",
    "        else:\n",
    "            p = kernel_size // 2\n",
    "            x = tf.pad(input, [[0, 0], [p, p], [0, 0]], \"CONSTANT\")\n",
    "            x = tf.nn.conv1d(x, w, 1, 'VALID')\n",
    "        if bias:\n",
    "            b = tf.Variable(init(out_channel), name='bias')\n",
    "            x = x + b\n",
    "    return x\n",
    "    \n",
    "def MaxPool1d(input):\n",
    "    with tf.variable_scope('MaxPool1d'):\n",
    "        x = tf.expand_dims(input, 1)\n",
    "        x = tf.nn.max_pool(x, [1, 1, 2, 1], [1, 1, 2, 1], 'SAME')\n",
    "        x = tf.squeeze(x, 1)\n",
    "    return x\n",
    "    \n",
    "\n",
    "def Encoder(input, init_channel):\n",
    "    def DownSampleBlock(input, in_channel, out_channel):\n",
    "        with tf.variable_scope('DownSampleBlock'):\n",
    "            x = Conv1d(input, in_channel, out_channel, 7, bias=True)\n",
    "            x = BatchNorm(x, out_channel)\n",
    "            x = selu(x)\n",
    "            x = MaxPool1d(x)\n",
    "        return x    \n",
    "    with tf.variable_scope('Encoder'):\n",
    "        x = DownSampleBlock(input, 1, init_channel)\n",
    "        x = DownSampleBlock(x, init_channel, init_channel*2)\n",
    "        x = DownSampleBlock(x, init_channel*2, init_channel*4)\n",
    "        x = DownSampleBlock(x, init_channel*4, init_channel*8)\n",
    "    return x\n",
    "    \n",
    "def DilatedBlock(input, channel=8, kernel_size=9, dilation=2):\n",
    "    # No change in # of channels -> identity mapping\n",
    "    with tf.variable_scope('DilatedBlock'):\n",
    "        x = BatchNorm(input, channel)\n",
    "        x = Conv1d(x, channel, channel, kernel_size)\n",
    "        x = selu(x)\n",
    "        x = BatchNorm(x, channel)\n",
    "        x = Conv1d(x, channel, channel, kernel_size, dilation)\n",
    "        x = selu(x)\n",
    "    return x + input    \n",
    "    \n",
    "def ResNet(input, channel):\n",
    "    with tf.variable_scope('ResNet'):\n",
    "        x = DilatedBlock(input, channel)\n",
    "        for _ in range(8):\n",
    "            x = DilatedBlock(x, channel)\n",
    "        \n",
    "    return x\n",
    "\n",
    "def Features(input, init_channel):\n",
    "    x = Encoder(input, init_channel)\n",
    "    x = ResNet(x, init_channel*8)\n",
    "    return x\n",
    "\n",
    "def NET(input, init_channel=32, num_classes=3):\n",
    "    x = Features(input, init_channel)\n",
    "    with tf.variable_scope('Logit'):\n",
    "        logit = Conv1d(x, init_channel*8, num_classes, 1)\n",
    "        logit = tf.reduce_mean(logit, 1)\n",
    "        #print('MEGTÖRTÉNT')\n",
    "        \n",
    "    return logit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "channel = 1\n",
    "testinput = np.random.randn(*[10, channel, 1024])\n",
    "def forward(self, x, lens=None):\n",
    "    if lens is None:\n",
    "        lens = x.size(-1)\n",
    "    else:\n",
    "        lens = lens[:, None].expand(len(x), self.num_classes)\n",
    "    \n",
    "    #out = self.forward_encoder(x)\n",
    "    #out = self.forward_features(x)\n",
    "    #out = self.logit(out)\n",
    "    #out = th.sum(out, dim=-1).squeeze() / lens\n",
    "    out = self.forward_features(x)\n",
    "    out = th.mean(self.logit(out), -1)\n",
    "    return out\n",
    "\n",
    "pytorch_testmodule = pthnet\n",
    "pytorch_testmodule.eval()\n",
    "pth_input = th.autograd.Variable(th.FloatTensor(testinput))\n",
    "pth_result = forward(pytorch_testmodule, pth_input).data.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OK\n",
      "           Encoder/DownSampleBlock/Conv1d/weight:0\t                                  encoder.0.weight\n",
      "             Encoder/DownSampleBlock/Conv1d/bias:0\t                                    encoder.0.bias\n",
      "        Encoder/DownSampleBlock/BatchNorm/weight:0\t                                  encoder.1.weight\n",
      "          Encoder/DownSampleBlock/BatchNorm/bias:0\t                                    encoder.1.bias\n",
      "  Encoder/DownSampleBlock/BatchNorm/running_mean:0\t                            encoder.1.running_mean\n",
      "   Encoder/DownSampleBlock/BatchNorm/running_var:0\t                             encoder.1.running_var\n",
      "         Encoder/DownSampleBlock_1/Conv1d/weight:0\t                                  encoder.4.weight\n",
      "           Encoder/DownSampleBlock_1/Conv1d/bias:0\t                                    encoder.4.bias\n",
      "      Encoder/DownSampleBlock_1/BatchNorm/weight:0\t                                  encoder.5.weight\n",
      "        Encoder/DownSampleBlock_1/BatchNorm/bias:0\t                                    encoder.5.bias\n",
      "Encoder/DownSampleBlock_1/BatchNorm/running_mean:0\t                            encoder.5.running_mean\n",
      " Encoder/DownSampleBlock_1/BatchNorm/running_var:0\t                             encoder.5.running_var\n",
      "         Encoder/DownSampleBlock_2/Conv1d/weight:0\t                                  encoder.8.weight\n",
      "           Encoder/DownSampleBlock_2/Conv1d/bias:0\t                                    encoder.8.bias\n",
      "      Encoder/DownSampleBlock_2/BatchNorm/weight:0\t                                  encoder.9.weight\n",
      "        Encoder/DownSampleBlock_2/BatchNorm/bias:0\t                                    encoder.9.bias\n",
      "Encoder/DownSampleBlock_2/BatchNorm/running_mean:0\t                            encoder.9.running_mean\n",
      " Encoder/DownSampleBlock_2/BatchNorm/running_var:0\t                             encoder.9.running_var\n",
      "         Encoder/DownSampleBlock_3/Conv1d/weight:0\t                                 encoder.12.weight\n",
      "           Encoder/DownSampleBlock_3/Conv1d/bias:0\t                                   encoder.12.bias\n",
      "      Encoder/DownSampleBlock_3/BatchNorm/weight:0\t                                 encoder.13.weight\n",
      "        Encoder/DownSampleBlock_3/BatchNorm/bias:0\t                                   encoder.13.bias\n",
      "Encoder/DownSampleBlock_3/BatchNorm/running_mean:0\t                           encoder.13.running_mean\n",
      " Encoder/DownSampleBlock_3/BatchNorm/running_var:0\t                            encoder.13.running_var\n",
      "            ResNet/DilatedBlock/BatchNorm/weight:0\t               resnet.0.residuals.0.block.0.weight\n",
      "              ResNet/DilatedBlock/BatchNorm/bias:0\t                 resnet.0.residuals.0.block.0.bias\n",
      "      ResNet/DilatedBlock/BatchNorm/running_mean:0\t         resnet.0.residuals.0.block.0.running_mean\n",
      "       ResNet/DilatedBlock/BatchNorm/running_var:0\t          resnet.0.residuals.0.block.0.running_var\n",
      "               ResNet/DilatedBlock/Conv1d/weight:0\t               resnet.0.residuals.0.block.1.weight\n",
      "          ResNet/DilatedBlock/BatchNorm_1/weight:0\t               resnet.0.residuals.0.block.3.weight\n",
      "            ResNet/DilatedBlock/BatchNorm_1/bias:0\t                 resnet.0.residuals.0.block.3.bias\n",
      "    ResNet/DilatedBlock/BatchNorm_1/running_mean:0\t         resnet.0.residuals.0.block.3.running_mean\n",
      "     ResNet/DilatedBlock/BatchNorm_1/running_var:0\t          resnet.0.residuals.0.block.3.running_var\n",
      "             ResNet/DilatedBlock/Conv1d_1/weight:0\t               resnet.0.residuals.0.block.5.weight\n",
      "          ResNet/DilatedBlock_1/BatchNorm/weight:0\t               resnet.0.residuals.1.block.0.weight\n",
      "            ResNet/DilatedBlock_1/BatchNorm/bias:0\t                 resnet.0.residuals.1.block.0.bias\n",
      "    ResNet/DilatedBlock_1/BatchNorm/running_mean:0\t         resnet.0.residuals.1.block.0.running_mean\n",
      "     ResNet/DilatedBlock_1/BatchNorm/running_var:0\t          resnet.0.residuals.1.block.0.running_var\n",
      "             ResNet/DilatedBlock_1/Conv1d/weight:0\t               resnet.0.residuals.1.block.1.weight\n",
      "        ResNet/DilatedBlock_1/BatchNorm_1/weight:0\t               resnet.0.residuals.1.block.3.weight\n",
      "          ResNet/DilatedBlock_1/BatchNorm_1/bias:0\t                 resnet.0.residuals.1.block.3.bias\n",
      "  ResNet/DilatedBlock_1/BatchNorm_1/running_mean:0\t         resnet.0.residuals.1.block.3.running_mean\n",
      "   ResNet/DilatedBlock_1/BatchNorm_1/running_var:0\t          resnet.0.residuals.1.block.3.running_var\n",
      "           ResNet/DilatedBlock_1/Conv1d_1/weight:0\t               resnet.0.residuals.1.block.5.weight\n",
      "          ResNet/DilatedBlock_2/BatchNorm/weight:0\t               resnet.0.residuals.2.block.0.weight\n",
      "            ResNet/DilatedBlock_2/BatchNorm/bias:0\t                 resnet.0.residuals.2.block.0.bias\n",
      "    ResNet/DilatedBlock_2/BatchNorm/running_mean:0\t         resnet.0.residuals.2.block.0.running_mean\n",
      "     ResNet/DilatedBlock_2/BatchNorm/running_var:0\t          resnet.0.residuals.2.block.0.running_var\n",
      "             ResNet/DilatedBlock_2/Conv1d/weight:0\t               resnet.0.residuals.2.block.1.weight\n",
      "        ResNet/DilatedBlock_2/BatchNorm_1/weight:0\t               resnet.0.residuals.2.block.3.weight\n",
      "          ResNet/DilatedBlock_2/BatchNorm_1/bias:0\t                 resnet.0.residuals.2.block.3.bias\n",
      "  ResNet/DilatedBlock_2/BatchNorm_1/running_mean:0\t         resnet.0.residuals.2.block.3.running_mean\n",
      "   ResNet/DilatedBlock_2/BatchNorm_1/running_var:0\t          resnet.0.residuals.2.block.3.running_var\n",
      "           ResNet/DilatedBlock_2/Conv1d_1/weight:0\t               resnet.0.residuals.2.block.5.weight\n",
      "          ResNet/DilatedBlock_3/BatchNorm/weight:0\t               resnet.1.residuals.0.block.0.weight\n",
      "            ResNet/DilatedBlock_3/BatchNorm/bias:0\t                 resnet.1.residuals.0.block.0.bias\n",
      "    ResNet/DilatedBlock_3/BatchNorm/running_mean:0\t         resnet.1.residuals.0.block.0.running_mean\n",
      "     ResNet/DilatedBlock_3/BatchNorm/running_var:0\t          resnet.1.residuals.0.block.0.running_var\n",
      "             ResNet/DilatedBlock_3/Conv1d/weight:0\t               resnet.1.residuals.0.block.1.weight\n",
      "        ResNet/DilatedBlock_3/BatchNorm_1/weight:0\t               resnet.1.residuals.0.block.3.weight\n",
      "          ResNet/DilatedBlock_3/BatchNorm_1/bias:0\t                 resnet.1.residuals.0.block.3.bias\n",
      "  ResNet/DilatedBlock_3/BatchNorm_1/running_mean:0\t         resnet.1.residuals.0.block.3.running_mean\n",
      "   ResNet/DilatedBlock_3/BatchNorm_1/running_var:0\t          resnet.1.residuals.0.block.3.running_var\n",
      "           ResNet/DilatedBlock_3/Conv1d_1/weight:0\t               resnet.1.residuals.0.block.5.weight\n",
      "          ResNet/DilatedBlock_4/BatchNorm/weight:0\t               resnet.1.residuals.1.block.0.weight\n",
      "            ResNet/DilatedBlock_4/BatchNorm/bias:0\t                 resnet.1.residuals.1.block.0.bias\n",
      "    ResNet/DilatedBlock_4/BatchNorm/running_mean:0\t         resnet.1.residuals.1.block.0.running_mean\n",
      "     ResNet/DilatedBlock_4/BatchNorm/running_var:0\t          resnet.1.residuals.1.block.0.running_var\n",
      "             ResNet/DilatedBlock_4/Conv1d/weight:0\t               resnet.1.residuals.1.block.1.weight\n",
      "        ResNet/DilatedBlock_4/BatchNorm_1/weight:0\t               resnet.1.residuals.1.block.3.weight\n",
      "          ResNet/DilatedBlock_4/BatchNorm_1/bias:0\t                 resnet.1.residuals.1.block.3.bias\n",
      "  ResNet/DilatedBlock_4/BatchNorm_1/running_mean:0\t         resnet.1.residuals.1.block.3.running_mean\n",
      "   ResNet/DilatedBlock_4/BatchNorm_1/running_var:0\t          resnet.1.residuals.1.block.3.running_var\n",
      "           ResNet/DilatedBlock_4/Conv1d_1/weight:0\t               resnet.1.residuals.1.block.5.weight\n",
      "          ResNet/DilatedBlock_5/BatchNorm/weight:0\t               resnet.1.residuals.2.block.0.weight\n",
      "            ResNet/DilatedBlock_5/BatchNorm/bias:0\t                 resnet.1.residuals.2.block.0.bias\n",
      "    ResNet/DilatedBlock_5/BatchNorm/running_mean:0\t         resnet.1.residuals.2.block.0.running_mean\n",
      "     ResNet/DilatedBlock_5/BatchNorm/running_var:0\t          resnet.1.residuals.2.block.0.running_var\n",
      "             ResNet/DilatedBlock_5/Conv1d/weight:0\t               resnet.1.residuals.2.block.1.weight\n",
      "        ResNet/DilatedBlock_5/BatchNorm_1/weight:0\t               resnet.1.residuals.2.block.3.weight\n",
      "          ResNet/DilatedBlock_5/BatchNorm_1/bias:0\t                 resnet.1.residuals.2.block.3.bias\n",
      "  ResNet/DilatedBlock_5/BatchNorm_1/running_mean:0\t         resnet.1.residuals.2.block.3.running_mean\n",
      "   ResNet/DilatedBlock_5/BatchNorm_1/running_var:0\t          resnet.1.residuals.2.block.3.running_var\n",
      "           ResNet/DilatedBlock_5/Conv1d_1/weight:0\t               resnet.1.residuals.2.block.5.weight\n",
      "          ResNet/DilatedBlock_6/BatchNorm/weight:0\t               resnet.2.residuals.0.block.0.weight\n",
      "            ResNet/DilatedBlock_6/BatchNorm/bias:0\t                 resnet.2.residuals.0.block.0.bias\n",
      "    ResNet/DilatedBlock_6/BatchNorm/running_mean:0\t         resnet.2.residuals.0.block.0.running_mean\n",
      "     ResNet/DilatedBlock_6/BatchNorm/running_var:0\t          resnet.2.residuals.0.block.0.running_var\n",
      "             ResNet/DilatedBlock_6/Conv1d/weight:0\t               resnet.2.residuals.0.block.1.weight\n",
      "        ResNet/DilatedBlock_6/BatchNorm_1/weight:0\t               resnet.2.residuals.0.block.3.weight\n",
      "          ResNet/DilatedBlock_6/BatchNorm_1/bias:0\t                 resnet.2.residuals.0.block.3.bias\n",
      "  ResNet/DilatedBlock_6/BatchNorm_1/running_mean:0\t         resnet.2.residuals.0.block.3.running_mean\n",
      "   ResNet/DilatedBlock_6/BatchNorm_1/running_var:0\t          resnet.2.residuals.0.block.3.running_var\n",
      "           ResNet/DilatedBlock_6/Conv1d_1/weight:0\t               resnet.2.residuals.0.block.5.weight\n",
      "          ResNet/DilatedBlock_7/BatchNorm/weight:0\t               resnet.2.residuals.1.block.0.weight\n",
      "            ResNet/DilatedBlock_7/BatchNorm/bias:0\t                 resnet.2.residuals.1.block.0.bias\n",
      "    ResNet/DilatedBlock_7/BatchNorm/running_mean:0\t         resnet.2.residuals.1.block.0.running_mean\n",
      "     ResNet/DilatedBlock_7/BatchNorm/running_var:0\t          resnet.2.residuals.1.block.0.running_var\n",
      "             ResNet/DilatedBlock_7/Conv1d/weight:0\t               resnet.2.residuals.1.block.1.weight\n",
      "        ResNet/DilatedBlock_7/BatchNorm_1/weight:0\t               resnet.2.residuals.1.block.3.weight\n",
      "          ResNet/DilatedBlock_7/BatchNorm_1/bias:0\t                 resnet.2.residuals.1.block.3.bias\n",
      "  ResNet/DilatedBlock_7/BatchNorm_1/running_mean:0\t         resnet.2.residuals.1.block.3.running_mean\n",
      "   ResNet/DilatedBlock_7/BatchNorm_1/running_var:0\t          resnet.2.residuals.1.block.3.running_var\n",
      "           ResNet/DilatedBlock_7/Conv1d_1/weight:0\t               resnet.2.residuals.1.block.5.weight\n",
      "          ResNet/DilatedBlock_8/BatchNorm/weight:0\t               resnet.2.residuals.2.block.0.weight\n",
      "            ResNet/DilatedBlock_8/BatchNorm/bias:0\t                 resnet.2.residuals.2.block.0.bias\n",
      "    ResNet/DilatedBlock_8/BatchNorm/running_mean:0\t         resnet.2.residuals.2.block.0.running_mean\n",
      "     ResNet/DilatedBlock_8/BatchNorm/running_var:0\t          resnet.2.residuals.2.block.0.running_var\n",
      "             ResNet/DilatedBlock_8/Conv1d/weight:0\t               resnet.2.residuals.2.block.1.weight\n",
      "        ResNet/DilatedBlock_8/BatchNorm_1/weight:0\t               resnet.2.residuals.2.block.3.weight\n",
      "          ResNet/DilatedBlock_8/BatchNorm_1/bias:0\t                 resnet.2.residuals.2.block.3.bias\n",
      "  ResNet/DilatedBlock_8/BatchNorm_1/running_mean:0\t         resnet.2.residuals.2.block.3.running_mean\n",
      "   ResNet/DilatedBlock_8/BatchNorm_1/running_var:0\t          resnet.2.residuals.2.block.3.running_var\n",
      "           ResNet/DilatedBlock_8/Conv1d_1/weight:0\t               resnet.2.residuals.2.block.5.weight\n",
      "                             Logit/Conv1d/weight:0\t                                      logit.weight\n"
     ]
    }
   ],
   "source": [
    "####################################\n",
    "print('OK')\n",
    "tf.reset_default_graph()\n",
    "dilation = 2\n",
    "x = tf.constant(testinput.transpose(0,2,1), 1)\n",
    "#y1 = selu(x)\n",
    "#y1 = Conv1d(x, 1, 1, 3, 2, bias=False)\n",
    "#y1 = DilatedBlock(x, channel=channel, kernel_size=9, dilation=dilation)\n",
    "#pytorch_testmodule = dm.DilatedBlock(in_channels=channel, \n",
    "#                                     out_channels=channel, kernel_size=9, nonlin=dm.SELU(), dilation=dilation)\n",
    "\n",
    "y1 = NET(x, init_channel=32)\n",
    "with tf.Session() as sess:\n",
    "    tf.global_variables_initializer().run()\n",
    "    for f, (p_names, p) in zip(tf.global_variables(), pytorch_testmodule.state_dict().items()):\n",
    "        print('%50s\\t%50s'%(f.name, p_names))\n",
    "        #print(p_names)\n",
    "        #initializer = tf.constant(p.transpose(0, -1).numpy(), 1)\n",
    "        #f.assign(initializer).eval()\n",
    "        #print(initializer.eval()-f.eval())\n",
    "    tf_result = sess.run(y1)#.transpose(0,2,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10, 3) (10, 3)\n",
      "\n",
      "P-T =\n",
      " [[  1.22070312e-04   1.52587891e-05   0.00000000e+00]\n",
      " [  3.81469727e-05   1.37329102e-04   4.67300415e-05]\n",
      " [  1.22070312e-04   7.62939453e-06   6.86645508e-05]\n",
      " [  6.10351562e-05   0.00000000e+00   3.81469727e-06]\n",
      " [  6.10351562e-05   9.91821289e-05   1.14440918e-05]\n",
      " [  1.22070312e-04   1.52587891e-05   2.47955322e-05]\n",
      " [  3.05175781e-05   3.05175781e-05   7.62939453e-06]\n",
      " [  3.05175781e-05   4.57763672e-05   0.00000000e+00]\n",
      " [  3.81469727e-05   8.39233398e-05   2.52723694e-05]\n",
      " [  1.52587891e-05   3.05175781e-05   7.62939453e-06]]\n",
      "\n",
      "mean abs(P-T)/max(P,T) =\n",
      " 1.62921e-07\n"
     ]
    }
   ],
   "source": [
    "print(tf_result.shape, pth_result.shape)\n",
    "#print('\\nT=\\n',tf_result)\n",
    "#print('choice\\n', tf_result.argmax(1))\n",
    "#print('\\nP=\\n',pth_result)\n",
    "#print('choice\\n', pth_result.argmax(1))\n",
    "\n",
    "print('\\nP-T =\\n', np.abs(tf_result-pth_result))\n",
    "print('\\nmean abs(P-T)/max(P,T) =\\n', (np.abs(tf_result-pth_result)/np.max(np.abs([tf_result, pth_result]))).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn import Parameter\n",
    "from torch.autograd import Variable, Function\n",
    "from collections import defaultdict\n",
    "import graphviz\n",
    "old_function__call__ = Function.__call__\n",
    "\n",
    "def register_creator(inputs, creator, output):\n",
    "    \"\"\"\n",
    "    In the forward pass, our Function.__call__ and BatchNorm.forward_hook both call this method to register the creators\n",
    "    inputs: list of input variables\n",
    "    creator: one of\n",
    "        - Function\n",
    "        - BatchNorm module\n",
    "    output: a single output variable\n",
    "    \"\"\"\n",
    "    cid = id(creator)\n",
    "    oid = id(output)\n",
    "    if oid in vars: \n",
    "        return\n",
    "    # connect creator to input\n",
    "    for input in inputs:\n",
    "        iid = id(input)\n",
    "        func_trace[cid][iid] = input\n",
    "        # register input\n",
    "        vars[iid] = input\n",
    "    # connect output to creator\n",
    "    assert type(output) not in [tuple, list, dict]\n",
    "    var_trace[oid][cid] = creator\n",
    "    # register creator and output and all inputs\n",
    "    vars[oid] = output\n",
    "    funcs[cid] = creator\n",
    "\n",
    "hooks = []\n",
    "\n",
    "def register_vis_hooks(model):\n",
    "    global var_trace, func_trace, vars, funcs\n",
    "    remove_vis_hooks()\n",
    "    var_trace  = defaultdict(lambda: {})     # map oid to {cid:creator}\n",
    "    func_trace = defaultdict(lambda: {})     # map cid to {iid:input}\n",
    "    vars  = {}                               # map vid to Variable/Parameter\n",
    "    funcs = {}                               # map cid to Function/BatchNorm module\n",
    "    hooks = []                               # contains the forward hooks, needed for hook removal\n",
    "\n",
    "    def hook_func(module, inputs, output):\n",
    "        print('HOOK', mod.__class__.__name__)\n",
    "        #assert 'BatchNorm' in mod.__class__.__name__        # batchnorms don't have shared superclass\n",
    "        inputs = list(inputs)\n",
    "        try:\n",
    "            for p in [module.weight, module.bias]:\n",
    "                if p is not None:\n",
    "                    inputs.append(p)\n",
    "        except AttributeError:\n",
    "            pass\n",
    "        register_creator(inputs, module, output)\n",
    "\n",
    "    for mod in model.modules():\n",
    "        #if 'BatchNorm' in mod.__class__.__name__:           # batchnorms don't have shared superclass\n",
    "        hook = mod.register_forward_hook(hook_func)\n",
    "        hooks.append(hook)\n",
    "\n",
    "    def new_function__call__(self, *args, **kwargs):\n",
    "        inputs =  [a for a in args            if isinstance(a, Variable)]\n",
    "        inputs += [a for a in kwargs.values() if isinstance(a, Variable)]\n",
    "        output = old_function__call__(self, *args, **kwargs)\n",
    "        register_creator(inputs, self, output)\n",
    "        return output\n",
    "\n",
    "    Function.__call__ = new_function__call__\n",
    "\n",
    "\n",
    "def remove_vis_hooks():\n",
    "    for hook in hooks:\n",
    "        hook.remove()\n",
    "\n",
    "    Function.__call__ = old_function__call__\n",
    "\n",
    "\n",
    "def save_visualization(name, format='svg'):\n",
    "    g = graphviz.Digraph(format=format)\n",
    "    def sizestr(var):\n",
    "        size = [int(i) for i in list(var.size())]\n",
    "        return str(size)\n",
    "    # add variable nodes\n",
    "    for vid, var in vars.items():\n",
    "        if isinstance(var, nn.Parameter):\n",
    "            g.node(str(vid), label=sizestr(var), shape='ellipse', style='filled', fillcolor='red')\n",
    "        elif isinstance(var, Variable):\n",
    "            g.node(str(vid), label=sizestr(var), shape='ellipse', style='filled', fillcolor='lightblue')\n",
    "        else:\n",
    "            assert False, var.__class__\n",
    "    # add creator nodes\n",
    "    for cid in func_trace:\n",
    "        creator = funcs[cid]\n",
    "        g.node(str(cid), label=str(creator.__class__.__name__), shape='rectangle', style='filled', fillcolor='orange')\n",
    "    # add edges between creator and inputs\n",
    "    for cid in func_trace:\n",
    "        for iid in func_trace[cid]:\n",
    "            g.edge(str(iid), str(cid))\n",
    "    # add edges between outputs and creators\n",
    "    for oid in var_trace:\n",
    "        for cid in var_trace[oid]:\n",
    "            g.edge(str(cid), str(oid))\n",
    "    g.render(name)\n",
    "    \n",
    "#register_forward_hook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n"
     ]
    }
   ],
   "source": [
    "register_vis_hooks(pthnet)\n",
    "\n",
    "pth_result = pthnet(Variable(th.FloatTensor(testinput)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "remove_vis_hooks()\n",
    "save_visualization('graphviz', format='png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'sd' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-2d9e1ee5e8fe>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;31m#sd = th.load('saved/ENCODE-selu-adam/0002/state_dict_highscore')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mpthnet\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_state_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msd\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m \u001b[0mpthnet\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'sd' is not defined"
     ]
    }
   ],
   "source": [
    "import dilated_model as dm\n",
    "import torch as th\n",
    "pthnet = dm.EncodeWideResNet(in_channel=1, init_channel=32, num_enc_layer=4, N_res_in_block=1, use_selu=True, num_classes=3)\n",
    "#sd = th.nn.BatchNorm1d(4)\n",
    "#sd.eval()\n",
    "\n",
    "#sd = th.load('saved/ENCODE-selu-adam/0002/state_dict_highscore')\n",
    "pthnet.load_state_dict(sd)\n",
    "pthnet.cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "testinput = np.random.randn(*[3, 1, 100])\n",
    "x = tf.constant(testinput.transpose(0,2,1), 1)\n",
    "y1 = NET(x)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    for f, (p_names, p) in zip(tf.global_variables(), pthnet.state_dict().items()):\n",
    "        #print('%50s\\t%50s'%(f.name, p_names))\n",
    "        #print(p_names)\n",
    "        initializer = tf.constant(p.transpose(0, -1).numpy(), 1)\n",
    "        f.assign(initializer).eval()\n",
    "        #print(initializer.eval()-f.eval())\n",
    "    tf_result = sess.run(y1)\n",
    "    pth_result = pthnet(th.autograd.Variable(th.FloatTensor(testinput))).data.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "            ResNet/DilatedBlock/BatchNorm/weight:0\t                        residuals.0.block.0.weight\n",
      "              ResNet/DilatedBlock/BatchNorm/bias:0\t                          residuals.0.block.0.bias\n",
      "      ResNet/DilatedBlock/BatchNorm/running_mean:0\t                  residuals.0.block.0.running_mean\n",
      "       ResNet/DilatedBlock/BatchNorm/running_var:0\t                   residuals.0.block.0.running_var\n",
      "               ResNet/DilatedBlock/Conv1d/weight:0\t                        residuals.0.block.1.weight\n",
      "          ResNet/DilatedBlock/BatchNorm_1/weight:0\t                        residuals.0.block.3.weight\n",
      "            ResNet/DilatedBlock/BatchNorm_1/bias:0\t                          residuals.0.block.3.bias\n",
      "    ResNet/DilatedBlock/BatchNorm_1/running_mean:0\t                  residuals.0.block.3.running_mean\n",
      "     ResNet/DilatedBlock/BatchNorm_1/running_var:0\t                   residuals.0.block.3.running_var\n",
      "             ResNet/DilatedBlock/Conv1d_1/weight:0\t                        residuals.0.block.5.weight\n",
      "          ResNet/DilatedBlock_1/BatchNorm/weight:0\t                        residuals.1.block.0.weight\n",
      "            ResNet/DilatedBlock_1/BatchNorm/bias:0\t                          residuals.1.block.0.bias\n",
      "    ResNet/DilatedBlock_1/BatchNorm/running_mean:0\t                  residuals.1.block.0.running_mean\n",
      "     ResNet/DilatedBlock_1/BatchNorm/running_var:0\t                   residuals.1.block.0.running_var\n",
      "             ResNet/DilatedBlock_1/Conv1d/weight:0\t                        residuals.1.block.1.weight\n",
      "        ResNet/DilatedBlock_1/BatchNorm_1/weight:0\t                        residuals.1.block.3.weight\n",
      "          ResNet/DilatedBlock_1/BatchNorm_1/bias:0\t                          residuals.1.block.3.bias\n",
      "  ResNet/DilatedBlock_1/BatchNorm_1/running_mean:0\t                  residuals.1.block.3.running_mean\n",
      "   ResNet/DilatedBlock_1/BatchNorm_1/running_var:0\t                   residuals.1.block.3.running_var\n",
      "           ResNet/DilatedBlock_1/Conv1d_1/weight:0\t                        residuals.1.block.5.weight\n",
      "          ResNet/DilatedBlock_2/BatchNorm/weight:0\t                        residuals.2.block.0.weight\n",
      "            ResNet/DilatedBlock_2/BatchNorm/bias:0\t                          residuals.2.block.0.bias\n",
      "    ResNet/DilatedBlock_2/BatchNorm/running_mean:0\t                  residuals.2.block.0.running_mean\n",
      "     ResNet/DilatedBlock_2/BatchNorm/running_var:0\t                   residuals.2.block.0.running_var\n",
      "             ResNet/DilatedBlock_2/Conv1d/weight:0\t                        residuals.2.block.1.weight\n",
      "        ResNet/DilatedBlock_2/BatchNorm_1/weight:0\t                        residuals.2.block.3.weight\n",
      "          ResNet/DilatedBlock_2/BatchNorm_1/bias:0\t                          residuals.2.block.3.bias\n",
      "  ResNet/DilatedBlock_2/BatchNorm_1/running_mean:0\t                  residuals.2.block.3.running_mean\n",
      "   ResNet/DilatedBlock_2/BatchNorm_1/running_var:0\t                   residuals.2.block.3.running_var\n",
      "           ResNet/DilatedBlock_2/Conv1d_1/weight:0\t                        residuals.2.block.5.weight\n",
      "          ResNet/DilatedBlock_3/BatchNorm/weight:0\t                        residuals.3.block.0.weight\n",
      "            ResNet/DilatedBlock_3/BatchNorm/bias:0\t                          residuals.3.block.0.bias\n",
      "    ResNet/DilatedBlock_3/BatchNorm/running_mean:0\t                  residuals.3.block.0.running_mean\n",
      "     ResNet/DilatedBlock_3/BatchNorm/running_var:0\t                   residuals.3.block.0.running_var\n",
      "             ResNet/DilatedBlock_3/Conv1d/weight:0\t                        residuals.3.block.1.weight\n",
      "        ResNet/DilatedBlock_3/BatchNorm_1/weight:0\t                        residuals.3.block.3.weight\n",
      "          ResNet/DilatedBlock_3/BatchNorm_1/bias:0\t                          residuals.3.block.3.bias\n",
      "  ResNet/DilatedBlock_3/BatchNorm_1/running_mean:0\t                  residuals.3.block.3.running_mean\n",
      "   ResNet/DilatedBlock_3/BatchNorm_1/running_var:0\t                   residuals.3.block.3.running_var\n",
      "           ResNet/DilatedBlock_3/Conv1d_1/weight:0\t                        residuals.3.block.5.weight\n",
      "          ResNet/DilatedBlock_4/BatchNorm/weight:0\t                        residuals.4.block.0.weight\n",
      "            ResNet/DilatedBlock_4/BatchNorm/bias:0\t                          residuals.4.block.0.bias\n",
      "    ResNet/DilatedBlock_4/BatchNorm/running_mean:0\t                  residuals.4.block.0.running_mean\n",
      "     ResNet/DilatedBlock_4/BatchNorm/running_var:0\t                   residuals.4.block.0.running_var\n",
      "             ResNet/DilatedBlock_4/Conv1d/weight:0\t                        residuals.4.block.1.weight\n",
      "        ResNet/DilatedBlock_4/BatchNorm_1/weight:0\t                        residuals.4.block.3.weight\n",
      "          ResNet/DilatedBlock_4/BatchNorm_1/bias:0\t                          residuals.4.block.3.bias\n",
      "  ResNet/DilatedBlock_4/BatchNorm_1/running_mean:0\t                  residuals.4.block.3.running_mean\n",
      "   ResNet/DilatedBlock_4/BatchNorm_1/running_var:0\t                   residuals.4.block.3.running_var\n",
      "           ResNet/DilatedBlock_4/Conv1d_1/weight:0\t                        residuals.4.block.5.weight\n",
      "          ResNet/DilatedBlock_5/BatchNorm/weight:0\t                        residuals.5.block.0.weight\n",
      "            ResNet/DilatedBlock_5/BatchNorm/bias:0\t                          residuals.5.block.0.bias\n",
      "    ResNet/DilatedBlock_5/BatchNorm/running_mean:0\t                  residuals.5.block.0.running_mean\n",
      "     ResNet/DilatedBlock_5/BatchNorm/running_var:0\t                   residuals.5.block.0.running_var\n",
      "             ResNet/DilatedBlock_5/Conv1d/weight:0\t                        residuals.5.block.1.weight\n",
      "        ResNet/DilatedBlock_5/BatchNorm_1/weight:0\t                        residuals.5.block.3.weight\n",
      "          ResNet/DilatedBlock_5/BatchNorm_1/bias:0\t                          residuals.5.block.3.bias\n",
      "  ResNet/DilatedBlock_5/BatchNorm_1/running_mean:0\t                  residuals.5.block.3.running_mean\n",
      "   ResNet/DilatedBlock_5/BatchNorm_1/running_var:0\t                   residuals.5.block.3.running_var\n",
      "           ResNet/DilatedBlock_5/Conv1d_1/weight:0\t                        residuals.5.block.5.weight\n",
      "          ResNet/DilatedBlock_6/BatchNorm/weight:0\t                        residuals.6.block.0.weight\n",
      "            ResNet/DilatedBlock_6/BatchNorm/bias:0\t                          residuals.6.block.0.bias\n",
      "    ResNet/DilatedBlock_6/BatchNorm/running_mean:0\t                  residuals.6.block.0.running_mean\n",
      "     ResNet/DilatedBlock_6/BatchNorm/running_var:0\t                   residuals.6.block.0.running_var\n",
      "             ResNet/DilatedBlock_6/Conv1d/weight:0\t                        residuals.6.block.1.weight\n",
      "        ResNet/DilatedBlock_6/BatchNorm_1/weight:0\t                        residuals.6.block.3.weight\n",
      "          ResNet/DilatedBlock_6/BatchNorm_1/bias:0\t                          residuals.6.block.3.bias\n",
      "  ResNet/DilatedBlock_6/BatchNorm_1/running_mean:0\t                  residuals.6.block.3.running_mean\n",
      "   ResNet/DilatedBlock_6/BatchNorm_1/running_var:0\t                   residuals.6.block.3.running_var\n",
      "           ResNet/DilatedBlock_6/Conv1d_1/weight:0\t                        residuals.6.block.5.weight\n",
      "          ResNet/DilatedBlock_7/BatchNorm/weight:0\t                        residuals.7.block.0.weight\n",
      "            ResNet/DilatedBlock_7/BatchNorm/bias:0\t                          residuals.7.block.0.bias\n",
      "    ResNet/DilatedBlock_7/BatchNorm/running_mean:0\t                  residuals.7.block.0.running_mean\n",
      "     ResNet/DilatedBlock_7/BatchNorm/running_var:0\t                   residuals.7.block.0.running_var\n",
      "             ResNet/DilatedBlock_7/Conv1d/weight:0\t                        residuals.7.block.1.weight\n",
      "        ResNet/DilatedBlock_7/BatchNorm_1/weight:0\t                        residuals.7.block.3.weight\n",
      "          ResNet/DilatedBlock_7/BatchNorm_1/bias:0\t                          residuals.7.block.3.bias\n",
      "  ResNet/DilatedBlock_7/BatchNorm_1/running_mean:0\t                  residuals.7.block.3.running_mean\n",
      "   ResNet/DilatedBlock_7/BatchNorm_1/running_var:0\t                   residuals.7.block.3.running_var\n",
      "           ResNet/DilatedBlock_7/Conv1d_1/weight:0\t                        residuals.7.block.5.weight\n",
      "          ResNet/DilatedBlock_8/BatchNorm/weight:0\t                        residuals.8.block.0.weight\n",
      "            ResNet/DilatedBlock_8/BatchNorm/bias:0\t                          residuals.8.block.0.bias\n",
      "    ResNet/DilatedBlock_8/BatchNorm/running_mean:0\t                  residuals.8.block.0.running_mean\n",
      "     ResNet/DilatedBlock_8/BatchNorm/running_var:0\t                   residuals.8.block.0.running_var\n",
      "             ResNet/DilatedBlock_8/Conv1d/weight:0\t                        residuals.8.block.1.weight\n",
      "        ResNet/DilatedBlock_8/BatchNorm_1/weight:0\t                        residuals.8.block.3.weight\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          ResNet/DilatedBlock_8/BatchNorm_1/bias:0\t                          residuals.8.block.3.bias\n",
      "  ResNet/DilatedBlock_8/BatchNorm_1/running_mean:0\t                  residuals.8.block.3.running_mean\n",
      "   ResNet/DilatedBlock_8/BatchNorm_1/running_var:0\t                   residuals.8.block.3.running_var\n",
      "           ResNet/DilatedBlock_8/Conv1d_1/weight:0\t                        residuals.8.block.5.weight\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "channel = 30\n",
    "dilation = 2\n",
    "testinput = np.random.randn(*[2, channel, 1000])\n",
    "x = tf.constant(testinput.transpose(0,2,1), 1)\n",
    "#y1 = selu(x)\n",
    "#y1 = Conv1d(x, 1, 1, 3, 2, bias=False)\n",
    "#y1 = DilatedBlock(x, channel=channel, kernel_size=9, dilation=dilation)\n",
    "#pytorch_testmodule = dm.DilatedBlock(in_channels=channel, \n",
    "#                                     out_channels=channel, kernel_size=9, nonlin=dm.SELU(), dilation=dilation)\n",
    "y1 = ResNet(x, channel=channel)\n",
    "pytorch_testmodule = dm.ConvModule(nonlin=dm.SELU(),\n",
    "    in_channel=channel, channel=channel, kernel_size=9, N=9)\n",
    "pytorch_testmodule.eval()\n",
    "with tf.Session() as sess:\n",
    "    for f, (p_names, p) in zip(tf.global_variables(), pytorch_testmodule.state_dict().items()):\n",
    "        print('%50s\\t%50s'%(f.name, p_names))\n",
    "        #print(p_names)\n",
    "        initializer = tf.constant(p.transpose(0, -1).numpy(), 1)\n",
    "        f.assign(initializer).eval()\n",
    "        #print(initializer.eval()-f.eval())\n",
    "    tf_result = sess.run(y1)#.transpose(0,2,1)\n",
    "    pth_result = pytorch_testmodule(th.autograd.Variable(th.FloatTensor(testinput))).data.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "           Encoder/DownSampleBlock/Conv1d/weight:0\t                                          0.weight\n",
      "             Encoder/DownSampleBlock/Conv1d/bias:0\t                                            0.bias\n",
      "        Encoder/DownSampleBlock/BatchNorm/weight:0\t                                          1.weight\n",
      "          Encoder/DownSampleBlock/BatchNorm/bias:0\t                                            1.bias\n",
      "  Encoder/DownSampleBlock/BatchNorm/running_mean:0\t                                    1.running_mean\n",
      "   Encoder/DownSampleBlock/BatchNorm/running_var:0\t                                     1.running_var\n",
      "         Encoder/DownSampleBlock_1/Conv1d/weight:0\t                                          4.weight\n",
      "           Encoder/DownSampleBlock_1/Conv1d/bias:0\t                                            4.bias\n",
      "      Encoder/DownSampleBlock_1/BatchNorm/weight:0\t                                          5.weight\n",
      "        Encoder/DownSampleBlock_1/BatchNorm/bias:0\t                                            5.bias\n",
      "Encoder/DownSampleBlock_1/BatchNorm/running_mean:0\t                                    5.running_mean\n",
      " Encoder/DownSampleBlock_1/BatchNorm/running_var:0\t                                     5.running_var\n",
      "         Encoder/DownSampleBlock_2/Conv1d/weight:0\t                                          8.weight\n",
      "           Encoder/DownSampleBlock_2/Conv1d/bias:0\t                                            8.bias\n",
      "      Encoder/DownSampleBlock_2/BatchNorm/weight:0\t                                          9.weight\n",
      "        Encoder/DownSampleBlock_2/BatchNorm/bias:0\t                                            9.bias\n",
      "Encoder/DownSampleBlock_2/BatchNorm/running_mean:0\t                                    9.running_mean\n",
      " Encoder/DownSampleBlock_2/BatchNorm/running_var:0\t                                     9.running_var\n",
      "         Encoder/DownSampleBlock_3/Conv1d/weight:0\t                                         12.weight\n",
      "           Encoder/DownSampleBlock_3/Conv1d/bias:0\t                                           12.bias\n",
      "      Encoder/DownSampleBlock_3/BatchNorm/weight:0\t                                         13.weight\n",
      "        Encoder/DownSampleBlock_3/BatchNorm/bias:0\t                                           13.bias\n",
      "Encoder/DownSampleBlock_3/BatchNorm/running_mean:0\t                                   13.running_mean\n",
      " Encoder/DownSampleBlock_3/BatchNorm/running_var:0\t                                    13.running_var\n"
     ]
    }
   ],
   "source": [
    "pthnet = dm.EncodeWideResNet(in_channel=1, init_channel=32, num_enc_layer=4, N_res_in_block=1, use_selu=True, num_classes=3)\n",
    "#sd = th.load('saved/ENCODE-selu-adam/0002/state_dict_highscore')\n",
    "#pthnet.load_state_dict(sd)\n",
    "\n",
    "####################################\n",
    "\n",
    "tf.reset_default_graph()\n",
    "channel = 1\n",
    "dilation = 2\n",
    "testinput = np.random.randn(*[2, channel, 16])\n",
    "x = tf.constant(testinput.transpose(0,2,1), 1)\n",
    "#y1 = selu(x)\n",
    "#y1 = Conv1d(x, 1, 1, 3, 2, bias=False)\n",
    "#y1 = DilatedBlock(x, channel=channel, kernel_size=9, dilation=dilation)\n",
    "#pytorch_testmodule = dm.DilatedBlock(in_channels=channel, \n",
    "#                                     out_channels=channel, kernel_size=9, nonlin=dm.SELU(), dilation=dilation)\n",
    "\n",
    "y1 = Encoder(x, init_channel=32)\n",
    "pytorch_testmodule = pthnet.encoder\n",
    "pytorch_testmodule.eval()\n",
    "with tf.Session() as sess:\n",
    "    for f, (p_names, p) in zip(tf.global_variables(), pytorch_testmodule.state_dict().items()):\n",
    "        print('%50s\\t%50s'%(f.name, p_names))\n",
    "        #print(p_names)\n",
    "        initializer = tf.constant(p.transpose(0, -1).numpy(), 1)\n",
    "        f.assign(initializer).eval()\n",
    "        #print(initializer.eval()-f.eval())\n",
    "    tf_result = sess.run(y1).transpose(0,2,1)\n",
    "    pth_result = pytorch_testmodule(th.autograd.Variable(th.FloatTensor(testinput))).data.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2, 64, 3) (2, 3)\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "operands could not be broadcast together with shapes (2,64,3) (2,3) ",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-14-6ca7775e0247>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;31m#print('\\nP-T =\\n', np.abs(tf_result-pth_result))\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'\\nmean abs(P-T)/max(P,T) =\\n'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mabs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf_result\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mpth_result\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtf_result\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpth_result\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m: operands could not be broadcast together with shapes (2,64,3) (2,3) "
     ]
    }
   ],
   "source": [
    "print(tf_result.shape, pth_result.shape)\n",
    "#print('\\nT=\\n',tf_result)\n",
    "#print('choice\\n', tf_result.argmax(1))\n",
    "#print('\\nP=\\n',pth_result)\n",
    "#print('choice\\n', pth_result.argmax(1))\n",
    "\n",
    "#print('\\nP-T =\\n', np.abs(tf_result-pth_result))\n",
    "print('\\nmean abs(P-T)/max(P,T) =\\n', (np.abs(tf_result-pth_result)/np.max([tf_result, pth_result])).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10, 4)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "summary_writer = tf.summary.FileWriter('/tmp/model/', graph=tf.get_default_graph())\n",
    "with tf.Session() as sess:\n",
    "    tf.global_variables_initializer().run()\n",
    "    print(y1.eval({x:np.ones([10, 256, 1])}).shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DilatedBlock (\n",
       "  (nonlin): SELU (\n",
       "  )\n",
       "  (block): Sequential (\n",
       "    (0): BatchNorm1d(8, eps=1e-05, momentum=0.1, affine=True)\n",
       "    (1): Conv1d(8, 8, kernel_size=(9,), stride=(1,), padding=(4,), bias=False)\n",
       "    (2): SELU (\n",
       "    )\n",
       "    (3): BatchNorm1d(8, eps=1e-05, momentum=0.1, affine=True)\n",
       "    (4): Dropout (p = 0.5, inplace)\n",
       "    (5): Conv1d(8, 8, kernel_size=(9,), stride=(1,), padding=(8,), dilation=(2,), bias=False)\n",
       "    (6): SELU (\n",
       "    )\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 256,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dm.DilatedBlock(nonlin=dm.SELU())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.Session() as sess:\n",
    "    tf.global_variables_initializer().run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.]],\n",
       "\n",
       "       [[ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.]],\n",
       "\n",
       "       [[ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.]],\n",
       "\n",
       "       [[ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.]],\n",
       "\n",
       "       [[ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.]],\n",
       "\n",
       "       [[ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.]],\n",
       "\n",
       "       [[ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.]],\n",
       "\n",
       "       [[ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.]],\n",
       "\n",
       "       [[ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.]],\n",
       "\n",
       "       [[ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.]]], dtype=float32)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(7, 1, 32)\n",
      "(32,)\n",
      "(32,)\n",
      "(32,)\n",
      "(32,)\n",
      "(32,)\n",
      "(7, 32, 64)\n",
      "(64,)\n",
      "(64,)\n",
      "(64,)\n",
      "(64,)\n",
      "(64,)\n",
      "(7, 64, 128)\n",
      "(128,)\n",
      "(128,)\n",
      "(128,)\n",
      "(128,)\n",
      "(128,)\n",
      "(7, 128, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(1, 256, 4)\n"
     ]
    }
   ],
   "source": [
    "for i, a in enumerate(tf.global_variables()):\n",
    "    print(a.get_shape())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a.assign()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "115"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tf.global_variables())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "res = sd(th.autograd.Variable(th.FloatTensor(10, 1, 1000)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       "    0\n",
       "    2\n",
       "    0\n",
       "    0\n",
       "    0\n",
       "    0\n",
       "    2\n",
       "    0\n",
       "    0\n",
       "    2\n",
       "[torch.LongTensor of size 10x1]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res.max(1)[1][:, None]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([32, 1, 7])\n",
      "torch.Size([32])\n",
      "torch.Size([32])\n",
      "torch.Size([32])\n",
      "torch.Size([32])\n",
      "torch.Size([32])\n",
      "torch.Size([64, 32, 7])\n",
      "torch.Size([64])\n",
      "torch.Size([64])\n",
      "torch.Size([64])\n",
      "torch.Size([64])\n",
      "torch.Size([64])\n",
      "torch.Size([128, 64, 7])\n",
      "torch.Size([128])\n",
      "torch.Size([128])\n",
      "torch.Size([128])\n",
      "torch.Size([128])\n",
      "torch.Size([128])\n",
      "torch.Size([256, 128, 7])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([3, 256, 1])\n"
     ]
    }
   ],
   "source": [
    "for k, v in sd.state_dict().items():\n",
    "    print(v.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a = nn.Conv1d(3, 4, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('weight', \n",
       "              (0 ,.,.) = \n",
       "               -0.1830  0.1597  0.1484 -0.1916  0.2417\n",
       "               -0.1660  0.0768 -0.1457  0.1034 -0.2297\n",
       "               -0.1860  0.0322  0.0990  0.2096 -0.1983\n",
       "              \n",
       "              (1 ,.,.) = \n",
       "               -0.1521  0.1666 -0.0709  0.0931  0.1345\n",
       "                0.1034 -0.0586  0.1383 -0.1212 -0.1902\n",
       "               -0.2440 -0.0988 -0.0854  0.1089  0.1279\n",
       "              \n",
       "              (2 ,.,.) = \n",
       "                0.0536  0.2212  0.0824  0.0108  0.2315\n",
       "               -0.2318  0.0847 -0.2136 -0.0735  0.2240\n",
       "                0.2383  0.0270  0.2266 -0.1901 -0.0234\n",
       "              \n",
       "              (3 ,.,.) = \n",
       "                0.2279  0.2184  0.0534 -0.0578  0.1607\n",
       "                0.1085  0.1057  0.0550 -0.0299  0.0537\n",
       "                0.0227  0.1641 -0.1961  0.0515 -0.1527\n",
       "              [torch.FloatTensor of size 4x3x5]), ('bias', \n",
       "               0.1855\n",
       "               0.2336\n",
       "              -0.0204\n",
       "              -0.0937\n",
       "              [torch.FloatTensor of size 4])])"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.state_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}