PTH2TF-combined.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import dilated_model as dm\n",
    "import data_handler\n",
    "import torch as th\n",
    "import combined_wrapper as cw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "timeNet = dm.EncodeWideResNetFIXED(in_channel=1, init_channel=16, \n",
    "    num_enc_layer=4, N_res_in_block=1, use_selu=True)\n",
    "#timeNet.load_state_dict(th.load('time-params.pt'))\n",
    "\n",
    "freqNet = dm.SkipFCN(in_channel=16, use_selu=True,\n",
    "    channels=[16,16,  32,32,  64,64,64,  128,128,128,  128,128,128])\n",
    "#freqNet.load_state_dict(th.load('freq-params.pt'))\n",
    "\n",
    "classifier = th.nn.Sequential(th.nn.BatchNorm1d(256), dm.SELU(), th.nn.Conv1d(256, 2, 1))\n",
    "#classifier = th.nn.Conv1d(256, 3, 1)\n",
    "pthnet = dm.CombinedTransform(\n",
    "#    pretrained=True,\n",
    "    pretrained=False,\n",
    "    feature_length=20,\n",
    "    #time=timeNet,freq=freqNet,\n",
    "    freq=freqNet,time=timeNet,\n",
    "    #freq=timeNet,time=freqNet,\n",
    "    classifier=classifier)\n",
    "pthnet.eval()\n",
    "freqNet.eval()\n",
    "timeNet.eval()\n",
    "pthsd_filename = 'dry/saved/noise-TimeANDFreq-80-shallow/0001/state_dict_highscore'\n",
    "pthsd = th.load(pthsd_filename)\n",
    "#pthsd = th.load('combined.pt')\n",
    "pthnet.load_state_dict(pthsd)\n",
    "#th.save(pthnet.state_dict(), 'combined.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def forward_features(self, x):\n",
    "\n",
    "        out = self.activation(self.bn1(self.conv1(x)))\n",
    "        out = self.activation(self.bn2(self.conv2(out)))\n",
    "        out = self.pool(out)\n",
    "        x = out\n",
    "        out = self.activation(self.bn3(self.conv3(out)))\n",
    "        out = self.activation(self.bn4(self.conv4(out)))\n",
    "        out = self.pool(out)\n",
    "        x = self.pool(x)\n",
    "        out = th.cat([x, out], dim=1)\n",
    "        out = self.activation(self.bn5(self.conv5(out)))\n",
    "        out = self.activation(self.bn6(self.conv6(out)))\n",
    "        out = self.activation(self.bn7(self.conv7(out)))\n",
    "        out = self.pool(out)\n",
    "        x = self.pool(x)\n",
    "        out = th.cat([x, out], dim=1)\n",
    "        out = self.activation(self.bn8(self.conv8(out)))\n",
    "        out = self.activation(self.bn9(self.conv9(out)))\n",
    "        out = self.activation(self.bn10(self.conv10(out)))\n",
    "        out = self.pool(out)\n",
    "        x = self.pool(x)\n",
    "        out = th.cat([x, out], dim=1)\n",
    "        out = self.activation(self.bn11(self.conv11(out)))\n",
    "        out = self.activation(self.bn12(self.conv12(out)))\n",
    "        out = self.activation(self.bn13(self.conv13(out)))\n",
    "        out = self.drop1(out)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time size (2, 128, 375)\n",
      "Freq size (2, 128, 375)\n"
     ]
    }
   ],
   "source": [
    "global_transforms = [\n",
    "    data_handler.Crop(6000),\n",
    "]\n",
    "\n",
    "transTime = [\n",
    "    data_handler.Threshold(sigma=2.2),\n",
    "    data_handler.RandomMultiplier(-1),\n",
    "]\n",
    "\n",
    "transFreq = [\n",
    "    data_handler.RandomMultiplier(-1),\n",
    "    data_handler.Spectogram(31),\n",
    "    #data_handler.Logarithm()\n",
    "]\n",
    "\n",
    "dataset = data_handler.DataSet(\n",
    "    'data/raw/training2017/REFERENCE.csv', data_handler.load_forked,\n",
    "    global_transforms=global_transforms,\n",
    "    fork_transforms={'time':transTime, 'freq':transFreq},\n",
    "    path='data/raw/training2017/',\n",
    "    #remove_unlisted=False, tokens='NAO', remove_noise=True)\n",
    "    remove_noise=False, tokens=data_handler.noise_tokens)\n",
    "train_set, eval_set = dataset.disjunct_split(.80)\n",
    "\n",
    "train_producer = th.utils.data.DataLoader(\n",
    "    dataset=dataset, batch_size=2, shuffle=True,\n",
    "    num_workers=0, collate_fn=data_handler.batchify_forked)\n",
    "pth_input = next(iter(train_producer))['x']\n",
    "freqNet.load_state_dict(pthnet.models[0].state_dict())\n",
    "timeNet.load_state_dict(pthnet.models[1].state_dict())\n",
    "pytorch_testmodule = timeNet\n",
    "pytorch_testmodule.cpu()\n",
    "pytorch_testmodule.eval()\n",
    "pth_result_time = timeNet.forward_features(pth_input['time']).data.numpy()\n",
    "pth_result_freq = forward_features(freqNet, pth_input['freq']).data.numpy()\n",
    "#pth_result_ = pthnet.forward_features(pth_input).data.numpy()\n",
    "pth_result_ = pthnet(pth_input).data.numpy()\n",
    "timetestinput = pth_input['time'].data.numpy()\n",
    "freqtestinput = pth_input['freq'].data.numpy()\n",
    "print('Time size', pth_result_time.shape)\n",
    "print('Freq size', pth_result_freq.shape)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "for k, v in pthsd.items():\n",
    "    print(k, list(v.size()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OK\n"
     ]
    }
   ],
   "source": [
    "####################################\n",
    "print('OK')\n",
    "tf.reset_default_graph()\n",
    "dilation = 2\n",
    "xtime = tf.constant(timetestinput.transpose(0,2,1), 1)\n",
    "xfreq = tf.constant(freqtestinput.transpose(0,2,1), 1)\n",
    "#y1 = selu(x)\n",
    "#y1 = Conv1d(x, 1, 1, 3, 2, bias=False)\n",
    "#y1 = DilatedBlock(x, channel=channel, kernel_size=9, dilation=dilation)\n",
    "#pytorch_testmodule = dm.DilatedBlock(in_channels=channel, \n",
    "#                                     out_channels=channel, kernel_size=9, nonlin=dm.SELU(), dilation=dilation)\n",
    "\n",
    "#FF, TF = cw.get_logits(xtime, xfreq, pthsd_filename, testfeatures=True, noGlobalAvg=False)\n",
    "y = cw.get_logits(xtime, xfreq, pthsd_filename, num_classes=2)\n",
    "with tf.Session() as sess:\n",
    "    tf.global_variables_initializer().run()\n",
    "    #for f, (p_names, p) in zip(tf.global_variables(), pytorch_testmodule.state_dict().items()):\n",
    "        #print('%50s\\t%50s'%(f.name, p_names))\n",
    "        #print(p_names)\n",
    "        #initializer = tf.constant(p.transpose(0, -1).numpy(), 1)\n",
    "        #f.assign(initializer).eval()\n",
    "        #print(initializer.eval()-f.eval())\n",
    "    #tf_result_time, tf_result_freq = sess.run([TF, FF])    \n",
    "    #tf_result_ = sess.run(tf.concat([FF, TF], -1))#.transpose(0,2,1)\n",
    "    tf_result_ = sess.run(y)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2, 2) (2, 2)\n",
      "\n",
      "P-T =\n",
      " 1.3262e-06\n",
      "\n",
      "mean abs(P-T)/max(P,T) =\n",
      " 2.50602e-07\n"
     ]
    }
   ],
   "source": [
    "tf_result = tf_result_#.transpose(0,2,1)[..., :]\n",
    "pth_result = pth_result_\n",
    "print(tf_result.shape, pth_result.shape)\n",
    "#print('\\nT=\\n',tf_result)\n",
    "#print('choice\\n', tf_result.argmax(1))\n",
    "#print('\\nP=\\n',pth_result)\n",
    "#print('choice\\n', pth_result.argmax(1))\n",
    "\n",
    "#print('\\nP/T =\\n', np.abs(tf_result/pth_result))\n",
    "print('\\nP-T =\\n', (np.abs(tf_result-pth_result).mean()))\n",
    "print('\\nmean abs(P-T)/max(P,T) =\\n', (np.abs(tf_result-pth_result)/np.max(np.abs([tf_result, pth_result]))).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn import Parameter\n",
    "from torch.autograd import Variable, Function\n",
    "from collections import defaultdict\n",
    "import graphviz\n",
    "old_function__call__ = Function.__call__\n",
    "\n",
    "def register_creator(inputs, creator, output):\n",
    "    \"\"\"\n",
    "    In the forward pass, our Function.__call__ and BatchNorm.forward_hook both call this method to register the creators\n",
    "    inputs: list of input variables\n",
    "    creator: one of\n",
    "        - Function\n",
    "        - BatchNorm module\n",
    "    output: a single output variable\n",
    "    \"\"\"\n",
    "    cid = id(creator)\n",
    "    oid = id(output)\n",
    "    if oid in vars: \n",
    "        return\n",
    "    # connect creator to input\n",
    "    for input in inputs:\n",
    "        iid = id(input)\n",
    "        func_trace[cid][iid] = input\n",
    "        # register input\n",
    "        vars[iid] = input\n",
    "    # connect output to creator\n",
    "    assert type(output) not in [tuple, list, dict]\n",
    "    var_trace[oid][cid] = creator\n",
    "    # register creator and output and all inputs\n",
    "    vars[oid] = output\n",
    "    funcs[cid] = creator\n",
    "\n",
    "hooks = []\n",
    "\n",
    "def register_vis_hooks(model):\n",
    "    global var_trace, func_trace, vars, funcs\n",
    "    remove_vis_hooks()\n",
    "    var_trace  = defaultdict(lambda: {})     # map oid to {cid:creator}\n",
    "    func_trace = defaultdict(lambda: {})     # map cid to {iid:input}\n",
    "    vars  = {}                               # map vid to Variable/Parameter\n",
    "    funcs = {}                               # map cid to Function/BatchNorm module\n",
    "    hooks = []                               # contains the forward hooks, needed for hook removal\n",
    "\n",
    "    def hook_func(module, inputs, output):\n",
    "        print('HOOK', mod.__class__.__name__)\n",
    "        #assert 'BatchNorm' in mod.__class__.__name__        # batchnorms don't have shared superclass\n",
    "        inputs = list(inputs)\n",
    "        try:\n",
    "            for p in [module.weight, module.bias]:\n",
    "                if p is not None:\n",
    "                    inputs.append(p)\n",
    "        except AttributeError:\n",
    "            pass\n",
    "        register_creator(inputs, module, output)\n",
    "\n",
    "    for mod in model.modules():\n",
    "        #if 'BatchNorm' in mod.__class__.__name__:           # batchnorms don't have shared superclass\n",
    "        hook = mod.register_forward_hook(hook_func)\n",
    "        hooks.append(hook)\n",
    "\n",
    "    def new_function__call__(self, *args, **kwargs):\n",
    "        inputs =  [a for a in args            if isinstance(a, Variable)]\n",
    "        inputs += [a for a in kwargs.values() if isinstance(a, Variable)]\n",
    "        output = old_function__call__(self, *args, **kwargs)\n",
    "        register_creator(inputs, self, output)\n",
    "        return output\n",
    "\n",
    "    Function.__call__ = new_function__call__\n",
    "\n",
    "\n",
    "def remove_vis_hooks():\n",
    "    for hook in hooks:\n",
    "        hook.remove()\n",
    "\n",
    "    Function.__call__ = old_function__call__\n",
    "\n",
    "\n",
    "def save_visualization(name, format='svg'):\n",
    "    g = graphviz.Digraph(format=format)\n",
    "    def sizestr(var):\n",
    "        size = [int(i) for i in list(var.size())]\n",
    "        return str(size)\n",
    "    # add variable nodes\n",
    "    for vid, var in vars.items():\n",
    "        if isinstance(var, nn.Parameter):\n",
    "            g.node(str(vid), label=sizestr(var), shape='ellipse', style='filled', fillcolor='red')\n",
    "        elif isinstance(var, Variable):\n",
    "            g.node(str(vid), label=sizestr(var), shape='ellipse', style='filled', fillcolor='lightblue')\n",
    "        else:\n",
    "            assert False, var.__class__\n",
    "    # add creator nodes\n",
    "    for cid in func_trace:\n",
    "        creator = funcs[cid]\n",
    "        g.node(str(cid), label=str(creator.__class__.__name__), shape='rectangle', style='filled', fillcolor='orange')\n",
    "    # add edges between creator and inputs\n",
    "    for cid in func_trace:\n",
    "        for iid in func_trace[cid]:\n",
    "            g.edge(str(iid), str(cid))\n",
    "    # add edges between outputs and creators\n",
    "    for oid in var_trace:\n",
    "        for cid in var_trace[oid]:\n",
    "            g.edge(str(cid), str(oid))\n",
    "    g.render(name)\n",
    "    \n",
    "#register_forward_hook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n",
      "HOOK Conv1d\n"
     ]
    }
   ],
   "source": [
    "register_vis_hooks(pthnet)\n",
    "\n",
    "pth_result = pthnet(Variable(th.FloatTensor(testinput)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "remove_vis_hooks()\n",
    "save_visualization('graphviz', format='png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'sd' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-2d9e1ee5e8fe>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;31m#sd = th.load('saved/ENCODE-selu-adam/0002/state_dict_highscore')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mpthnet\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_state_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msd\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m \u001b[0mpthnet\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'sd' is not defined"
     ]
    }
   ],
   "source": [
    "import dilated_model as dm\n",
    "import torch as th\n",
    "pthnet = dm.EncodeWideResNet(in_channel=1, init_channel=32, num_enc_layer=4, N_res_in_block=1, use_selu=True, num_classes=3)\n",
    "#sd = th.nn.BatchNorm1d(4)\n",
    "#sd.eval()\n",
    "\n",
    "#sd = th.load('saved/ENCODE-selu-adam/0002/state_dict_highscore')\n",
    "pthnet.load_state_dict(sd)\n",
    "pthnet.cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "testinput = np.random.randn(*[3, 1, 100])\n",
    "x = tf.constant(testinput.transpose(0,2,1), 1)\n",
    "y1 = NET(x)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    for f, (p_names, p) in zip(tf.global_variables(), pthnet.state_dict().items()):\n",
    "        #print('%50s\\t%50s'%(f.name, p_names))\n",
    "        #print(p_names)\n",
    "        initializer = tf.constant(p.transpose(0, -1).numpy(), 1)\n",
    "        f.assign(initializer).eval()\n",
    "        #print(initializer.eval()-f.eval())\n",
    "    tf_result = sess.run(y1)\n",
    "    pth_result = pthnet(th.autograd.Variable(th.FloatTensor(testinput))).data.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "            ResNet/DilatedBlock/BatchNorm/weight:0\t                        residuals.0.block.0.weight\n",
      "              ResNet/DilatedBlock/BatchNorm/bias:0\t                          residuals.0.block.0.bias\n",
      "      ResNet/DilatedBlock/BatchNorm/running_mean:0\t                  residuals.0.block.0.running_mean\n",
      "       ResNet/DilatedBlock/BatchNorm/running_var:0\t                   residuals.0.block.0.running_var\n",
      "               ResNet/DilatedBlock/Conv1d/weight:0\t                        residuals.0.block.1.weight\n",
      "          ResNet/DilatedBlock/BatchNorm_1/weight:0\t                        residuals.0.block.3.weight\n",
      "            ResNet/DilatedBlock/BatchNorm_1/bias:0\t                          residuals.0.block.3.bias\n",
      "    ResNet/DilatedBlock/BatchNorm_1/running_mean:0\t                  residuals.0.block.3.running_mean\n",
      "     ResNet/DilatedBlock/BatchNorm_1/running_var:0\t                   residuals.0.block.3.running_var\n",
      "             ResNet/DilatedBlock/Conv1d_1/weight:0\t                        residuals.0.block.5.weight\n",
      "          ResNet/DilatedBlock_1/BatchNorm/weight:0\t                        residuals.1.block.0.weight\n",
      "            ResNet/DilatedBlock_1/BatchNorm/bias:0\t                          residuals.1.block.0.bias\n",
      "    ResNet/DilatedBlock_1/BatchNorm/running_mean:0\t                  residuals.1.block.0.running_mean\n",
      "     ResNet/DilatedBlock_1/BatchNorm/running_var:0\t                   residuals.1.block.0.running_var\n",
      "             ResNet/DilatedBlock_1/Conv1d/weight:0\t                        residuals.1.block.1.weight\n",
      "        ResNet/DilatedBlock_1/BatchNorm_1/weight:0\t                        residuals.1.block.3.weight\n",
      "          ResNet/DilatedBlock_1/BatchNorm_1/bias:0\t                          residuals.1.block.3.bias\n",
      "  ResNet/DilatedBlock_1/BatchNorm_1/running_mean:0\t                  residuals.1.block.3.running_mean\n",
      "   ResNet/DilatedBlock_1/BatchNorm_1/running_var:0\t                   residuals.1.block.3.running_var\n",
      "           ResNet/DilatedBlock_1/Conv1d_1/weight:0\t                        residuals.1.block.5.weight\n",
      "          ResNet/DilatedBlock_2/BatchNorm/weight:0\t                        residuals.2.block.0.weight\n",
      "            ResNet/DilatedBlock_2/BatchNorm/bias:0\t                          residuals.2.block.0.bias\n",
      "    ResNet/DilatedBlock_2/BatchNorm/running_mean:0\t                  residuals.2.block.0.running_mean\n",
      "     ResNet/DilatedBlock_2/BatchNorm/running_var:0\t                   residuals.2.block.0.running_var\n",
      "             ResNet/DilatedBlock_2/Conv1d/weight:0\t                        residuals.2.block.1.weight\n",
      "        ResNet/DilatedBlock_2/BatchNorm_1/weight:0\t                        residuals.2.block.3.weight\n",
      "          ResNet/DilatedBlock_2/BatchNorm_1/bias:0\t                          residuals.2.block.3.bias\n",
      "  ResNet/DilatedBlock_2/BatchNorm_1/running_mean:0\t                  residuals.2.block.3.running_mean\n",
      "   ResNet/DilatedBlock_2/BatchNorm_1/running_var:0\t                   residuals.2.block.3.running_var\n",
      "           ResNet/DilatedBlock_2/Conv1d_1/weight:0\t                        residuals.2.block.5.weight\n",
      "          ResNet/DilatedBlock_3/BatchNorm/weight:0\t                        residuals.3.block.0.weight\n",
      "            ResNet/DilatedBlock_3/BatchNorm/bias:0\t                          residuals.3.block.0.bias\n",
      "    ResNet/DilatedBlock_3/BatchNorm/running_mean:0\t                  residuals.3.block.0.running_mean\n",
      "     ResNet/DilatedBlock_3/BatchNorm/running_var:0\t                   residuals.3.block.0.running_var\n",
      "             ResNet/DilatedBlock_3/Conv1d/weight:0\t                        residuals.3.block.1.weight\n",
      "        ResNet/DilatedBlock_3/BatchNorm_1/weight:0\t                        residuals.3.block.3.weight\n",
      "          ResNet/DilatedBlock_3/BatchNorm_1/bias:0\t                          residuals.3.block.3.bias\n",
      "  ResNet/DilatedBlock_3/BatchNorm_1/running_mean:0\t                  residuals.3.block.3.running_mean\n",
      "   ResNet/DilatedBlock_3/BatchNorm_1/running_var:0\t                   residuals.3.block.3.running_var\n",
      "           ResNet/DilatedBlock_3/Conv1d_1/weight:0\t                        residuals.3.block.5.weight\n",
      "          ResNet/DilatedBlock_4/BatchNorm/weight:0\t                        residuals.4.block.0.weight\n",
      "            ResNet/DilatedBlock_4/BatchNorm/bias:0\t                          residuals.4.block.0.bias\n",
      "    ResNet/DilatedBlock_4/BatchNorm/running_mean:0\t                  residuals.4.block.0.running_mean\n",
      "     ResNet/DilatedBlock_4/BatchNorm/running_var:0\t                   residuals.4.block.0.running_var\n",
      "             ResNet/DilatedBlock_4/Conv1d/weight:0\t                        residuals.4.block.1.weight\n",
      "        ResNet/DilatedBlock_4/BatchNorm_1/weight:0\t                        residuals.4.block.3.weight\n",
      "          ResNet/DilatedBlock_4/BatchNorm_1/bias:0\t                          residuals.4.block.3.bias\n",
      "  ResNet/DilatedBlock_4/BatchNorm_1/running_mean:0\t                  residuals.4.block.3.running_mean\n",
      "   ResNet/DilatedBlock_4/BatchNorm_1/running_var:0\t                   residuals.4.block.3.running_var\n",
      "           ResNet/DilatedBlock_4/Conv1d_1/weight:0\t                        residuals.4.block.5.weight\n",
      "          ResNet/DilatedBlock_5/BatchNorm/weight:0\t                        residuals.5.block.0.weight\n",
      "            ResNet/DilatedBlock_5/BatchNorm/bias:0\t                          residuals.5.block.0.bias\n",
      "    ResNet/DilatedBlock_5/BatchNorm/running_mean:0\t                  residuals.5.block.0.running_mean\n",
      "     ResNet/DilatedBlock_5/BatchNorm/running_var:0\t                   residuals.5.block.0.running_var\n",
      "             ResNet/DilatedBlock_5/Conv1d/weight:0\t                        residuals.5.block.1.weight\n",
      "        ResNet/DilatedBlock_5/BatchNorm_1/weight:0\t                        residuals.5.block.3.weight\n",
      "          ResNet/DilatedBlock_5/BatchNorm_1/bias:0\t                          residuals.5.block.3.bias\n",
      "  ResNet/DilatedBlock_5/BatchNorm_1/running_mean:0\t                  residuals.5.block.3.running_mean\n",
      "   ResNet/DilatedBlock_5/BatchNorm_1/running_var:0\t                   residuals.5.block.3.running_var\n",
      "           ResNet/DilatedBlock_5/Conv1d_1/weight:0\t                        residuals.5.block.5.weight\n",
      "          ResNet/DilatedBlock_6/BatchNorm/weight:0\t                        residuals.6.block.0.weight\n",
      "            ResNet/DilatedBlock_6/BatchNorm/bias:0\t                          residuals.6.block.0.bias\n",
      "    ResNet/DilatedBlock_6/BatchNorm/running_mean:0\t                  residuals.6.block.0.running_mean\n",
      "     ResNet/DilatedBlock_6/BatchNorm/running_var:0\t                   residuals.6.block.0.running_var\n",
      "             ResNet/DilatedBlock_6/Conv1d/weight:0\t                        residuals.6.block.1.weight\n",
      "        ResNet/DilatedBlock_6/BatchNorm_1/weight:0\t                        residuals.6.block.3.weight\n",
      "          ResNet/DilatedBlock_6/BatchNorm_1/bias:0\t                          residuals.6.block.3.bias\n",
      "  ResNet/DilatedBlock_6/BatchNorm_1/running_mean:0\t                  residuals.6.block.3.running_mean\n",
      "   ResNet/DilatedBlock_6/BatchNorm_1/running_var:0\t                   residuals.6.block.3.running_var\n",
      "           ResNet/DilatedBlock_6/Conv1d_1/weight:0\t                        residuals.6.block.5.weight\n",
      "          ResNet/DilatedBlock_7/BatchNorm/weight:0\t                        residuals.7.block.0.weight\n",
      "            ResNet/DilatedBlock_7/BatchNorm/bias:0\t                          residuals.7.block.0.bias\n",
      "    ResNet/DilatedBlock_7/BatchNorm/running_mean:0\t                  residuals.7.block.0.running_mean\n",
      "     ResNet/DilatedBlock_7/BatchNorm/running_var:0\t                   residuals.7.block.0.running_var\n",
      "             ResNet/DilatedBlock_7/Conv1d/weight:0\t                        residuals.7.block.1.weight\n",
      "        ResNet/DilatedBlock_7/BatchNorm_1/weight:0\t                        residuals.7.block.3.weight\n",
      "          ResNet/DilatedBlock_7/BatchNorm_1/bias:0\t                          residuals.7.block.3.bias\n",
      "  ResNet/DilatedBlock_7/BatchNorm_1/running_mean:0\t                  residuals.7.block.3.running_mean\n",
      "   ResNet/DilatedBlock_7/BatchNorm_1/running_var:0\t                   residuals.7.block.3.running_var\n",
      "           ResNet/DilatedBlock_7/Conv1d_1/weight:0\t                        residuals.7.block.5.weight\n",
      "          ResNet/DilatedBlock_8/BatchNorm/weight:0\t                        residuals.8.block.0.weight\n",
      "            ResNet/DilatedBlock_8/BatchNorm/bias:0\t                          residuals.8.block.0.bias\n",
      "    ResNet/DilatedBlock_8/BatchNorm/running_mean:0\t                  residuals.8.block.0.running_mean\n",
      "     ResNet/DilatedBlock_8/BatchNorm/running_var:0\t                   residuals.8.block.0.running_var\n",
      "             ResNet/DilatedBlock_8/Conv1d/weight:0\t                        residuals.8.block.1.weight\n",
      "        ResNet/DilatedBlock_8/BatchNorm_1/weight:0\t                        residuals.8.block.3.weight\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          ResNet/DilatedBlock_8/BatchNorm_1/bias:0\t                          residuals.8.block.3.bias\n",
      "  ResNet/DilatedBlock_8/BatchNorm_1/running_mean:0\t                  residuals.8.block.3.running_mean\n",
      "   ResNet/DilatedBlock_8/BatchNorm_1/running_var:0\t                   residuals.8.block.3.running_var\n",
      "           ResNet/DilatedBlock_8/Conv1d_1/weight:0\t                        residuals.8.block.5.weight\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "channel = 30\n",
    "dilation = 2\n",
    "testinput = np.random.randn(*[2, channel, 1000])\n",
    "x = tf.constant(testinput.transpose(0,2,1), 1)\n",
    "#y1 = selu(x)\n",
    "#y1 = Conv1d(x, 1, 1, 3, 2, bias=False)\n",
    "#y1 = DilatedBlock(x, channel=channel, kernel_size=9, dilation=dilation)\n",
    "#pytorch_testmodule = dm.DilatedBlock(in_channels=channel, \n",
    "#                                     out_channels=channel, kernel_size=9, nonlin=dm.SELU(), dilation=dilation)\n",
    "y1 = ResNet(x, channel=channel)\n",
    "pytorch_testmodule = dm.ConvModule(nonlin=dm.SELU(),\n",
    "    in_channel=channel, channel=channel, kernel_size=9, N=9)\n",
    "pytorch_testmodule.eval()\n",
    "with tf.Session() as sess:\n",
    "    for f, (p_names, p) in zip(tf.global_variables(), pytorch_testmodule.state_dict().items()):\n",
    "        print('%50s\\t%50s'%(f.name, p_names))\n",
    "        #print(p_names)\n",
    "        initializer = tf.constant(p.transpose(0, -1).numpy(), 1)\n",
    "        f.assign(initializer).eval()\n",
    "        #print(initializer.eval()-f.eval())\n",
    "    tf_result = sess.run(y1)#.transpose(0,2,1)\n",
    "    pth_result = pytorch_testmodule(th.autograd.Variable(th.FloatTensor(testinput))).data.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "           Encoder/DownSampleBlock/Conv1d/weight:0\t                                          0.weight\n",
      "             Encoder/DownSampleBlock/Conv1d/bias:0\t                                            0.bias\n",
      "        Encoder/DownSampleBlock/BatchNorm/weight:0\t                                          1.weight\n",
      "          Encoder/DownSampleBlock/BatchNorm/bias:0\t                                            1.bias\n",
      "  Encoder/DownSampleBlock/BatchNorm/running_mean:0\t                                    1.running_mean\n",
      "   Encoder/DownSampleBlock/BatchNorm/running_var:0\t                                     1.running_var\n",
      "         Encoder/DownSampleBlock_1/Conv1d/weight:0\t                                          4.weight\n",
      "           Encoder/DownSampleBlock_1/Conv1d/bias:0\t                                            4.bias\n",
      "      Encoder/DownSampleBlock_1/BatchNorm/weight:0\t                                          5.weight\n",
      "        Encoder/DownSampleBlock_1/BatchNorm/bias:0\t                                            5.bias\n",
      "Encoder/DownSampleBlock_1/BatchNorm/running_mean:0\t                                    5.running_mean\n",
      " Encoder/DownSampleBlock_1/BatchNorm/running_var:0\t                                     5.running_var\n",
      "         Encoder/DownSampleBlock_2/Conv1d/weight:0\t                                          8.weight\n",
      "           Encoder/DownSampleBlock_2/Conv1d/bias:0\t                                            8.bias\n",
      "      Encoder/DownSampleBlock_2/BatchNorm/weight:0\t                                          9.weight\n",
      "        Encoder/DownSampleBlock_2/BatchNorm/bias:0\t                                            9.bias\n",
      "Encoder/DownSampleBlock_2/BatchNorm/running_mean:0\t                                    9.running_mean\n",
      " Encoder/DownSampleBlock_2/BatchNorm/running_var:0\t                                     9.running_var\n",
      "         Encoder/DownSampleBlock_3/Conv1d/weight:0\t                                         12.weight\n",
      "           Encoder/DownSampleBlock_3/Conv1d/bias:0\t                                           12.bias\n",
      "      Encoder/DownSampleBlock_3/BatchNorm/weight:0\t                                         13.weight\n",
      "        Encoder/DownSampleBlock_3/BatchNorm/bias:0\t                                           13.bias\n",
      "Encoder/DownSampleBlock_3/BatchNorm/running_mean:0\t                                   13.running_mean\n",
      " Encoder/DownSampleBlock_3/BatchNorm/running_var:0\t                                    13.running_var\n"
     ]
    }
   ],
   "source": [
    "pthnet = dm.EncodeWideResNet(in_channel=1, init_channel=32, num_enc_layer=4, N_res_in_block=1, use_selu=True, num_classes=3)\n",
    "#sd = th.load('saved/ENCODE-selu-adam/0002/state_dict_highscore')\n",
    "#pthnet.load_state_dict(sd)\n",
    "\n",
    "####################################\n",
    "\n",
    "tf.reset_default_graph()\n",
    "channel = 1\n",
    "dilation = 2\n",
    "testinput = np.random.randn(*[2, channel, 16])\n",
    "x = tf.constant(testinput.transpose(0,2,1), 1)\n",
    "#y1 = selu(x)\n",
    "#y1 = Conv1d(x, 1, 1, 3, 2, bias=False)\n",
    "#y1 = DilatedBlock(x, channel=channel, kernel_size=9, dilation=dilation)\n",
    "#pytorch_testmodule = dm.DilatedBlock(in_channels=channel, \n",
    "#                                     out_channels=channel, kernel_size=9, nonlin=dm.SELU(), dilation=dilation)\n",
    "\n",
    "y1 = Encoder(x, init_channel=32)\n",
    "pytorch_testmodule = pthnet.encoder\n",
    "pytorch_testmodule.eval()\n",
    "with tf.Session() as sess:\n",
    "    for f, (p_names, p) in zip(tf.global_variables(), pytorch_testmodule.state_dict().items()):\n",
    "        print('%50s\\t%50s'%(f.name, p_names))\n",
    "        #print(p_names)\n",
    "        initializer = tf.constant(p.transpose(0, -1).numpy(), 1)\n",
    "        f.assign(initializer).eval()\n",
    "        #print(initializer.eval()-f.eval())\n",
    "    tf_result = sess.run(y1).transpose(0,2,1)\n",
    "    pth_result = pytorch_testmodule(th.autograd.Variable(th.FloatTensor(testinput))).data.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2, 64, 3) (2, 3)\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "operands could not be broadcast together with shapes (2,64,3) (2,3) ",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-14-6ca7775e0247>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;31m#print('\\nP-T =\\n', np.abs(tf_result-pth_result))\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'\\nmean abs(P-T)/max(P,T) =\\n'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mabs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf_result\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mpth_result\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtf_result\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpth_result\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m: operands could not be broadcast together with shapes (2,64,3) (2,3) "
     ]
    }
   ],
   "source": [
    "print(tf_result.shape, pth_result.shape)\n",
    "#print('\\nT=\\n',tf_result)\n",
    "#print('choice\\n', tf_result.argmax(1))\n",
    "#print('\\nP=\\n',pth_result)\n",
    "#print('choice\\n', pth_result.argmax(1))\n",
    "\n",
    "#print('\\nP-T =\\n', np.abs(tf_result-pth_result))\n",
    "print('\\nmean abs(P-T)/max(P,T) =\\n', (np.abs(tf_result-pth_result)/np.max([tf_result, pth_result])).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10, 4)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "summary_writer = tf.summary.FileWriter('/tmp/model/', graph=tf.get_default_graph())\n",
    "with tf.Session() as sess:\n",
    "    tf.global_variables_initializer().run()\n",
    "    print(y1.eval({x:np.ones([10, 256, 1])}).shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DilatedBlock (\n",
       "  (nonlin): SELU (\n",
       "  )\n",
       "  (block): Sequential (\n",
       "    (0): BatchNorm1d(8, eps=1e-05, momentum=0.1, affine=True)\n",
       "    (1): Conv1d(8, 8, kernel_size=(9,), stride=(1,), padding=(4,), bias=False)\n",
       "    (2): SELU (\n",
       "    )\n",
       "    (3): BatchNorm1d(8, eps=1e-05, momentum=0.1, affine=True)\n",
       "    (4): Dropout (p = 0.5, inplace)\n",
       "    (5): Conv1d(8, 8, kernel_size=(9,), stride=(1,), padding=(8,), dilation=(2,), bias=False)\n",
       "    (6): SELU (\n",
       "    )\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 256,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dm.DilatedBlock(nonlin=dm.SELU())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.Session() as sess:\n",
    "    tf.global_variables_initializer().run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.]],\n",
       "\n",
       "       [[ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.]],\n",
       "\n",
       "       [[ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.]],\n",
       "\n",
       "       [[ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.]],\n",
       "\n",
       "       [[ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.]],\n",
       "\n",
       "       [[ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.]],\n",
       "\n",
       "       [[ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.]],\n",
       "\n",
       "       [[ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.]],\n",
       "\n",
       "       [[ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.]],\n",
       "\n",
       "       [[ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.]]], dtype=float32)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(7, 1, 32)\n",
      "(32,)\n",
      "(32,)\n",
      "(32,)\n",
      "(32,)\n",
      "(32,)\n",
      "(7, 32, 64)\n",
      "(64,)\n",
      "(64,)\n",
      "(64,)\n",
      "(64,)\n",
      "(64,)\n",
      "(7, 64, 128)\n",
      "(128,)\n",
      "(128,)\n",
      "(128,)\n",
      "(128,)\n",
      "(128,)\n",
      "(7, 128, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(256,)\n",
      "(9, 256, 256)\n",
      "(1, 256, 4)\n"
     ]
    }
   ],
   "source": [
    "for i, a in enumerate(tf.global_variables()):\n",
    "    print(a.get_shape())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a.assign()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "115"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tf.global_variables())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "res = sd(th.autograd.Variable(th.FloatTensor(10, 1, 1000)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       "    0\n",
       "    2\n",
       "    0\n",
       "    0\n",
       "    0\n",
       "    0\n",
       "    2\n",
       "    0\n",
       "    0\n",
       "    2\n",
       "[torch.LongTensor of size 10x1]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res.max(1)[1][:, None]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([32, 1, 7])\n",
      "torch.Size([32])\n",
      "torch.Size([32])\n",
      "torch.Size([32])\n",
      "torch.Size([32])\n",
      "torch.Size([32])\n",
      "torch.Size([64, 32, 7])\n",
      "torch.Size([64])\n",
      "torch.Size([64])\n",
      "torch.Size([64])\n",
      "torch.Size([64])\n",
      "torch.Size([64])\n",
      "torch.Size([128, 64, 7])\n",
      "torch.Size([128])\n",
      "torch.Size([128])\n",
      "torch.Size([128])\n",
      "torch.Size([128])\n",
      "torch.Size([128])\n",
      "torch.Size([256, 128, 7])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256])\n",
      "torch.Size([256, 256, 9])\n",
      "torch.Size([3, 256, 1])\n"
     ]
    }
   ],
   "source": [
    "for k, v in sd.state_dict().items():\n",
    "    print(v.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a = nn.Conv1d(3, 4, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('weight', \n",
       "              (0 ,.,.) = \n",
       "               -0.1830  0.1597  0.1484 -0.1916  0.2417\n",
       "               -0.1660  0.0768 -0.1457  0.1034 -0.2297\n",
       "               -0.1860  0.0322  0.0990  0.2096 -0.1983\n",
       "              \n",
       "              (1 ,.,.) = \n",
       "               -0.1521  0.1666 -0.0709  0.0931  0.1345\n",
       "                0.1034 -0.0586  0.1383 -0.1212 -0.1902\n",
       "               -0.2440 -0.0988 -0.0854  0.1089  0.1279\n",
       "              \n",
       "              (2 ,.,.) = \n",
       "                0.0536  0.2212  0.0824  0.0108  0.2315\n",
       "               -0.2318  0.0847 -0.2136 -0.0735  0.2240\n",
       "                0.2383  0.0270  0.2266 -0.1901 -0.0234\n",
       "              \n",
       "              (3 ,.,.) = \n",
       "                0.2279  0.2184  0.0534 -0.0578  0.1607\n",
       "                0.1085  0.1057  0.0550 -0.0299  0.0537\n",
       "                0.0227  0.1641 -0.1961  0.0515 -0.1527\n",
       "              [torch.FloatTensor of size 4x3x5]), ('bias', \n",
       "               0.1855\n",
       "               0.2336\n",
       "              -0.0204\n",
       "              -0.0937\n",
       "              [torch.FloatTensor of size 4])])"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.state_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}