PySyft_Simple_Federated_Learning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "PySyft Simple Federated Learning.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Berenice2018/DeepLearning/blob/master/PySyft_Simple_Federated_Learning.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0AdE3SBhPLvM",
        "colab_type": "text"
      },
      "source": [
        "### Imports, setup\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZKqONFBkLobW",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 785
        },
        "outputId": "7c3e88d4-d4c3-48a8-ed7d-e7ea2c301c15"
      },
      "source": [
        "!pip install syft"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: syft in /usr/local/lib/python3.6/dist-packages (0.1.21a1)\n",
            "Requirement already satisfied: websocket-client>=0.56.0 in /usr/local/lib/python3.6/dist-packages (from syft) (0.56.0)\n",
            "Requirement already satisfied: Flask>=1.0.2 in /usr/local/lib/python3.6/dist-packages (from syft) (1.1.1)\n",
            "Requirement already satisfied: scikit-learn>=0.21.0 in /usr/local/lib/python3.6/dist-packages (from syft) (0.21.2)\n",
            "Requirement already satisfied: zstd>=1.4.0.0 in /usr/local/lib/python3.6/dist-packages (from syft) (1.4.1.0)\n",
            "Requirement already satisfied: tblib>=1.4.0 in /usr/local/lib/python3.6/dist-packages (from syft) (1.4.0)\n",
            "Requirement already satisfied: torchvision>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from syft) (0.3.0)\n",
            "Requirement already satisfied: websockets>=7.0 in /usr/local/lib/python3.6/dist-packages (from syft) (8.0.1)\n",
            "Requirement already satisfied: lz4>=2.1.6 in /usr/local/lib/python3.6/dist-packages (from syft) (2.1.10)\n",
            "Requirement already satisfied: numpy>=1.14.0 in /usr/local/lib/python3.6/dist-packages (from syft) (1.16.4)\n",
            "Requirement already satisfied: flask-socketio>=3.3.2 in /usr/local/lib/python3.6/dist-packages (from syft) (4.1.0)\n",
            "Requirement already satisfied: msgpack>=0.6.1 in /usr/local/lib/python3.6/dist-packages (from syft) (0.6.1)\n",
            "Requirement already satisfied: tf-encrypted>=0.5.4 in /usr/local/lib/python3.6/dist-packages (from syft) (0.5.7)\n",
            "Requirement already satisfied: torch>=1.1 in /usr/local/lib/python3.6/dist-packages (from syft) (1.1.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from websocket-client>=0.56.0->syft) (1.12.0)\n",
            "Requirement already satisfied: Jinja2>=2.10.1 in /usr/local/lib/python3.6/dist-packages (from Flask>=1.0.2->syft) (2.10.1)\n",
            "Requirement already satisfied: Werkzeug>=0.15 in /usr/local/lib/python3.6/dist-packages (from Flask>=1.0.2->syft) (0.15.5)\n",
            "Requirement already satisfied: click>=5.1 in /usr/local/lib/python3.6/dist-packages (from Flask>=1.0.2->syft) (7.0)\n",
            "Requirement already satisfied: itsdangerous>=0.24 in /usr/local/lib/python3.6/dist-packages (from Flask>=1.0.2->syft) (1.1.0)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn>=0.21.0->syft) (0.13.2)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.6/dist-packages (from scikit-learn>=0.21.0->syft) (1.3.0)\n",
            "Requirement already satisfied: pillow>=4.1.1 in /usr/local/lib/python3.6/dist-packages (from torchvision>=0.3.0->syft) (4.3.0)\n",
            "Requirement already satisfied: python-socketio>=2.1.0 in /usr/local/lib/python3.6/dist-packages (from flask-socketio>=3.3.2->syft) (4.2.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.6/dist-packages (from tf-encrypted>=0.5.4->syft) (5.1.1)\n",
            "Requirement already satisfied: tensorflow<2,>=1.12.0 in /usr/local/lib/python3.6/dist-packages (from tf-encrypted>=0.5.4->syft) (1.14.0)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.6/dist-packages (from Jinja2>=2.10.1->Flask>=1.0.2->syft) (1.1.1)\n",
            "Requirement already satisfied: olefile in /usr/local/lib/python3.6/dist-packages (from pillow>=4.1.1->torchvision>=0.3.0->syft) (0.46)\n",
            "Requirement already satisfied: python-engineio>=3.8.0 in /usr/local/lib/python3.6/dist-packages (from python-socketio>=2.1.0->flask-socketio>=3.3.2->syft) (3.8.2.post1)\n",
            "Requirement already satisfied: tensorboard<1.15.0,>=1.14.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2,>=1.12.0->tf-encrypted>=0.5.4->syft) (1.14.0)\n",
            "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2,>=1.12.0->tf-encrypted>=0.5.4->syft) (0.7.1)\n",
            "Requirement already satisfied: tensorflow-estimator<1.15.0rc0,>=1.14.0rc0 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2,>=1.12.0->tf-encrypted>=0.5.4->syft) (1.14.0)\n",
            "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2,>=1.12.0->tf-encrypted>=0.5.4->syft) (1.11.2)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2,>=1.12.0->tf-encrypted>=0.5.4->syft) (0.33.4)\n",
            "Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2,>=1.12.0->tf-encrypted>=0.5.4->syft) (0.2.2)\n",
            "Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2,>=1.12.0->tf-encrypted>=0.5.4->syft) (1.0.8)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2,>=1.12.0->tf-encrypted>=0.5.4->syft) (1.1.0)\n",
            "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2,>=1.12.0->tf-encrypted>=0.5.4->syft) (1.1.0)\n",
            "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2,>=1.12.0->tf-encrypted>=0.5.4->syft) (1.15.0)\n",
            "Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2,>=1.12.0->tf-encrypted>=0.5.4->syft) (3.7.1)\n",
            "Requirement already satisfied: google-pasta>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2,>=1.12.0->tf-encrypted>=0.5.4->syft) (0.1.7)\n",
            "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2,>=1.12.0->tf-encrypted>=0.5.4->syft) (0.8.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow<2,>=1.12.0->tf-encrypted>=0.5.4->syft) (3.1.1)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow<2,>=1.12.0->tf-encrypted>=0.5.4->syft) (41.0.1)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tensorflow<2,>=1.12.0->tf-encrypted>=0.5.4->syft) (2.8.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AYwkx91pLr_h",
        "colab_type": "code",
        "outputId": "2f27f986-7bd2-4c65-964e-9cacdc24b8f0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "import time\n",
        "import datetime\n",
        "import logging\n",
        "import math\n",
        "\n",
        "import numpy as np # linear algebra\n",
        "#import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "import torchvision\n",
        "from torchvision import datasets, models, transforms\n",
        "from torch.utils.data.sampler import SubsetRandomSampler\n",
        "from torch.utils.data import Dataset\n",
        "\n",
        "import syft as sy\n",
        "\n",
        "# Input data files are available in the \"../input/\" directory.\n",
        "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
        "\n",
        "import os\n",
        "\n",
        "logger = logging.getLogger(__name__)\n",
        "\n",
        "print(torch.cuda.is_available())"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING: Logging before flag parsing goes to stderr.\n",
            "W0722 22:33:18.348394 140407260469120 secure_random.py:26] Falling back to insecure randomness since the required custom op could not be found for the installed version of TensorFlow. Fix this by compiling custom ops. Missing file was '/usr/local/lib/python3.6/dist-packages/tf_encrypted/operations/secure_random/secure_random_module_tf_1.14.0.so'\n",
            "W0722 22:33:18.369588 140407260469120 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/tf_encrypted/session.py:26: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead.\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "True\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U1-OjJVKL0TC",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "d617c019-2954-4aba-cce0-90941f6d2fdc"
      },
      "source": [
        "# save the model on Google Drive, link Google drive to this notebook\n",
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')\n",
        "\n",
        "\n",
        "# After executing this cell above, Drive\n",
        "# files will be present in \"/content/drive/My Drive\".\n",
        "!ls \"/content/gdrive/My Drive/Colab Notebooks/flower_data/\""
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n",
            "picco_test  test  train  train_ofgdrive  valid\tvalid_ofgdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3FfPa7ijL94N",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# paths to training and test data\n",
        "data_dir = '/content/gdrive/My Drive/Colab Notebooks/flower_data/'\n",
        "train_dir = data_dir + 'train'\n",
        "valid_dir = data_dir + 'valid'\n",
        "\n",
        "#os.chdir(\"/content/gdrive/My Drive/Colab Notebooks/\")\n",
        "test_dir = data_dir + 'test'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_8qj0XFdPVaP",
        "colab_type": "text"
      },
      "source": [
        "### Architecture and helpers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "42QrxlZWMFpP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Make data loader based on the selected pre-trained model\n",
        "def create_loaders(base, final = False):\n",
        "    print('returning datasets')\n",
        "    # ResNet, DenseNet expect 224, Inception expects 299\n",
        "    img_size = 299 if base == 'Inception' else 224 \n",
        "\n",
        "    transforms_train = transforms.Compose([\n",
        "        transforms.RandomRotation(30),\n",
        "        transforms.RandomResizedCrop(img_size),\n",
        "        transforms.RandomHorizontalFlip(),\n",
        "        transforms.RandomVerticalFlip(),\n",
        "        transforms.ToTensor(),\n",
        "        transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))\n",
        "    ])\n",
        "\n",
        "    transforms_test = transforms.Compose([\n",
        "        transforms.Resize(img_size + 1),\n",
        "        transforms.CenterCrop(img_size),\n",
        "        transforms.ToTensor(),\n",
        "        transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))\n",
        "    ])\n",
        "\n",
        "    # Load the datasets with ImageFolder\n",
        "    trainset = datasets.ImageFolder(train_dir, transform=transforms_train)\n",
        "    validationset = datasets.ImageFolder(valid_dir, transform=transforms_train)\n",
        "    testset = datasets.ImageFolder(valid_dir, transform=transforms_test)\n",
        "       \n",
        "    return trainset, validationset, testset"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pimMQCE-LfV8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#transforms a torch.Dataset or a sy.BaseDataset into a sy.FederatedDataset. \n",
        "def dataset_federate(dataset, workers):\n",
        "    print('dataset_federate')\n",
        "\n",
        "    datasets = []\n",
        "    data_loader = torch.utils.data.DataLoader(dataset, batch_size=32)\n",
        "    \n",
        "    for dataset_idx, (data, targets) in enumerate(data_loader):\n",
        "        worker = workers[dataset_idx % len(workers)]\n",
        "        data = data.send(worker)\n",
        "        targets = targets.send(worker)\n",
        "        datasets.append(sy.BaseDataset(data, targets))\n",
        "    \n",
        "    fed_dataset = sy.FederatedDataset(datasets)\n",
        "    fed_loader = sy.FederatedDataLoader(fed_dataset, batch_size=32, shuffle=False, drop_last=False)\n",
        "    \n",
        "    return fed_loader"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nVFx0OAMqJsR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def train(args, model, device, train_loader, optimizer, epoch):\n",
        "    model.train()\n",
        "    for batch_idx, (data, target) in enumerate(train_loader): # distributed dataset\n",
        "        \n",
        "        model.send(data.location) # send the model to the right location\n",
        "        \n",
        "        #data, target = data.to(device), target.to(device)\n",
        "        optimizer.zero_grad()\n",
        "        output = model(data)\n",
        "        loss = F.nll_loss(output, target)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        \n",
        "        model.get() # get the model back\n",
        "        \n",
        "        if batch_idx % args.log_interval == 0:\n",
        "            loss = loss.get() # get the loss back\n",
        "            print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n",
        "                epoch, batch_idx * args.batch_size, len(train_loader) * args.batch_size, #batch_idx * len(data), len(train_loader.dataset),\n",
        "                100. * batch_idx / len(train_loader), loss.item()))\n",
        "            \n",
        "    print('finished training')  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PGXMjxfOOJ60",
        "colab_type": "text"
      },
      "source": [
        "### Instantiation,  hyperparams, model training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y25JyF08Ng0t",
        "colab_type": "code",
        "outputId": "811be734-43fd-4d9e-e1b7-ffa91e22bd55",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "class Arguments():\n",
        "    def __init__(self):\n",
        "        self.batch_size = 64\n",
        "        self.test_batch_size = 1000\n",
        "        self.epochs = 2\n",
        "        self.lr = 0.01\n",
        "        self.momentum = 0.5\n",
        "        self.no_cuda = False\n",
        "        self.seed = 1\n",
        "        self.log_interval = 10\n",
        "        self.save_model = False\n",
        "\n",
        "args = Arguments()\n",
        "\n",
        "use_cuda = not args.no_cuda and torch.cuda.is_available()\n",
        "print(torch.cuda.is_available())\n",
        "\n",
        "torch.manual_seed(args.seed)\n",
        "\n",
        "device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n",
        "\n",
        "kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}\n",
        "\n",
        "\n",
        "model = models.densenet161(pretrained=True)\n",
        "#model.classifier\n",
        "\n",
        "optimizer = optim.SGD(model.parameters(), lr=args.lr) # momentum is not supported yet\n",
        "scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience = 4)\n",
        "\n"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "True\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1qAp9y-oow4J",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "for param in model.parameters():\n",
        "    param.requires_grad = True\n",
        "    \n",
        "fc_in = model.classifier.in_features\n",
        "\n",
        "transferclassifier = nn.Sequential(\n",
        "                        nn.BatchNorm1d(fc_in),\n",
        "                        nn.Linear(fc_in, 102)\n",
        "                        )\n",
        "\n",
        "#transfermodel.fc = transferclassifier # resnet\n",
        "model.classifier = transferclassifier\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qXrWb7vbPjFx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# create workers, \n",
        "hook = sy.TorchHook(torch)\n",
        "\n",
        "ada = sy.VirtualWorker(hook, 'ada')\n",
        "bob = sy.VirtualWorker(hook, 'bob')\n",
        "cyd = sy.VirtualWorker(hook, 'cyd')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sif7E44LLnv4",
        "colab_type": "code",
        "outputId": "6d2db404-69c8-4d08-a1fa-6036b8497529",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "# Create the data loaders, federated PySyft loaders are returned\n",
        "my_trainset, my_validset , _ = create_loaders('Densenet')\n",
        "train_loader = dataset_federate(my_trainset, (ada,bob,cyd))\n",
        "#valid_loader = dataset_federate(my_validset, (ada,bob,cyd))"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "returning datasets\n",
            "dataset_federate\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FgfPMF2JZ05r",
        "colab_type": "text"
      },
      "source": [
        "### start the training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6P6WWu0nNxny",
        "colab_type": "code",
        "outputId": "d6445ee1-f741-4108-88f3-fdb6555192c2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "print(f'objects of ada= {len(ada._objects)}, bob= {len(bob._objects)}, cyd= {len(cyd._objects)}')\n",
        "print(device)\n",
        "epochs = 2\n",
        "#model.to(device)\n",
        "##### START THE TRAINING #### \n",
        "trainedmodel = train(args, model, device, train_loader, optimizer, epochs)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "objects of ada= 2, bob= 2, cyd= 2\n",
            "cuda\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "TypeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-12-aa45b1519919>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;31m#model.to(device)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;31m##### START THE TRAINING ####\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mtrainedmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-7-d8f02bd89808>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(args, model, device, train_loader, optimizer, epoch)\u001b[0m\n\u001b[1;32m      7\u001b[0m         \u001b[0;31m#data, target = data.to(device), target.to(device)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m         \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnll_loss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    492\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 493\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    494\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    495\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torchvision/models/densenet.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m    117\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    118\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 119\u001b[0;31m         \u001b[0mfeatures\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    120\u001b[0m         \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minplace\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    121\u001b[0m         \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madaptive_avg_pool2d\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    492\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 493\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    494\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    495\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/container.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     90\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_modules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 92\u001b[0;31m             \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     93\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    492\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 493\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    494\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    495\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/batchnorm.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     59\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mweak_script_method\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 61\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_input_dim\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     62\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m         \u001b[0;31m# exponential_average_factor is self.momentum set to\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/batchnorm.py\u001b[0m in \u001b[0;36m_check_input_dim\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    246\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mweak_script_method\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    247\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_check_input_dim\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 248\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m4\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    249\u001b[0m             raise ValueError('expected 4D input (got {}D input)'\n\u001b[1;32m    250\u001b[0m                              .format(input.dim()))\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/syft/frameworks/torch/hook/hook.py\u001b[0m in \u001b[0;36moverloaded_native_method\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    675\u001b[0m                 \u001b[0;31m# Send the new command to the appropriate class and get the response\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    676\u001b[0m                 \u001b[0mmethod\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew_self\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 677\u001b[0;31m                 \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mnew_args\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mnew_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    678\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    679\u001b[0m                 \u001b[0;31m# For inplace methods, just directly return self\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/syft/frameworks/torch/pointers/pointer_tensor.py\u001b[0m in \u001b[0;36mdim\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    194\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    195\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 196\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_shape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    197\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    198\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mshare\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTypeError\u001b[0m: object of type 'NoneType' has no len()"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NToncEuLt-ns",
        "colab_type": "text"
      },
      "source": [
        "### Clear the worker"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E1G7HggpsFE_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#ada.clear_objects()\n",
        "#bob.clear_objects()\n",
        "#cyd.clear_objects()"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}