FER_squeezenet_berenice.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "FER-squeezenet-berenice.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [
        "GcA_vwhUrURY"
      ],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Berenice2018/DeepLearning/blob/master/FER_squeezenet_berenice.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yh9XQ8vR84zw",
        "colab_type": "text"
      },
      "source": [
        "### Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5xdgcVYCJpkj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Imports here\n",
        "#!pip install git+https://github.com/pytorch/vision\n",
        "\n",
        "import torch\n",
        "import torch.functional as F\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "\n",
        "from torch import nn, optim\n",
        "from torch.optim import lr_scheduler\n",
        "from torch.autograd import Variable\n",
        "from torch.utils.data.sampler import SubsetRandomSampler\n",
        "\n",
        "import torchvision.models as models\n",
        "from torchvision.models.squeezenet import  model_urls\n",
        "\n",
        "import torchvision\n",
        "from torchvision import datasets, transforms\n",
        "\n",
        "\n",
        "import os,cv2\n",
        "import json\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "import time\n",
        "import datetime\n",
        "import copy\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "#from pylab import rcParams\n",
        "#rcParams['figure.figsize'] = 20, 10"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fCJINZX0Kcpk",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "c92654ec-e412-40d9-b477-fb1962cf58d8"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Nl8ynY8-MVOg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "fer= \"./gdrive/My Drive/Colab Notebooks/Fer-dataset/fer2013/fer2013.csv\"\n",
        "ferr= \"./gdrive/My Drive/Colab Notebooks/Fer-dataset\"\n",
        "\n",
        "#!ls   '/content/gdrive/My Drive/Colab Notebooks/Fer-dataset/fer2013/'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h2lVhUAPOZa4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install mxnet"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GcA_vwhUrURY",
        "colab_type": "text"
      },
      "source": [
        "### Convert and save images to Gdrive"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5I5WuBM0VOJJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#! /usr/bin/env python3\n",
        "# -*-coding: utf-8-*-\n",
        "\n",
        "__author__ = 'Moonkie'\n",
        "\n",
        "import numpy as np\n",
        "import cv2\n",
        "import mxnet as mx\n",
        "import pandas as pd\n",
        "import random\n",
        "import os\n",
        "\n",
        "curdir = os.path.abspath(os.path.dirname(fer))\n",
        "\n",
        "def gen_record(csvfile,channel):\n",
        "    data = pd.read_csv(csvfile,delimiter=',',dtype='a')\n",
        "    labels = np.array(data['emotion'],np.float)\n",
        "    # print(labels,'\\n',data['emotion'])\n",
        "        \n",
        "    imagebuffer = np.array(data['pixels'])\n",
        "    images = np.array([np.fromstring(image,np.uint8,sep=' ') for image in imagebuffer])\n",
        "    del imagebuffer\n",
        "    num_shape = int(np.sqrt(images.shape[-1]))\n",
        "    images.shape = (images.shape[0],num_shape,num_shape)\n",
        "    # img=images[0];cv2.imshow('test',img);cv2.waitKey(0);cv2.destroyAllWindow();exit()\n",
        "    dirs = set(data['Usage'])\n",
        "    subdirs = set(labels)\n",
        "    class_dir = {}\n",
        "    for dr in dirs:\n",
        "        dest = os.path.join(curdir,dr)\n",
        "        class_dir[dr] = dest\n",
        "        if not os.path.exists(dest):\n",
        "            os.mkdir(dest)\n",
        "            \n",
        "    data = zip(labels,images,data['Usage'])\n",
        "    \n",
        "    for d in data:\n",
        "        destdir = os.path.join(class_dir[d[-1]],str(int(d[0])))\n",
        "        if not os.path.exists(destdir):\n",
        "            os.mkdir(destdir)\n",
        "        img = d[1]\n",
        "        filepath = unique_name(destdir,d[-1])\n",
        "        print('[^_^] Write image to %s' % filepath)\n",
        "        if not filepath:\n",
        "            continue\n",
        "        sig = cv2.imwrite(filepath,img)\n",
        "        if not sig:\n",
        "            print('Error')\n",
        "            exit(-1)\n",
        "\n",
        "\n",
        "def unique_name(pardir,prefix,suffix='jpg'):\n",
        "    filename = '{0}_{1}.{2}'.format(prefix,random.randint(1,10**8),suffix)\n",
        "    filepath = os.path.join(pardir,filename)\n",
        "    if not os.path.exists(filepath):\n",
        "        return filepath\n",
        "    unique_name(pardir,prefix,suffix)\n",
        "    \n",
        "\n",
        "\n",
        "if __name__ == '__main__':\n",
        "    filename = 'fer2013.csv'\n",
        "    filename = os.path.join(curdir,filename)\n",
        "    gen_record(filename,1)\n",
        "    \n",
        "    # ##################### test\n",
        "    # tmp = unique_name('./Training','Training')\n",
        "    # print(tmp)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5yAg9oU_radg",
        "colab_type": "text"
      },
      "source": [
        "### Load data , image transform"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mcvDN0ydqjQ-",
        "colab_type": "code",
        "outputId": "3dd1d08c-f116-44cd-ec2f-de31d3c3723d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "\n",
        "data_dir = \"./gdrive/My Drive/Colab Notebooks/Fer-dataset/fer2013\"\n",
        "train_dir = data_dir + '/Training'\n",
        "valid_dir = data_dir + '/PublicTest'\n",
        "test_dir = data_dir + '/PrivateTest'\n",
        "\n",
        "# original img size = 48 x48 px\n",
        "# TODO: Define your transforms for the training and validation sets\n",
        "data_transforms = {\n",
        "    'train': transforms.Compose([\n",
        "        transforms.Grayscale(3),\n",
        "        transforms.RandomHorizontalFlip(),\n",
        "        transforms.Resize(224),\n",
        "        transforms.ToTensor(),\n",
        "        transforms.Normalize([0.485, 0.456, 0.406], \n",
        "                             [0.229, 0.224, 0.225])\n",
        "    ]),\n",
        "    'valid': transforms.Compose([\n",
        "        transforms.Grayscale(3),\n",
        "        #transforms.CenterCrop(256),\n",
        "        transforms.RandomHorizontalFlip(),\n",
        "        transforms.Resize(224),\n",
        "        transforms.ToTensor(),\n",
        "        transforms.Normalize([0.485, 0.456, 0.406], \n",
        "                             [0.229, 0.224, 0.225])\n",
        "    ]),\n",
        "    'test': transforms.Compose([\n",
        "        transforms.Grayscale(3),\n",
        "        transforms.Resize(224),\n",
        "        transforms.ToTensor(),\n",
        "        transforms.Normalize([0.485, 0.456, 0.406], \n",
        "                             [0.229, 0.224, 0.225])\n",
        "    ])\n",
        "    }\n",
        "\n",
        "# TODO: Load the datasets with ImageFolder\n",
        "\n",
        "dirs = {'train': train_dir, \n",
        "        'valid': valid_dir,\n",
        "        'test': test_dir}\n",
        "\n",
        "datasets = {x: torchvision.datasets.ImageFolder(dirs[x], transform=data_transforms[x]) \n",
        "            for x in ['train', 'valid', 'test']}\n",
        "\n",
        "dataloaders = {x: torch.utils.data.DataLoader(datasets[x], batch_size=32, shuffle=True, num_workers=7) \n",
        "               for x in ['train', 'valid','test']}\n",
        "\n",
        "\n",
        "dataset_sizes = {x: len(datasets[x]) \n",
        "                              for x in ['train', 'valid','test']}\n",
        "print(dataset_sizes)"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'train': 32556, 'valid': 3589, 'test': 3589}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xR9fxwj-f-wG",
        "colab_type": "code",
        "outputId": "86ebc428-337c-4192-b606-26a9055cbf72",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "class_names = datasets['train'].classes\n",
        "print(class_names)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['0', '1', '2', '3', '4', '5', '6']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rj5m7HOg3gHn",
        "colab_type": "text"
      },
      "source": [
        "### Show some sample images"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O9_rLvF03kGg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# get a batch of training images\n",
        "dataiter = iter(dataloaders['train'])\n",
        "images, _ = dataiter.next()\n",
        "images = images.numpy() # convert images to numpy\n",
        "\n",
        "# plot the images of the batch, along with the corresponding labels\n",
        "fig = plt.figure(figsize=(20, 4))\n",
        "\n",
        "for idx in np.arange(16):\n",
        "    ax = fig.add_subplot(2, 16/2, idx+1, xticks=[], yticks=[])\n",
        "    print(images[idx].shape)\n",
        "    plt.imshow(np.transpose(images[idx], (1, 2, 0)))  # convert from Tensor image"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o1gi4Auz4D_l",
        "colab_type": "text"
      },
      "source": [
        "### Model train helper"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "REMm_P_CERmw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_time():\n",
        "      hour = datetime.datetime.today().hour +2\n",
        "      minute = datetime.datetime.today().minute\n",
        "      second = datetime.datetime.today().second\n",
        "      return hour, minute, second\n",
        "    \n",
        "\n",
        "\n",
        "def weights_init_normal(m):\n",
        "    '''Takes in a module and initializes all linear layers with weight\n",
        "       values taken from a normal distribution.'''\n",
        "    classname = m.__class__.__name__\n",
        "    # for every Linear layer in a model\n",
        "    if classname.find('Linear') != -1:\n",
        "        n = m.in_features\n",
        "        # m.weight.data shoud be taken from a normal distribution\n",
        "        m.weight.data.normal_(0, 1/np.sqrt(n))\n",
        "        # m.bias.data should be 0\n",
        "        m.bias.data.fill_(0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vwzo4GzQsAYy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def train_model(model, name, criteria, optimizer, scheduler = None, epochs=50, device='cuda'):\n",
        "    h, m, s = get_time()\n",
        "    print('starting the training at {}:{}:{}'.format(h, m, s))\n",
        "    since = time.time()\n",
        "    best_model = copy.deepcopy(model.state_dict())\n",
        "    \n",
        "    # initialize tracker for accuracy \n",
        "    min_loss = np.Inf\n",
        "    TS_loss = pd.DataFrame(np.nan, index = range(1,epochs+1), columns = ['train loss','valid loss'])\n",
        "    TS_acc = pd.DataFrame(np.nan,index = range(1,epochs+1), columns = ['train acc','valid acc'])\n",
        "    \n",
        "    for epoch in range(1, epochs+1):        \n",
        "        # monitor loss/accuracy\n",
        "        train_loss_running = 0.0\n",
        "        valid_loss_running = 0.0\n",
        "        train_acc_running = 0.0\n",
        "        valid_acc_running = 0.0\n",
        "\n",
        "        ###################\n",
        "        # train the model #\n",
        "        ###################\n",
        "        if scheduler:\n",
        "            scheduler.step()\n",
        "        model.train() # prep model for training\n",
        "        \n",
        "        for data, target in dataloaders['train']:\n",
        "    \n",
        "            # Move input and data tensors to the default device\n",
        "            data, target = data.to(device), target.to(device)\n",
        "            # clear the gradients of all optimized variables\n",
        "            optimizer.zero_grad()\n",
        "            # forward pass: compute predicted outputs by passing inputs to the model\n",
        "            output = model(data)\n",
        "            # calculate the batch loss\n",
        "            loss = criteria(output, target)\n",
        "            # backward pass: compute gradient of the loss with respect to model parameters\n",
        "            loss.backward()\n",
        "            # perform a single optimization step (parameter update)\n",
        "            optimizer.step()\n",
        "            # update training loss \n",
        "            train_loss_running += loss.item()*data.size(0)\n",
        "            # accuracy\n",
        "            _, preds = torch.max(output, 1)\n",
        "            train_acc_running  += torch.sum(preds == target.data)\n",
        "    \n",
        "\n",
        "        ######################    \n",
        "        # validate the model #\n",
        "        ######################\n",
        "        model.eval()\n",
        "        with torch.no_grad():\n",
        "            for data, target in dataloaders['valid']:\n",
        "                data, target = data.to(device), target.to(device)\n",
        "                output = model(data)\n",
        "                loss = criteria(output, target)\n",
        "                valid_loss_running += loss.item()*data.size(0)\n",
        "                \n",
        "                _, preds = torch.max(output, 1)\n",
        "                valid_acc_running  += torch.sum(preds == target.data)\n",
        "                #print('valid_acc_running {:.6f}'.format(valid_acc_running))\n",
        "                \n",
        "                # Calculate accuracy\n",
        "                #ps = torch.exp(logps)\n",
        "                #top_p, top_class = ps.topk(1, dim=1)\n",
        "                #equals = top_class == labels.view(*top_class.shape)\n",
        "                #valid_loss_running += torch.mean(equals.type(torch.FloatTensor)).item()\n",
        "\n",
        "        # statistics\n",
        "        train_loss = train_loss_running/ dataset_sizes['train']\n",
        "        valid_loss = valid_loss_running/ dataset_sizes['valid']\n",
        "        train_acc = train_acc_running.double()/ dataset_sizes['train']\n",
        "        valid_acc = valid_acc_running.double()/ dataset_sizes['valid']\n",
        "        \n",
        "        TS_loss.loc[epoch] = [train_loss, valid_loss]\n",
        "        TS_acc.loc[epoch] = [(train_acc.cpu().numpy()), (valid_acc.cpu().numpy())]                                         \n",
        "\n",
        "        # print training/validation statistics \n",
        "        hour, minute, second = get_time()\n",
        "        print('Epoch: {} at {}:{}:{} \\tTrain. Loss: {:.6f} \\tValid. Loss: {:.6f} \\t Accur.: {:.10f}'.format(\n",
        "                  epoch,\n",
        "                  hour, minute, second,\n",
        "                  train_loss,\n",
        "                  valid_loss,\n",
        "                  valid_acc \n",
        "        ))\n",
        "                                                             \n",
        "        # save model if validation acc has increased\n",
        "        # BC: using the lower validation loss as criterium, when saving a model\n",
        "        if valid_loss <= min_loss:\n",
        "            old_loss_min = min_loss\n",
        "            min_loss = valid_loss\n",
        "            save = True\n",
        "            \n",
        "            print('*** Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(\n",
        "            old_loss_min, min_loss))        \n",
        "    \n",
        "    #End\n",
        "    time_elapsed = time.time() - since\n",
        "    print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))\n",
        "    #print('Best Valid Acc: {:4f}'.format(best_acc))\n",
        "                                                   \n",
        "    TS_loss.plot(title = 'loss')\n",
        "    TS_acc.plot(title = 'acc')\n",
        "                                                   \n",
        "    # load best model weights\n",
        "    model.load_state_dict(best_model)\n",
        "    torch.save(model.state_dict(), name)\n",
        "    \n",
        "    return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0AGfumpM4Lik",
        "colab_type": "text"
      },
      "source": [
        "### Hyper params, start training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XeUCHPa_40ky",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#model = torch.hub.load_state_dict_from_url(model_urls['squeezenet1_0'])\n",
        "#model = torch.hub.load('pytorch/vision', 'squeezenet1_0', pretrained=True)\n",
        "\n",
        "no_of_classes = 7\n",
        "dropout = 0.2\n",
        "transfermodel = models.squeezenet1_0(pretrained =True)\n",
        "\n",
        "for param in transfermodel.parameters():\n",
        "    param.requires_grad = False\n",
        "    \n",
        "transfermodel.classifier[1] = nn.Conv2d(512, no_of_classes, kernel_size=(1,1), stride=(1,1))\n",
        "transfermodel.classifier[0] = nn.Dropout(dropout)\n",
        "\n",
        "#transfermodel.apply(weights_init_normal)\n",
        "\n",
        "#transfermodel.eval()\n",
        "\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "transfermodel.to(device);\n",
        "\n",
        "transfermodel"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6hrVk_TkscuM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "epochs = 20\n",
        "\n",
        "criteria = nn.CrossEntropyLoss()    \n",
        "lr = 3e-2 #1e-2 #1e-3\n",
        "\n",
        "#optimizer = optim.Adam(transfermodel.parameters(), lr=lr)\n",
        "optimizer = optim.SGD(transfermodel.parameters(), lr=lr)\n",
        "\n",
        "stepsize = 4\n",
        "scheduler = lr_scheduler.StepLR(optimizer, step_size=stepsize, gamma=0.1)\n",
        "#scheduler = optim.lr_scheduler.MultiStepLR(optimizer, [4, 7, 10, 12], 0.2)\n",
        "\n",
        "\n",
        "# create model name\n",
        "save_path = './gdrive/My Drive/Colab Notebooks/Fer-dataset/'\n",
        "modelname = '{}squeeze1_0-drop{}-{}-lr{}-{}{}-epo{}.pth'.format(\n",
        "    save_path, dropout, 'SGD', lr, 'StepLR', stepsize, epochs)\n",
        "\n",
        "print(modelname)\n",
        "\n",
        "model_ft = train_model(transfermodel, modelname, criteria, optimizer, scheduler, epochs, device)\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c0csjXEyJbjK",
        "colab_type": "text"
      },
      "source": [
        "### Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wBcUMyEhJasV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def test(loader, model, criterion, use_cuda):\n",
        "\n",
        "    # monitor test loss and accuracy\n",
        "    test_loss = 0.\n",
        "    correct = 0.\n",
        "    total = 0.\n",
        "    \n",
        "    print('test started at ', get_time())\n",
        "    model.eval()\n",
        "    for batch_idx, (data, target) in enumerate(loader):\n",
        "        # move to GPU\n",
        "        if use_cuda:\n",
        "            data, target = data.cuda(), target.cuda()\n",
        "        # forward pass: compute predicted outputs by passing inputs to the model\n",
        "        output = model(data)\n",
        "        # calculate the loss\n",
        "        loss = criterion(output, target)\n",
        "        # update average test loss \n",
        "        test_loss = test_loss + ((1 / (batch_idx + 1)) * (loss.data - test_loss))\n",
        "        # convert output probabilities to predicted class\n",
        "        pred = output.data.max(1, keepdim=True)[1]\n",
        "        # compare predictions to true label\n",
        "        correct += np.sum(np.squeeze(pred.eq(target.data.view_as(pred))).cpu().numpy())\n",
        "        total += data.size(0)\n",
        "            \n",
        "    print('test finished at ', get_time())\n",
        "    print('Test Loss: {:.6f}\\n'.format(test_loss))\n",
        "\n",
        "    print('\\nTest Accuracy: %2d%% (%2d/%2d)' % (\n",
        "        100. * correct / total, correct, total))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bh5CaGYNLErC",
        "colab_type": "code",
        "outputId": "f80edbb2-5be5-4810-9855-46f0b12cd653",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        }
      },
      "source": [
        "\n",
        "# call test function    \n",
        "test(dataloaders['test'], model_ft, criteria, device)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "test started at  (11, 19, 37)\n",
            "test finished at  (11, 21, 48)\n",
            "Test Loss: 1.883936\n",
            "\n",
            "\n",
            "Test Accuracy: 24% (875/3589)\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}