FER_CNN_from_scratch_Berenice.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "FER CNN from scratch Berenice.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [
        "OwJzTTM1pe-s",
        "IiK7hITUqQDr"
      ],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Berenice2018/DeepLearning/blob/master/FER_CNN_from_scratch_Berenice.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qV5Lz9WppDsP",
        "colab_type": "code",
        "outputId": "68338086-d9a8-49e7-bfea-f6e10feec9b1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "\n",
        "import torch\n",
        "import torch.functional as F\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "import torchvision\n",
        "from torch import nn, optim\n",
        "from torch.optim import lr_scheduler\n",
        "from torch.autograd import Variable\n",
        "from torch.utils.data.sampler import SubsetRandomSampler\n",
        "from torchvision import datasets, models, transforms\n",
        "from torchvision import datasets, transforms, models\n",
        "from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\n",
        "\n",
        "from PIL import ImageFile\n",
        "ImageFile.LOAD_TRUNCATED_IMAGES = True\n",
        "\n",
        "import json\n",
        "import numpy as np\n",
        "import time\n",
        "import datetime\n",
        "import copy\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "import os, cv2\n",
        "import pandas as pd\n",
        "#from pylab import rcParams\n",
        "#rcParams['figure.figsize'] = 20, 10\n",
        "\n",
        "print(torch.__version__)"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1.1.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LpSbHOBSpRDR",
        "colab_type": "code",
        "outputId": "e8a67945-3572-4db2-9acd-c8baa484d10a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OwJzTTM1pe-s",
        "colab_type": "text"
      },
      "source": [
        "### Convert and save images to GDrive"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5RpGjhIHpdXP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#! /usr/bin/env python3\n",
        "# -*-coding: utf-8-*-\n",
        "\n",
        "#!pip install mxnet\n",
        "\n",
        "__author__ = 'Moonkie'\n",
        "\n",
        "import numpy as np\n",
        "import cv2\n",
        "import mxnet as mx\n",
        "import pandas as pd\n",
        "import random\n",
        "import os\n",
        "\n",
        "curdir = os.path.abspath(os.path.dirname(fer))\n",
        "\n",
        "def gen_record(csvfile,channel):\n",
        "    data = pd.read_csv(csvfile,delimiter=',',dtype='a')\n",
        "    labels = np.array(data['emotion'],np.float)\n",
        "    # print(labels,'\\n',data['emotion'])\n",
        "        \n",
        "    imagebuffer = np.array(data['pixels'])\n",
        "    images = np.array([np.fromstring(image,np.uint8,sep=' ') for image in imagebuffer])\n",
        "    del imagebuffer\n",
        "    num_shape = int(np.sqrt(images.shape[-1]))\n",
        "    images.shape = (images.shape[0],num_shape,num_shape)\n",
        "    # img=images[0];cv2.imshow('test',img);cv2.waitKey(0);cv2.destroyAllWindow();exit()\n",
        "    dirs = set(data['Usage'])\n",
        "    subdirs = set(labels)\n",
        "    class_dir = {}\n",
        "    for dr in dirs:\n",
        "        dest = os.path.join(curdir,dr)\n",
        "        class_dir[dr] = dest\n",
        "        if not os.path.exists(dest):\n",
        "            os.mkdir(dest)\n",
        "            \n",
        "    data = zip(labels,images,data['Usage'])\n",
        "    \n",
        "    for d in data:\n",
        "        destdir = os.path.join(class_dir[d[-1]],str(int(d[0])))\n",
        "        if not os.path.exists(destdir):\n",
        "            os.mkdir(destdir)\n",
        "        img = d[1]\n",
        "        filepath = unique_name(destdir,d[-1])\n",
        "        print('[^_^] Write image to %s' % filepath)\n",
        "        if not filepath:\n",
        "            continue\n",
        "        sig = cv2.imwrite(filepath,img)\n",
        "        if not sig:\n",
        "            print('Error')\n",
        "            exit(-1)\n",
        "\n",
        "\n",
        "def unique_name(pardir,prefix,suffix='jpg'):\n",
        "    filename = '{0}_{1}.{2}'.format(prefix,random.randint(1,10**8),suffix)\n",
        "    filepath = os.path.join(pardir,filename)\n",
        "    if not os.path.exists(filepath):\n",
        "        return filepath\n",
        "    unique_name(pardir,prefix,suffix)\n",
        "    \n",
        "\n",
        "\n",
        "if __name__ == '__main__':\n",
        "    filename = 'fer2013.csv'\n",
        "    filename = os.path.join(curdir,filename)\n",
        "    gen_record(filename,1)\n",
        "    \n",
        "    # ##################### test\n",
        "    # tmp = unique_name('./Training','Training')\n",
        "    # print(tmp)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Dd_klMuOpmWR",
        "colab_type": "text"
      },
      "source": [
        "### Load data , image transform"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NC-QCr7SpVvS",
        "colab_type": "code",
        "outputId": "176b341d-f1e7-48bc-dd9b-001ad8e0d193",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "fer= \"./gdrive/My Drive/Colab Notebooks/Fer-dataset/fer2013/fer2013.csv\"\n",
        "ferr= \"./gdrive/My Drive/Colab Notebooks/Fer-dataset\"\n",
        "\n",
        "!ls   '/content/gdrive/My Drive/Colab Notebooks/Fer-dataset/fer2013/'"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "fer2013.bib  fer2013.csv  PrivateTest  PublicTest  README  Training\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Of9aDCc8pqY9",
        "colab_type": "code",
        "outputId": "50048459-da20-41e4-f1b2-50de4f1b97d9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "data_dir = \"./gdrive/My Drive/Colab Notebooks/Fer-dataset/fer2013\"\n",
        "train_dir = data_dir + '/Training'\n",
        "valid_dir = data_dir + '/PublicTest'\n",
        "test_dir = data_dir + '/PrivateTest'\n",
        "\n",
        "# original img size = 48 x48 px\n",
        "# TODO: Define your transforms for the training and validation sets\n",
        "data_transforms = {\n",
        "    'train': transforms.Compose([\n",
        "        transforms.RandomHorizontalFlip(),\n",
        "        #transforms.RandomResizedCrop(256),\n",
        "        transforms.Resize(256), #320\n",
        "        transforms.ToTensor(),\n",
        "        transforms.Normalize([0.485,], \n",
        "                             [0.229, ])\n",
        "    ]),\n",
        "    'valid': transforms.Compose([\n",
        "        #transforms.CenterCrop(256),\n",
        "        transforms.Resize(256),  \n",
        "        transforms.ToTensor(),\n",
        "        transforms.Normalize([0.485, 0.456, 0.406], \n",
        "                             [0.229, 0.224, 0.225])\n",
        "    ]),\n",
        "    'test': transforms.Compose([\n",
        "        #transforms.CenterCrop(224),\n",
        "        transforms.Resize(256),\n",
        "        transforms.ToTensor(),\n",
        "        transforms.Normalize([0.485, 0.456, 0.406], \n",
        "                             [0.229, 0.224, 0.225])\n",
        "    ])\n",
        "    }\n",
        "\n",
        "# TODO: Load the datasets with ImageFolder\n",
        "\n",
        "dirs = {'train': train_dir, \n",
        "        'valid': valid_dir,\n",
        "        'test': test_dir}\n",
        "\n",
        "datasets = {x: torchvision.datasets.ImageFolder(dirs[x],   transform=data_transforms[x]) for x in ['train', 'valid', 'test']}\n",
        "\n",
        "# TODO: Using the image datasets and the trainforms, define the dataloaders\n",
        "\n",
        "dataloaders = {x: torch.utils.data.DataLoader(datasets[x], batch_size=64, shuffle=True, num_workers=7) \n",
        "               for x in ['train', 'valid','test']}\n",
        "\n",
        "dataset_sizes = {x: len(datasets[x]) \n",
        "                              for x in ['train', 'valid','test']}\n",
        "print(dataset_sizes)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'train': 32556, 'valid': 3589, 'test': 3589}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PtTvsEhCpwyu",
        "colab_type": "code",
        "outputId": "d7a2c4c1-07e8-47f6-a43a-12e18f8eee9f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "class_names = datasets['train'].classes\n",
        "print(class_names)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['0', '1', '2', '3', '4', '5', '6']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iHGUaFNGpz6L",
        "colab_type": "text"
      },
      "source": [
        "### Show some sample images"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mw2I5ywip2uZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# get a batch of training images\n",
        "dataiter = iter(dataloaders['train'])\n",
        "images, _ = dataiter.next()\n",
        "images = images.numpy() # convert images to numpy\n",
        "\n",
        "# plot the images of the batch, along with the corresponding labels\n",
        "fig = plt.figure(figsize=(20, 4))\n",
        "\n",
        "for idx in np.arange(16):\n",
        "    ax = fig.add_subplot(2, 16/2, idx+1, xticks=[], yticks=[])\n",
        "    plt.imshow(np.transpose(images[idx], (1, 2, 0)))  # convert from Tensor image"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-jQVxEZpp56a",
        "colab_type": "text"
      },
      "source": [
        "### Helpers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NSNZ79L2p8bQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_time():\n",
        "      hour = datetime.datetime.today().hour +2\n",
        "      minute = datetime.datetime.today().minute\n",
        "      second = datetime.datetime.today().second\n",
        "      return hour, minute, second\n",
        "    \n",
        "    \n",
        "def weights_init_normal(m):\n",
        "  '''Takes in a module and initializes all linear layers with weight\n",
        "     values taken from a normal distribution.'''\n",
        "  classname = m.__class__.__name__\n",
        "  # for every Linear layer in a model\n",
        "  if classname.find('Linear') != -1:\n",
        "      n = m.in_features\n",
        "      # m.weight.data shoud be taken from a normal distribution\n",
        "      m.weight.data.normal_(0, 1/np.sqrt(n))\n",
        "      # m.bias.data should be 0\n",
        "      m.bias.data.fill_(0)\n",
        "      \n",
        "      \n",
        "# Visualize plot\n",
        "def plot_loss_acc(n_epochs, train_losses, valid_losses, valid_accuracies):\n",
        "    fig, (ax1, ax2) = plt.subplots(figsize=(14,6), ncols=2)\n",
        "    ax1.plot(valid_losses, label='Validation loss')\n",
        "    ax1.plot(train_losses, label='Training loss')\n",
        "    ax1.legend(frameon=False)\n",
        "    ax1.set_xlabel('Epochs')\n",
        "    ax1.set_ylabel('Loss')\n",
        "    #x_ticks = [x for x in range(0,n_epochs,2)]\n",
        "    #plt.xticks(x_ticks)\n",
        "    \n",
        "    ax2.plot(valid_accuracies, label = 'Validation accuracy')\n",
        "    ax2.legend(frameon=False)\n",
        "    ax2.set_xlabel('Epochs')\n",
        "    \n",
        "    plt.tight_layout()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IiK7hITUqQDr",
        "colab_type": "text"
      },
      "source": [
        "### Architecture & train functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WgT9WO7aqR5B",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "\n",
        "def weights_init_normal(m):\n",
        "    '''Takes in a module and initializes all linear layers with weight\n",
        "       values taken from a normal distribution.'''\n",
        "    classname = m.__class__.__name__\n",
        "    # for every Linear layer in a model\n",
        "    if classname.find('Linear') != -1:\n",
        "        n = m.in_features\n",
        "        # m.weight.data shoud be taken from a normal distribution\n",
        "        m.weight.data.normal_(0, 1/np.sqrt(n))\n",
        "        # m.bias.data should be 0\n",
        "        m.bias.data.fill_(0)\n",
        "        \n",
        "        \n",
        "# normalize each layer:\n",
        "class MyBatchNormLayer(nn.Module):\n",
        "    def __init__(self, input_number, filter_number, stride=2, kernel_size=3, padding=1):\n",
        "        super().__init__()\n",
        "        self.conv = nn.Conv2d(input_number, filter_number, kernel_size=kernel_size,\n",
        "                             stride=stride, bias=False, padding=1)\n",
        "        self.normalized = nn.BatchNorm2d(filter_number)\n",
        "    \n",
        "    def forward(self,x):\n",
        "        x = F.relu(self.conv(x))\n",
        "        #print('MyBatchNormLayer x.shape= ', x.shape)\n",
        "        return self.normalized(x)\n",
        "\n",
        "    \n",
        "class ResnetLayer(MyBatchNormLayer):\n",
        "    def forward(self, x): return x + super().forward(x)\n",
        "    \n",
        "    \n",
        "class MyConvBnNet(nn.Module):\n",
        "    def __init__(self, layers, classes_number, dropout=0):\n",
        "        super().__init__()\n",
        "        # define the first convolutional layer manually\n",
        "        self.conv1 = MyBatchNormLayer(3, 64,kernel_size=7, stride=2, padding=1)\n",
        "        self.pool1 = nn.MaxPool2d(3,2)\n",
        "        \n",
        "        self.layers1 = nn.ModuleList([MyBatchNormLayer(layers[i], layers[i+1]) for i in range(len(layers)-1)])\n",
        "        self.layers2 = nn.ModuleList([ResnetLayer(layers[i+1], layers[i+1], 1) for i in range(len(layers) - 1)])\n",
        "        self.layers3 = nn.ModuleList([ResnetLayer(layers[i+1], layers[i+1], 1) for i in range(len(layers) - 1)])\n",
        "        self.layers4 = nn.ModuleList([ResnetLayer(layers[i+1], layers[i+1], 1) for i in range(len(layers) - 1)])\n",
        "        self.layers5 = nn.ModuleList([ResnetLayer(layers[i+1], layers[i+1], 1) for i in range(len(layers) - 1)])\n",
        "        self.layers6 = nn.ModuleList([ResnetLayer(layers[i+1], layers[i+1], 1) for i in range(len(layers) - 1)])\n",
        "        self.layers7 = nn.ModuleList([ResnetLayer(layers[i+1], layers[i+1], 1) for i in range(len(layers) - 1)])\n",
        "        self.layers8 = nn.ModuleList([ResnetLayer(layers[i+1], layers[i+1], 1) for i in range(len(layers) - 1)])\n",
        "        \n",
        "        #self.linear1 = nn.Linear(layers[-1], 512)\n",
        "        self.outlayer = nn.Linear(layers[-1], classes_number)\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "        \n",
        "    def forward(self,x):\n",
        "        x = F.relu(self.conv1(x))\n",
        "        x = self.pool1(x)\n",
        "        for lyr1 ,lyr2, lyr3, lyr4, lyr5, lyr6 in zip(\n",
        "            self.layers1, self.layers2, self.layers3, self.layers4, self.layers5, self.layers6): \n",
        "            x = lyr4(lyr3(lyr2(lyr1(x))))\n",
        "            x =  lyr6(lyr5(x)) # lyr8(lyr7(lyr6(lyr5(x))))\n",
        "        x = F.adaptive_max_pool2d(x,1)\n",
        "        #print('MyConvBnNet after adaptive_max_pool2d, x.shape=', x.shape)\n",
        "        x = x.view(x.size(0), -1)\n",
        "        x = self.dropout(x)\n",
        "        #x = self.linear1(x)\n",
        "        x = self.outlayer(x)\n",
        "        return F.log_softmax(x, dim= -1)\n",
        "\n",
        "\n",
        "# check if CUDA is available\n",
        "use_cuda = torch.cuda.is_available()\n",
        "print('cuda available: ', use_cuda)\n",
        "\n",
        "# instantiate the CNN\n",
        "#model_scratch = MyConvBnNet([10, 20, 40], 133)\n",
        "model_scratch = MyConvBnNet([64, 128, 256, 512, 1024], 7, 0.1)\n",
        "print(model_scratch)\n",
        "\n",
        "model_scratch.apply(weights_init_normal)\n",
        "\n",
        "# move tensors to GPU if CUDA is available\n",
        "if use_cuda:\n",
        "    model_scratch.cuda()\n",
        "print('model is on cuda: ', next(model_scratch.parameters()).is_cuda)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FO8eNmEDqmbt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "def train_epoch(model, dataloader, criterion, optimizer, train_on_gpu):\n",
        "    # initialize variables to monitor training and validation loss\n",
        "    train_loss = 0.0\n",
        "    train_accuracy = 0.0\n",
        "    correct = 0.0\n",
        "    total = 0.0\n",
        "    \n",
        "    for batch_idx, (data, target) in enumerate(dataloader):\n",
        "        # move to GPU\n",
        "        if train_on_gpu:\n",
        "            data, target = data.cuda(), target.cuda()\n",
        "            \n",
        "        # clear the gradients of all optimized variables\n",
        "        optimizer.zero_grad()\n",
        "        \n",
        "        ## find the loss and update the model parameters accordingly\n",
        "        output = model(data)\n",
        "        loss = criterion(output, target)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        # get the loss per batch and accumulate\n",
        "        train_loss += loss.item()\n",
        "        \n",
        "        # get the class, highest probability\n",
        "        probabilities = torch.exp(output)\n",
        "        _, top_class = probabilities.topk(1, dim=1)\n",
        "        # The following line is equivalent to the previous (?)\n",
        "        #_, top_class = torch.max(probabilities, dim=1)\n",
        "        \n",
        "        # check if the predicted class is correct\n",
        "        equals = top_class == target.view(*top_class.shape)\n",
        "        # \n",
        "        train_accuracy += torch.mean(equals.type(torch.FloatTensor))\n",
        "\n",
        "    return train_loss, train_accuracy\n",
        "\n",
        "\n",
        "def validate_epoch(model, dataloader, criterion, train_on_gpu):\n",
        "    valid_loss = 0.0\n",
        "    valid_accuracy = 0.0\n",
        "    correct = 0.0\n",
        "    total = 0.0\n",
        "    \n",
        "    with torch.no_grad():\n",
        "        for batch_idx, (data, target) in enumerate(dataloader):\n",
        "            # move to GPU\n",
        "            if train_on_gpu:\n",
        "                data, target = data.cuda(), target.cuda()\n",
        "            ## update the average validation loss\n",
        "            output = model(data)\n",
        "            loss = criterion(output,target)\n",
        "            \n",
        "            valid_loss += loss.item()\n",
        "\n",
        "            ps = torch.exp(output)\n",
        "            _ , top_class = ps.topk(1,dim = 1)\n",
        "            #_, top_class = torch.max(ps, dim=1)\n",
        "            equals = top_class == target.view(*top_class.shape) # shape is (batch size x 1)\n",
        "            valid_accuracy += torch.mean(equals.type(torch.FloatTensor))\n",
        "\n",
        "    return valid_loss, valid_accuracy"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e7vq3pCoqsqZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "def train(n_epochs, loaders, model, optimizer, criterion, scheduler, use_cuda, save_path):\n",
        "    print('Training started at ', get_time())\n",
        "    \n",
        "    valid_losses = []\n",
        "    train_losses = []\n",
        "    valid_accuracies = []\n",
        "    \n",
        "    # initialize tracker for minimum validation loss\n",
        "    valid_loss_min = np.Inf \n",
        "    \n",
        "    for epoch in range(n_epochs):\n",
        "        \n",
        "         # initialize variables to monitor training and validation loss\n",
        "        training_loss = 0.0\n",
        "        training_accuracy = 0.0\n",
        "    \n",
        "        if scheduler is not None:\n",
        "          scheduler.step()\n",
        "        \n",
        "        ###################\n",
        "        # train the model #\n",
        "        model.train()\n",
        "        training_loss, training_accuracy = train_epoch(model, loaders['train'], criterion, optimizer, use_cuda)\n",
        "    \n",
        "        \n",
        "        ######################    \n",
        "        # validate the model #\n",
        "        model.eval()\n",
        "        validation_loss, validation_accuracy = validate_epoch(model, loaders['valid'], criterion, use_cuda)\n",
        "        \n",
        "        #if scheduler is not None:\n",
        "          #scheduler.step(validation_loss)\n",
        "        \n",
        "        ###### print training/validation statistics \n",
        "        # calculate the average loss per epoch\n",
        "        training_loss = training_loss/len(loaders['train'])\n",
        "        train_losses.append(training_loss)\n",
        "        \n",
        "        training_accuracy = training_accuracy/len(loaders['train'])\n",
        "        \n",
        "        validation_loss = validation_loss/len(loaders['valid'])\n",
        "        valid_losses.append(validation_loss)\n",
        "        \n",
        "        validation_accuracy = validation_accuracy/len(loaders['valid'])\n",
        "        valid_accuracies.append(validation_accuracy)\n",
        "        \n",
        "        hour, minute, second = get_time()\n",
        "        print('Epoch: {} at {}:{}:{} \\tTrain. Loss: {:.6f} \\tValid. Loss: {:.6f} \\t Accur.: {:.10f}'.format(\n",
        "                  epoch,\n",
        "                  hour, minute, second,\n",
        "                  training_loss,\n",
        "                  #training_accuracy, \n",
        "                  validation_loss,\n",
        "                  validation_accuracy ))\n",
        "        \n",
        "        ###### TODO: save the model if validation loss has decreased\n",
        "        if validation_loss <= valid_loss_min:\n",
        "            '''print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(\n",
        "                valid_loss_min,\n",
        "                validation_loss))'''\n",
        "            print('Validation loss decreased by {:.6f}'.format(validation_loss - valid_loss_min))\n",
        "            torch.save(model.state_dict(), save_path)\n",
        "            valid_loss_min = validation_loss\n",
        "            \n",
        "            \n",
        "    ##### visualize\n",
        "    plot_loss_acc(n_epochs, train_losses, valid_losses, valid_accuracies)\n",
        "    \n",
        "    return model\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BZE5GhsaQtgD",
        "colab_type": "text"
      },
      "source": [
        "### Hyper params & Start the training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7kkR-Ef3qx2B",
        "colab_type": "code",
        "outputId": "52e35bfb-8287-48a2-815f-7810ce94c82e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 715
        }
      },
      "source": [
        "\n",
        "epochs = 40\n",
        "\n",
        "### select loss function\n",
        "criterion_scratch = nn.NLLLoss()\n",
        "\n",
        "lr= 1e-4\n",
        "#optimizer_scratch = optim.SGD(model_scratch.parameters(), lr=0.001, momentum=0.9)\n",
        "optimizer_scratch = optim.Adam(model_scratch.parameters(), lr=lr)\n",
        "\n",
        "######### change the position of optim.step() in the train section\n",
        "gamma = 0.25\n",
        "#scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer_scratch, patience = 3, factor=0.1)\n",
        "scheduler_scratch = optim.lr_scheduler.MultiStepLR(optimizer_scratch, [10, 18, 26, 32, 36], gamma)\n",
        "stepsize = '10_18_26_32_36'\n",
        "\n",
        "# create model name\n",
        "save_path = './gdrive/My Drive/Colab Notebooks/Fer-dataset/'\n",
        "modelname = '{}cnn-{}-lr{}gamma{}-{}{}-epo{}.pt'.format(save_path, 'Adam', lr, gamma, 'Multi_', stepsize, epochs)\n",
        "\n",
        "\n",
        "######################################################\n",
        "############ start the training the model#############\n",
        "######################################################\n",
        "\n",
        "print(modelname)\n",
        "\n",
        "#def train(n_epochs, loaders, model, optimizer, criterion, scheduler, use_cuda, save_path):\n",
        "model_scratch = train(epochs, dataloaders, model_scratch, \n",
        "                      optimizer_scratch, criterion_scratch, scheduler_scratch,\n",
        "                      use_cuda, modelname)\n",
        "\n",
        "# load the model that got the best validation accuracy\n",
        "model_scratch.load_state_dict(torch.load(modelname))"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "./gdrive/My Drive/Colab Notebooks/Fer-dataset/cnn-Adam-lr0.0001gamma0.25-Multi_10_18_26_32_36-epo40.pt\n",
            "Training started at  (16, 34, 23)\n",
            "Epoch: 0 at 16:40:43 \tTrain. Loss: 1.258442 \tValid. Loss: 1.250045 \t Accur.: 0.5288925767\n",
            "Validation loss decreased by -inf\n",
            "Epoch: 1 at 16:47:8 \tTrain. Loss: 1.062675 \tValid. Loss: 1.289900 \t Accur.: 0.5160087943\n",
            "Epoch: 2 at 16:53:29 \tTrain. Loss: 0.960493 \tValid. Loss: 1.290856 \t Accur.: 0.5204495788\n",
            "Epoch: 3 at 16:59:50 \tTrain. Loss: 0.856625 \tValid. Loss: 1.203815 \t Accur.: 0.5729714632\n",
            "Validation loss decreased by -0.046231\n",
            "Epoch: 4 at 17:6:14 \tTrain. Loss: 0.738653 \tValid. Loss: 1.195024 \t Accur.: 0.5822368264\n",
            "Validation loss decreased by -0.008791\n",
            "Epoch: 5 at 17:12:38 \tTrain. Loss: 0.627804 \tValid. Loss: 1.203823 \t Accur.: 0.5820723176\n",
            "Epoch: 6 at 17:18:58 \tTrain. Loss: 0.513351 \tValid. Loss: 1.262835 \t Accur.: 0.5825109482\n",
            "Epoch: 7 at 17:25:18 \tTrain. Loss: 0.412372 \tValid. Loss: 1.282658 \t Accur.: 0.6019737124\n",
            "Epoch: 8 at 17:31:39 \tTrain. Loss: 0.402007 \tValid. Loss: 1.387586 \t Accur.: 0.5899670720\n",
            "Epoch: 9 at 17:38:2 \tTrain. Loss: 0.168181 \tValid. Loss: 1.374059 \t Accur.: 0.6040021777\n",
            "Epoch: 10 at 17:44:23 \tTrain. Loss: 0.101868 \tValid. Loss: 1.427137 \t Accur.: 0.6187499762\n",
            "Epoch: 11 at 17:50:43 \tTrain. Loss: 0.069564 \tValid. Loss: 1.500598 \t Accur.: 0.6084430218\n",
            "Epoch: 12 at 17:57:5 \tTrain. Loss: 0.075177 \tValid. Loss: 1.546456 \t Accur.: 0.6026315689\n",
            "Epoch: 13 at 18:3:28 \tTrain. Loss: 0.045175 \tValid. Loss: 1.611119 \t Accur.: 0.6116228104\n",
            "Epoch: 14 at 18:9:49 \tTrain. Loss: 0.037786 \tValid. Loss: 1.626273 \t Accur.: 0.6119517088\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-30-0f568524d989>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     29\u001b[0m model_scratch = train(epochs, dataloaders, model_scratch, \n\u001b[1;32m     30\u001b[0m                       \u001b[0moptimizer_scratch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion_scratch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscheduler_scratch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 31\u001b[0;31m                       use_cuda, modelname)\n\u001b[0m\u001b[1;32m     32\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m \u001b[0;31m# load the model that got the best validation accuracy\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-28-f6a72b69ef07>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(n_epochs, loaders, model, optimizer, criterion, scheduler, use_cuda, save_path)\u001b[0m\n\u001b[1;32m     22\u001b[0m         \u001b[0;31m# train the model #\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m         \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 24\u001b[0;31m         \u001b[0mtraining_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraining_accuracy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_epoch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloaders\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'train'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muse_cuda\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     25\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-24-17c6326c1829>\u001b[0m in \u001b[0;36mtrain_epoch\u001b[0;34m(model, dataloader, criterion, optimizer, train_on_gpu)\u001b[0m\n\u001b[1;32m     19\u001b[0m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     22\u001b[0m         \u001b[0;31m# get the loss per batch and accumulate\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m         \u001b[0mtrain_loss\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m     92\u001b[0m                 \u001b[0;31m# Decay the first and second moment running average coefficient\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m                 \u001b[0mexp_avg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbeta1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mbeta1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 94\u001b[0;31m                 \u001b[0mexp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbeta2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddcmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mbeta2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     95\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mamsgrad\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     96\u001b[0m                     \u001b[0;31m# Maintains the maximum of all 2nd moment running avg. till now\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KdtJmNd9tU_0",
        "colab_type": "text"
      },
      "source": [
        "### Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AeubTQlptWyj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def test(loaders, model, criterion, use_cuda):\n",
        "\n",
        "    # monitor test loss and accuracy\n",
        "    test_loss = 0.\n",
        "    correct = 0.\n",
        "    total = 0.\n",
        "    \n",
        "    print('test started at ', get_time())\n",
        "    model.eval()\n",
        "    for batch_idx, (data, target) in enumerate(loaders['test']):\n",
        "        # move to GPU\n",
        "        if use_cuda:\n",
        "            data, target = data.cuda(), target.cuda()\n",
        "        # forward pass: compute predicted outputs by passing inputs to the model\n",
        "        output = model(data)\n",
        "        # calculate the loss\n",
        "        loss = criterion(output, target)\n",
        "        # update average test loss \n",
        "        test_loss = test_loss + ((1 / (batch_idx + 1)) * (loss.data - test_loss))\n",
        "        # convert output probabilities to predicted class\n",
        "        pred = output.data.max(1, keepdim=True)[1]\n",
        "        # compare predictions to true label\n",
        "        correct += np.sum(np.squeeze(pred.eq(target.data.view_as(pred))).cpu().numpy())\n",
        "        total += data.size(0)\n",
        "            \n",
        "    print('test finished at ', get_time())\n",
        "    print('Test Loss: {:.6f}\\n'.format(test_loss))\n",
        "\n",
        "    print('\\nTest Accuracy: %2d%% (%2d/%2d)' % (\n",
        "        100. * correct / total, correct, total))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i3J2KYkRtZ-b",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "# call test function    \n",
        "test(dataloaders, model_scratch, criterion_scratch, use_cuda)\n"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}