DL4D_10_LeNet5.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "LeNet5.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "metadata": {
        "id": "nbfAnboFaTyb"
      },
      "cell_type": "code",
      "source": [
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "import numpy as np\n",
        "from keras.datasets import mnist\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Conv2D, AveragePooling2D\n",
        "from keras.layers import Dense, Flatten\n",
        "from keras.losses import categorical_crossentropy"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "metadata": {
        "id": "2y0VwkX0ac-e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "78dc0cbf-ee5f-4ced-d798-313c81a9d528"
      },
      "cell_type": "code",
      "source": [
        "(X_train, y_train), (X_test, y_test) = mnist.load_data()"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n",
            "11493376/11490434 [==============================] - 0s 0us/step\n",
            "11501568/11490434 [==============================] - 0s 0us/step\n"
          ]
        }
      ]
    },
    {
      "metadata": {
        "id": "Np0al_jRae03",
        "outputId": "ec0cc004-dc24-4bb8-b820-9ea3f43a61fa",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "cell_type": "code",
      "source": [
        "# transform targets into one-hot-encoded vectors\n",
        "num_classes = len(np.unique(y_train))\n",
        "print(y_train[0], end=' => ')\n",
        "y_train = keras.utils.to_categorical(y_train, 10)\n",
        "y_test = keras.utils.to_categorical(y_test, 10)\n",
        "print(y_train[0])"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "5 => [0. 0. 0. 0. 0. 1. 0. 0. 0. 0.]\n"
          ]
        }
      ]
    },
    {
      "metadata": {
        "id": "XndplEL7ag0u",
        "outputId": "021e8af5-6697-4fc8-e77f-c1bd0fe298fd",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "cell_type": "code",
      "source": [
        "# rescale 0-1 and cast training data as float32\n",
        "X_train = X_train.astype(np.float32) / 255\n",
        "X_test = X_test.astype(np.float32) / 255\n",
        "\n",
        "# reshape data to have also the channel dimension\n",
        "img_rows, img_cols = X_train.shape[1:]\n",
        "X_train = X_train.reshape(len(X_train), img_rows, img_cols, 1)\n",
        "X_test = X_test.reshape(len(X_test), img_rows, img_cols, 1)\n",
        "\n",
        "# notice the input shape\n",
        "input_shape = (img_rows, img_cols, 1)\n",
        "print(input_shape)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(28, 28, 1)\n"
          ]
        }
      ]
    },
    {
      "metadata": {
        "id": "wI6eDTO4a6b1"
      },
      "cell_type": "code",
      "source": [
        "lenet = Sequential()\n",
        "\n",
        "# Convolutional Layer C1\n",
        "lenet.add(Conv2D(6, kernel_size=(5, 5), activation='tanh', \n",
        "                 input_shape=input_shape, padding='same', name='C1'))\n",
        "\n",
        "# Pooling Layer S2\n",
        "lenet.add(AveragePooling2D(pool_size=(2, 2), name='S2'))\n",
        "\n",
        "# Convolutional Layer C3\n",
        "lenet.add(Conv2D(16, kernel_size=(5, 5), activation='tanh', name='C3'))\n",
        "\n",
        "# Pooling Layer S4\n",
        "lenet.add(AveragePooling2D(pool_size=(2, 2), name='S4'))\n",
        "\n",
        "# Fully Connected Convolutional Layer C5\n",
        "lenet.add(Conv2D(120, kernel_size=(5, 5), activation='tanh', name='C5'))\n",
        "\n",
        "# Fully Connected Layer FC6\n",
        "lenet.add(Flatten())\n",
        "lenet.add(Dense(84, activation='tanh', name='FC6'))\n",
        "\n",
        "#Output Layer (softmax activation)\n",
        "lenet.add(Dense(10, activation='softmax', name='OUTPUT'))"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "metadata": {
        "id": "abwGHbsJa9iW",
        "outputId": "3ce4d2d3-a3c7-4fe3-c58d-d87ef253aa14",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "cell_type": "code",
      "source": [
        "lenet.compile(loss=categorical_crossentropy, optimizer='SGD', metrics=['accuracy'])\n",
        "lenet.summary()"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " C1 (Conv2D)                 (None, 28, 28, 6)         156       \n",
            "                                                                 \n",
            " S2 (AveragePooling2D)       (None, 14, 14, 6)         0         \n",
            "                                                                 \n",
            " C3 (Conv2D)                 (None, 10, 10, 16)        2416      \n",
            "                                                                 \n",
            " S4 (AveragePooling2D)       (None, 5, 5, 16)          0         \n",
            "                                                                 \n",
            " C5 (Conv2D)                 (None, 1, 1, 120)         48120     \n",
            "                                                                 \n",
            " flatten (Flatten)           (None, 120)               0         \n",
            "                                                                 \n",
            " FC6 (Dense)                 (None, 84)                10164     \n",
            "                                                                 \n",
            " OUTPUT (Dense)              (None, 10)                850       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 61,706\n",
            "Trainable params: 61,706\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "metadata": {
        "id": "-eoP-iCZa-YO",
        "outputId": "66cffad3-35bc-4b69-cd6e-587f7f895400",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "cell_type": "code",
      "source": [
        "batch_size = 64\n",
        "epochs = 50\n",
        "history = lenet.fit(X_train, y_train,\n",
        "                      batch_size=batch_size,\n",
        "                      epochs=epochs,\n",
        "                      validation_data=(X_test, \n",
        "                                       y_test))"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/50\n",
            "938/938 [==============================] - 17s 5ms/step - loss: 0.9309 - accuracy: 0.7633 - val_loss: 0.4167 - val_accuracy: 0.8838\n",
            "Epoch 2/50\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.3716 - accuracy: 0.8934 - val_loss: 0.3102 - val_accuracy: 0.9113\n",
            "Epoch 3/50\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.2977 - accuracy: 0.9118 - val_loss: 0.2572 - val_accuracy: 0.9230\n",
            "Epoch 4/50\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.2529 - accuracy: 0.9248 - val_loss: 0.2205 - val_accuracy: 0.9345\n",
            "Epoch 5/50\n",
            "938/938 [==============================] - 4s 5ms/step - loss: 0.2192 - accuracy: 0.9352 - val_loss: 0.1934 - val_accuracy: 0.9432\n",
            "Epoch 6/50\n",
            "938/938 [==============================] - 4s 5ms/step - loss: 0.1922 - accuracy: 0.9423 - val_loss: 0.1693 - val_accuracy: 0.9505\n",
            "Epoch 7/50\n",
            "938/938 [==============================] - 4s 5ms/step - loss: 0.1704 - accuracy: 0.9485 - val_loss: 0.1523 - val_accuracy: 0.9555\n",
            "Epoch 8/50\n",
            "938/938 [==============================] - 4s 5ms/step - loss: 0.1524 - accuracy: 0.9546 - val_loss: 0.1357 - val_accuracy: 0.9596\n",
            "Epoch 9/50\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.1377 - accuracy: 0.9590 - val_loss: 0.1248 - val_accuracy: 0.9644\n",
            "Epoch 10/50\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.1253 - accuracy: 0.9627 - val_loss: 0.1128 - val_accuracy: 0.9672\n",
            "Epoch 11/50\n",
            "938/938 [==============================] - 5s 5ms/step - loss: 0.1147 - accuracy: 0.9656 - val_loss: 0.1028 - val_accuracy: 0.9710\n",
            "Epoch 12/50\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.1058 - accuracy: 0.9687 - val_loss: 0.0955 - val_accuracy: 0.9718\n",
            "Epoch 13/50\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0981 - accuracy: 0.9710 - val_loss: 0.0890 - val_accuracy: 0.9737\n",
            "Epoch 14/50\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0917 - accuracy: 0.9727 - val_loss: 0.0835 - val_accuracy: 0.9761\n",
            "Epoch 15/50\n",
            "938/938 [==============================] - 5s 5ms/step - loss: 0.0858 - accuracy: 0.9750 - val_loss: 0.0795 - val_accuracy: 0.9749\n",
            "Epoch 16/50\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0809 - accuracy: 0.9762 - val_loss: 0.0744 - val_accuracy: 0.9772\n",
            "Epoch 17/50\n",
            "938/938 [==============================] - 4s 5ms/step - loss: 0.0766 - accuracy: 0.9776 - val_loss: 0.0698 - val_accuracy: 0.9790\n",
            "Epoch 18/50\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0725 - accuracy: 0.9793 - val_loss: 0.0672 - val_accuracy: 0.9791\n",
            "Epoch 19/50\n",
            "938/938 [==============================] - 4s 5ms/step - loss: 0.0689 - accuracy: 0.9798 - val_loss: 0.0636 - val_accuracy: 0.9804\n",
            "Epoch 20/50\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0658 - accuracy: 0.9809 - val_loss: 0.0621 - val_accuracy: 0.9796\n",
            "Epoch 21/50\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0628 - accuracy: 0.9821 - val_loss: 0.0601 - val_accuracy: 0.9810\n",
            "Epoch 22/50\n",
            "938/938 [==============================] - 4s 5ms/step - loss: 0.0603 - accuracy: 0.9825 - val_loss: 0.0571 - val_accuracy: 0.9823\n",
            "Epoch 23/50\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0580 - accuracy: 0.9832 - val_loss: 0.0550 - val_accuracy: 0.9818\n",
            "Epoch 24/50\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0557 - accuracy: 0.9838 - val_loss: 0.0535 - val_accuracy: 0.9820\n",
            "Epoch 25/50\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0538 - accuracy: 0.9845 - val_loss: 0.0520 - val_accuracy: 0.9842\n",
            "Epoch 26/50\n",
            "938/938 [==============================] - 4s 5ms/step - loss: 0.0520 - accuracy: 0.9848 - val_loss: 0.0502 - val_accuracy: 0.9835\n",
            "Epoch 27/50\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0500 - accuracy: 0.9859 - val_loss: 0.0499 - val_accuracy: 0.9839\n",
            "Epoch 28/50\n",
            "938/938 [==============================] - 4s 5ms/step - loss: 0.0486 - accuracy: 0.9860 - val_loss: 0.0483 - val_accuracy: 0.9841\n",
            "Epoch 29/50\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0470 - accuracy: 0.9864 - val_loss: 0.0469 - val_accuracy: 0.9843\n",
            "Epoch 30/50\n",
            "938/938 [==============================] - 4s 5ms/step - loss: 0.0457 - accuracy: 0.9869 - val_loss: 0.0458 - val_accuracy: 0.9845\n",
            "Epoch 31/50\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0442 - accuracy: 0.9876 - val_loss: 0.0450 - val_accuracy: 0.9844\n",
            "Epoch 32/50\n",
            "938/938 [==============================] - 4s 5ms/step - loss: 0.0428 - accuracy: 0.9877 - val_loss: 0.0443 - val_accuracy: 0.9860\n",
            "Epoch 33/50\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0419 - accuracy: 0.9882 - val_loss: 0.0435 - val_accuracy: 0.9853\n",
            "Epoch 34/50\n",
            "938/938 [==============================] - 4s 5ms/step - loss: 0.0407 - accuracy: 0.9883 - val_loss: 0.0427 - val_accuracy: 0.9858\n",
            "Epoch 35/50\n",
            "938/938 [==============================] - 4s 5ms/step - loss: 0.0396 - accuracy: 0.9891 - val_loss: 0.0416 - val_accuracy: 0.9854\n",
            "Epoch 36/50\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0386 - accuracy: 0.9891 - val_loss: 0.0405 - val_accuracy: 0.9862\n",
            "Epoch 37/50\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0376 - accuracy: 0.9896 - val_loss: 0.0418 - val_accuracy: 0.9860\n",
            "Epoch 38/50\n",
            "938/938 [==============================] - 4s 5ms/step - loss: 0.0368 - accuracy: 0.9898 - val_loss: 0.0409 - val_accuracy: 0.9855\n",
            "Epoch 39/50\n",
            "938/938 [==============================] - 4s 5ms/step - loss: 0.0358 - accuracy: 0.9901 - val_loss: 0.0390 - val_accuracy: 0.9875\n",
            "Epoch 40/50\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0349 - accuracy: 0.9907 - val_loss: 0.0386 - val_accuracy: 0.9871\n",
            "Epoch 41/50\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0342 - accuracy: 0.9906 - val_loss: 0.0378 - val_accuracy: 0.9879\n",
            "Epoch 42/50\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0335 - accuracy: 0.9909 - val_loss: 0.0375 - val_accuracy: 0.9877\n",
            "Epoch 43/50\n",
            "938/938 [==============================] - 4s 5ms/step - loss: 0.0327 - accuracy: 0.9910 - val_loss: 0.0411 - val_accuracy: 0.9866\n",
            "Epoch 44/50\n",
            "938/938 [==============================] - 4s 5ms/step - loss: 0.0320 - accuracy: 0.9912 - val_loss: 0.0369 - val_accuracy: 0.9878\n",
            "Epoch 45/50\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0314 - accuracy: 0.9916 - val_loss: 0.0359 - val_accuracy: 0.9876\n",
            "Epoch 46/50\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0307 - accuracy: 0.9917 - val_loss: 0.0351 - val_accuracy: 0.9885\n",
            "Epoch 47/50\n",
            "938/938 [==============================] - 4s 5ms/step - loss: 0.0301 - accuracy: 0.9916 - val_loss: 0.0350 - val_accuracy: 0.9884\n",
            "Epoch 48/50\n",
            "938/938 [==============================] - 4s 5ms/step - loss: 0.0294 - accuracy: 0.9920 - val_loss: 0.0342 - val_accuracy: 0.9879\n",
            "Epoch 49/50\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0289 - accuracy: 0.9922 - val_loss: 0.0340 - val_accuracy: 0.9880\n",
            "Epoch 50/50\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0282 - accuracy: 0.9920 - val_loss: 0.0336 - val_accuracy: 0.9883\n"
          ]
        }
      ]
    }
  ]
}