lessons/week05/week05.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "toc_visible": true,
      "machine_shape": "hm",
      "gpuType": "L4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Classifying images"
      ],
      "metadata": {
        "id": "M1f38PnW3u5g"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Import"
      ],
      "metadata": {
        "id": "Kf3OvVTk3oz8"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8wjK319i20TH"
      },
      "outputs": [],
      "source": [
        "import random\n",
        "import numpy as np\n",
        "from PIL import Image\n",
        "\n",
        "import torch\n",
        "import torch.optim as optim\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "\n",
        "from torch.utils.data import DataLoader, Dataset, random_split, WeightedRandomSampler, SubsetRandomSampler\n",
        "from torchvision.transforms import Compose, ToTensor, Normalize, ToPILImage, RandomHorizontalFlip, Resize\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "plt.style.use('fivethirtyeight')\n",
        "%matplotlib inline"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Architecture class"
      ],
      "metadata": {
        "id": "h38OikK_-d4M"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class Architecture(object):\n",
        "    def __init__(self, model, loss_fn, optimizer):\n",
        "        # Here we define the attributes of our class\n",
        "\n",
        "        # We start by storing the arguments as attributes\n",
        "        # to use them later\n",
        "        self.model = model\n",
        "        self.loss_fn = loss_fn\n",
        "        self.optimizer = optimizer\n",
        "        self.device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "        # Let's send the model to the specified device right away\n",
        "        self.model.to(self.device)\n",
        "\n",
        "        # These attributes are defined here, but since they are\n",
        "        # not informed at the moment of creation, we keep them None\n",
        "        self.train_loader = None\n",
        "        self.val_loader = None\n",
        "\n",
        "        # These attributes are going to be computed internally\n",
        "        self.losses = []\n",
        "        self.val_losses = []\n",
        "        self.total_epochs = 0\n",
        "\n",
        "        # Creates the train_step function for our model,\n",
        "        # loss function and optimizer\n",
        "        # Note: there are NO ARGS there! It makes use of the class\n",
        "        # attributes directly\n",
        "        self.train_step_fn = self._make_train_step_fn()\n",
        "        # Creates the val_step function for our model and loss\n",
        "        self.val_step_fn = self._make_val_step_fn()\n",
        "\n",
        "    def to(self, device):\n",
        "        # This method allows the user to specify a different device\n",
        "        # It sets the corresponding attribute (to be used later in\n",
        "        # the mini-batches) and sends the model to the device\n",
        "        try:\n",
        "            self.device = device\n",
        "            self.model.to(self.device)\n",
        "        except RuntimeError:\n",
        "            self.device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "            print(f\"Couldn't send it to {device}, sending it to {self.device} instead.\")\n",
        "            self.model.to(self.device)\n",
        "\n",
        "    def set_loaders(self, train_loader, val_loader=None):\n",
        "        # This method allows the user to define which train_loader (and val_loader, optionally) to use\n",
        "        # Both loaders are then assigned to attributes of the class\n",
        "        # So they can be referred to later\n",
        "        self.train_loader = train_loader\n",
        "        self.val_loader = val_loader\n",
        "\n",
        "    def _make_train_step_fn(self):\n",
        "        # This method does not need ARGS... it can refer to\n",
        "        # the attributes: self.model, self.loss_fn and self.optimizer\n",
        "\n",
        "        # Builds function that performs a step in the train loop\n",
        "        def perform_train_step_fn(x, y):\n",
        "            # Sets model to TRAIN mode\n",
        "            self.model.train()\n",
        "\n",
        "            # Step 1 - Computes our model's predicted output - forward pass\n",
        "            yhat = self.model(x)\n",
        "            # Step 2 - Computes the loss\n",
        "            loss = self.loss_fn(yhat, y)\n",
        "            # Step 3 - Computes gradients for both \"a\" and \"b\" parameters\n",
        "            loss.backward()\n",
        "            # Step 4 - Updates parameters using gradients and the learning rate\n",
        "            self.optimizer.step()\n",
        "            self.optimizer.zero_grad()\n",
        "\n",
        "            # Returns the loss\n",
        "            return loss.item()\n",
        "\n",
        "        # Returns the function that will be called inside the train loop\n",
        "        return perform_train_step_fn\n",
        "\n",
        "    def _make_val_step_fn(self):\n",
        "        # Builds function that performs a step in the validation loop\n",
        "        def perform_val_step_fn(x, y):\n",
        "            # Sets model to EVAL mode\n",
        "            self.model.eval()\n",
        "\n",
        "            # Step 1 - Computes our model's predicted output - forward pass\n",
        "            yhat = self.model(x)\n",
        "            # Step 2 - Computes the loss\n",
        "            loss = self.loss_fn(yhat, y)\n",
        "            # There is no need to compute Steps 3 and 4, since we don't update parameters during evaluation\n",
        "            return loss.item()\n",
        "\n",
        "        return perform_val_step_fn\n",
        "\n",
        "    def _mini_batch(self, validation=False):\n",
        "        # The mini-batch can be used with both loaders\n",
        "        # The argument `validation`defines which loader and\n",
        "        # corresponding step function is going to be used\n",
        "        if validation:\n",
        "            data_loader = self.val_loader\n",
        "            step_fn = self.val_step_fn\n",
        "        else:\n",
        "            data_loader = self.train_loader\n",
        "            step_fn = self.train_step_fn\n",
        "\n",
        "        if data_loader is None:\n",
        "            return None\n",
        "\n",
        "        # Once the data loader and step function, this is the same\n",
        "        # mini-batch loop we had before\n",
        "        mini_batch_losses = []\n",
        "        for x_batch, y_batch in data_loader:\n",
        "            x_batch = x_batch.to(self.device)\n",
        "            y_batch = y_batch.to(self.device)\n",
        "\n",
        "            mini_batch_loss = step_fn(x_batch, y_batch)\n",
        "            mini_batch_losses.append(mini_batch_loss)\n",
        "\n",
        "        loss = np.mean(mini_batch_losses)\n",
        "        return loss\n",
        "\n",
        "    # this function was updated in this class\n",
        "    def set_seed(self, seed=42):\n",
        "        torch.backends.cudnn.deterministic = True\n",
        "        torch.backends.cudnn.benchmark = False\n",
        "        torch.manual_seed(seed)\n",
        "        np.random.seed(seed)\n",
        "        random.seed(seed)\n",
        "        try:\n",
        "            self.train_loader.sampler.generator.manual_seed(seed)\n",
        "        except AttributeError:\n",
        "            pass\n",
        "\n",
        "    def train(self, n_epochs, seed=42):\n",
        "        # To ensure reproducibility of the training process\n",
        "        self.set_seed(seed)\n",
        "\n",
        "        for epoch in range(n_epochs):\n",
        "            # Keeps track of the numbers of epochs\n",
        "            # by updating the corresponding attribute\n",
        "            self.total_epochs += 1\n",
        "\n",
        "            # inner loop\n",
        "            # Performs training using mini-batches\n",
        "            loss = self._mini_batch(validation=False)\n",
        "            self.losses.append(loss)\n",
        "\n",
        "            # VALIDATION\n",
        "            # no gradients in validation!\n",
        "            with torch.no_grad():\n",
        "                # Performs evaluation using mini-batches\n",
        "                val_loss = self._mini_batch(validation=True)\n",
        "                self.val_losses.append(val_loss)\n",
        "\n",
        "    def save_checkpoint(self, filename):\n",
        "        # Builds dictionary with all elements for resuming training\n",
        "        checkpoint = {'epoch': self.total_epochs,\n",
        "                      'model_state_dict': self.model.state_dict(),\n",
        "                      'optimizer_state_dict': self.optimizer.state_dict(),\n",
        "                      'loss': self.losses,\n",
        "                      'val_loss': self.val_losses}\n",
        "\n",
        "        torch.save(checkpoint, filename)\n",
        "\n",
        "    def load_checkpoint(self, filename):\n",
        "        # Loads dictionary\n",
        "        checkpoint = torch.load(filename)\n",
        "\n",
        "        # Restore state for model and optimizer\n",
        "        self.model.load_state_dict(checkpoint['model_state_dict'])\n",
        "        self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n",
        "\n",
        "        self.total_epochs = checkpoint['epoch']\n",
        "        self.losses = checkpoint['loss']\n",
        "        self.val_losses = checkpoint['val_loss']\n",
        "\n",
        "        self.model.train() # always use TRAIN for resuming training\n",
        "\n",
        "    def predict(self, x):\n",
        "        # Set is to evaluation mode for predictions\n",
        "        self.model.eval()\n",
        "        # Takes aNumpy input and make it a float tensor\n",
        "        x_tensor = torch.as_tensor(x).float()\n",
        "        # Send input to device and uses model for prediction\n",
        "        y_hat_tensor = self.model(x_tensor.to(self.device))\n",
        "        # Set it back to train mode\n",
        "        self.model.train()\n",
        "        # Detaches it, brings it to CPU and back to Numpy\n",
        "        return y_hat_tensor.detach().cpu().numpy()\n",
        "\n",
        "    def count_parameters(self):\n",
        "      return sum(p.numel() for p in self.model.parameters() if p.requires_grad)\n",
        "\n",
        "    def plot_losses(self):\n",
        "        fig = plt.figure(figsize=(10, 4))\n",
        "        plt.plot(self.losses, label='Training Loss', c='b')\n",
        "        plt.plot(self.val_losses, label='Validation Loss', c='r')\n",
        "        plt.yscale('log')\n",
        "        plt.xlabel('Epochs')\n",
        "        plt.ylabel('Loss')\n",
        "        plt.legend()\n",
        "        plt.tight_layout()\n",
        "        return fig"
      ],
      "metadata": {
        "id": "tYxCEnws8T7L"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KrYy397h45q8"
      },
      "source": [
        "## Data Generation"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def gen_img(start, target, fill=1, img_size=10):\n",
        "    # Create an image of specified size filled with zeros, data type is float for pixel values.\n",
        "    img = np.zeros((img_size, img_size), dtype=float)\n",
        "\n",
        "    # Initialize start_row and start_col to None. They are used to determine the drawing start point.\n",
        "    start_row, start_col = None, None\n",
        "\n",
        "    # If start is positive, it indicates a row, else start_col is the absolute value of start.\n",
        "    if start > 0:\n",
        "        start_row = start\n",
        "    else:\n",
        "        start_col = np.abs(start)\n",
        "\n",
        "    # If target is 0, fill a row or column based on whether start is a row or a column.\n",
        "    if target == 0:\n",
        "        if start_row is None:\n",
        "            img[:, start_col] = fill  # Fill the entire column with 'fill' value.\n",
        "        else:\n",
        "            img[start_row, :] = fill  # Fill the entire row with 'fill' value.\n",
        "    else:\n",
        "        # If start_col is 0, adjust to 1 to prevent out of index errors for negative indices.\n",
        "        if start_col == 0:\n",
        "            start_col = 1\n",
        "\n",
        "        # If target is 1, fill diagonally upwards depending on start_row or start_col.\n",
        "        if target == 1:\n",
        "            if start_row is not None:\n",
        "                up = (range(start_row, -1, -1),  # Decrement row index from start_row to 0.\n",
        "                      range(0, start_row + 1))  # Increment column index from 0 to start_row.\n",
        "            else:\n",
        "                up = (range(img_size - 1, start_col - 1, -1),  # Decrement row index starting near the bottom.\n",
        "                      range(start_col, img_size))  # Increment column index from start_col to the end.\n",
        "            img[up] = fill\n",
        "        else:\n",
        "            # If target is not 1, fill diagonally downwards depending on start_row or start_col.\n",
        "            if start_row is not None:\n",
        "                down = (range(start_row, img_size, 1),  # Increment row index from start_row to the end.\n",
        "                        range(0, img_size - start_row))  # Increment column index across the available width.\n",
        "            else:\n",
        "                down = (range(0, img_size - 1 - start_col + 1),  # Increment row index from 0 to an adjusted max.\n",
        "                        range(start_col, img_size))  # Increment column index from start_col to the end.\n",
        "            img[down] = fill\n",
        "\n",
        "    # Multiply all pixel values by 255 (for visualizing if needed) and reshape to a 3D array suitable for image handling.\n",
        "    return 255 * img.reshape(1, img_size, img_size)"
      ],
      "metadata": {
        "id": "ZtzNDFHO5Dx-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def generate_dataset(img_size=10, n_images=100, binary=True, seed=17):\n",
        "    \"\"\"\n",
        "    Generates a dataset of images with specified patterns and corresponding target values.\n",
        "\n",
        "    This function creates a dataset consisting of images generated based on random start positions\n",
        "    and target patterns. Each image is created using the `gen_img` function which defines\n",
        "    different filling patterns based on the target value. The function allows for control over the\n",
        "    size of the images, the number of images, whether the targets are binary, and the random seed\n",
        "    for reproducibility.\n",
        "\n",
        "    Parameters:\n",
        "        img_size (int): Size of the square image (default is 10x10 pixels).\n",
        "        n_images (int): Number of images to generate in the dataset.\n",
        "        binary (bool): If True, target values will be converted to binary (0, 1).\n",
        "                       0 remains 0, and 1 or 2 are converted to 1. Default is True.\n",
        "        seed (int): Seed for random number generation to ensure reproducibility. Default is 17.\n",
        "\n",
        "    Returns:\n",
        "        tuple: A tuple containing two elements:\n",
        "            - numpy.ndarray: An array of images, each shaped as (1, img_size, img_size) and having uint8 type.\n",
        "            - numpy.ndarray: An array of target values, either integer (0, 1, 2) or binary (0, 1),\n",
        "                             based on the 'binary' parameter.\n",
        "\n",
        "    Example:\n",
        "        >>> images, targets = generate_dataset(img_size=8, n_images=50, binary=False, seed=42)\n",
        "        >>> print(images.shape)\n",
        "        (50, 1, 8, 8)\n",
        "        >>> print(targets)\n",
        "        [2, 0, 1, ...]\n",
        "    \"\"\"\n",
        "    # Set the random seed for reproducibility of results\n",
        "    np.random.seed(seed)\n",
        "\n",
        "    # Generate random 'start' positions for each image within the allowed range.\n",
        "    # Negative values indicate columns, positive values indicate rows.\n",
        "    starts = np.random.randint(-(img_size - 1), img_size, size=(n_images,))\n",
        "\n",
        "    # Generate random 'target' codes for each image. The target dictates the filling pattern:\n",
        "    # 0 = fill a row or column, 1 = fill diagonally up, 2 = fill diagonally down.\n",
        "    targets = np.random.randint(0, 3, size=(n_images,))\n",
        "\n",
        "    # Create an array of images. For each start and target pair, generate an image using the gen_img function.\n",
        "    images = np.array([gen_img(s, t, img_size=img_size)\n",
        "                       for s, t in zip(starts, targets)], dtype=np.uint8)\n",
        "\n",
        "    # If binary is True, convert the targets to a binary format:\n",
        "    # 0 remains 0, 1 and 2 are converted to 1. This is typically used for classification tasks.\n",
        "    if binary:\n",
        "        targets = (targets > 0).astype(int)\n",
        "\n",
        "    # Return the array of generated images and the possibly converted target array.\n",
        "    return images, targets"
      ],
      "metadata": {
        "id": "whqS0WCy5aCa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def plot_images(images, targets, n_plot=30):\n",
        "    n_rows = n_plot // 6 + ((n_plot % 6) > 0)\n",
        "    fig, axes = plt.subplots(n_rows, 6, figsize=(9, 1.5 * n_rows))\n",
        "    axes = np.atleast_2d(axes)\n",
        "\n",
        "    for i, (image, target) in enumerate(zip(images[:n_plot], targets[:n_plot])):\n",
        "        row, col = i // 6, i % 6\n",
        "        ax = axes[row, col]\n",
        "        ax.set_title('#{} - Label:{}'.format(i, target), {'size': 12})\n",
        "        # plot filter channel in grayscale\n",
        "        ax.imshow(image.squeeze(), cmap='gray', vmin=0, vmax=1)\n",
        "\n",
        "    for ax in axes.flat:\n",
        "        ax.set_xticks([])\n",
        "        ax.set_yticks([])\n",
        "        ax.label_outer()\n",
        "\n",
        "    plt.tight_layout()\n",
        "    return fig"
      ],
      "metadata": {
        "id": "jYXR7Ymy6hZx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bYXgxpjk45q8"
      },
      "outputs": [],
      "source": [
        "images, labels = generate_dataset(img_size=5, n_images=300, binary=True, seed=13)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "images[0]"
      ],
      "metadata": {
        "id": "Nn23jImg6nRO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "labels[0]"
      ],
      "metadata": {
        "id": "7WEHXcub6tDB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tTgxjJhf45q8"
      },
      "outputs": [],
      "source": [
        "fig = plot_images(images, labels, n_plot=30)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LPX8BXqp45q9"
      },
      "source": [
        "### Images and Channels"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def image_channels(red, green, blue, rgb, gray, rows=(0, 1, 2)):\n",
        "    fig, axs = plt.subplots(len(rows), 4, figsize=(15, 5.5))\n",
        "\n",
        "    zeros = np.zeros((5, 5), dtype=np.uint8)\n",
        "\n",
        "    titles1 = ['Red', 'Green', 'Blue', 'Grayscale Image']\n",
        "    titles0 = ['image_r', 'image_g', 'image_b', 'image_gray']\n",
        "    titles2 = ['as first channel', 'as second channel', 'as third channel', 'RGB Image']\n",
        "\n",
        "    idx0 = np.argmax(np.array(rows) == 0)\n",
        "    idx1 = np.argmax(np.array(rows) == 1)\n",
        "    idx2 = np.argmax(np.array(rows) == 2)\n",
        "\n",
        "    for i, m in enumerate([red, green, blue, gray]):\n",
        "        if 0 in rows:\n",
        "            axs[idx0, i].axis('off')\n",
        "            axs[idx0, i].invert_yaxis()\n",
        "            if (1 in rows) or (i < 3):\n",
        "                axs[idx0, i].text(0.15, 0.25, str(m.astype(np.uint8)), verticalalignment='top')\n",
        "                axs[idx0, i].set_title(titles0[i], fontsize=16)\n",
        "\n",
        "        if 1 in rows:\n",
        "            axs[idx1, i].set_title(titles1[i], fontsize=16)\n",
        "            axs[idx1, i].set_xlabel('5x5', fontsize=14)\n",
        "            axs[idx1, i].imshow(m, cmap=plt.cm.gray)\n",
        "\n",
        "        if 2 in rows:\n",
        "            axs[idx2, i].set_title(titles2[i], fontsize=16)\n",
        "            axs[idx2, i].set_xlabel(f'5x5x3 - {titles1[i][0]} only', fontsize=14)\n",
        "            if i < 3:\n",
        "                stacked = [zeros] * 3\n",
        "                stacked[i] = m\n",
        "                axs[idx2, i].imshow(np.stack(stacked, axis=2))\n",
        "            else:\n",
        "                axs[idx2, i].imshow(rgb)\n",
        "\n",
        "        for r in [1, 2]:\n",
        "            if r in rows:\n",
        "                idx = idx1 if r == 1 else idx2\n",
        "                axs[idx, i].set_xticks([])\n",
        "                axs[idx, i].set_yticks([])\n",
        "                for k, v in axs[idx, i].spines.items():\n",
        "                    v.set_color('black')\n",
        "                    v.set_linewidth(.8)\n",
        "\n",
        "    if 1 in rows:\n",
        "        axs[idx1, 0].set_ylabel('Single\\nChannel\\n(grayscale)', rotation=0, labelpad=40, fontsize=12)\n",
        "        axs[idx1, 3].set_xlabel('5x5 = 0.21R + 0.72G + 0.07B')\n",
        "    if 2 in rows:\n",
        "        axs[idx2, 0].set_ylabel('Three\\nChannels\\n(color)', rotation=0, labelpad=40, fontsize=12)\n",
        "        axs[idx2, 3].set_xlabel('5x5x3 = (R, G, B) stacked')\n",
        "    fig.tight_layout()\n",
        "    return fig"
      ],
      "metadata": {
        "id": "e6k0cfsp-02y"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "A5MJ81ia45q9"
      },
      "outputs": [],
      "source": [
        "image_r  = np.zeros((5, 5), dtype=np.uint8)\n",
        "image_r[:, 0] = 255\n",
        "image_r[:, 1] = 128\n",
        "\n",
        "image_g = np.zeros((5, 5), dtype=np.uint8)\n",
        "image_g[:, 1] = 128\n",
        "image_g[:, 2] = 255\n",
        "image_g[:, 3] = 128\n",
        "\n",
        "image_b = np.zeros((5, 5), dtype=np.uint8)\n",
        "image_b[:, 3] = 128\n",
        "image_b[:, 4] = 255"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lPpk5WL545q9"
      },
      "outputs": [],
      "source": [
        "image_gray = .2126*image_r + .7152*image_g + .0722*image_b"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vVgLYvga45q9"
      },
      "outputs": [],
      "source": [
        "image_rgb = np.stack([image_r, image_g, image_b], axis=2)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "image_rgb.shape"
      ],
      "metadata": {
        "id": "aU7ARCdb9vQp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LY8nOjBc45q9"
      },
      "outputs": [],
      "source": [
        "fig = image_channels(image_r, image_g, image_b, image_rgb, image_gray, rows=(0, 1))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TjWZrzGm45q9"
      },
      "outputs": [],
      "source": [
        "fig = image_channels(image_r, image_g, image_b, image_rgb, image_gray, rows=(0, 2))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hWBv28Bg45q9"
      },
      "source": [
        "#### NCHW vs NHWC"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pQSC4d1U45q9"
      },
      "outputs": [],
      "source": [
        "images.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YmLlDUVf45q9"
      },
      "outputs": [],
      "source": [
        "example = images[7]\n",
        "example"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "v-hUq4xy45q9"
      },
      "outputs": [],
      "source": [
        "# PIL shape\n",
        "example_hwc = np.transpose(example, (1, 2, 0))\n",
        "example_hwc.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zASPz0th45q9"
      },
      "outputs": [],
      "source": [
        "example_hwc"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Torchvision"
      ],
      "metadata": {
        "id": "4HybDbXZAwuj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from torchvision.transforms import Compose, ToTensor\n",
        "from torchvision.transforms import Normalize, ToPILImage\n",
        "from torchvision.transforms import RandomHorizontalFlip, Resize"
      ],
      "metadata": {
        "id": "wjlQtlWFdU8d"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Transforms"
      ],
      "metadata": {
        "id": "e3MuxB4icqzU"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "> **ToTensor()** may scale the values from a [0, 255] range to a [0.0, 1.0] range if the input is either a Numpy array with dtype equal to uint8 (as in our example) or a PIL image belonging to one of the\n",
        "following modes (L, LA, P, I, F, RGB, YCbCr, RGBA, CMYK)."
      ],
      "metadata": {
        "id": "1wUKEw8abaox"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# ToTensor() to convert a Numpy array (in PIL shape) to a\n",
        "# PyTorch tensor.\n",
        "tensorizer = ToTensor()\n",
        "example_tensor = tensorizer(example_hwc)\n",
        "example_tensor.shape"
      ],
      "metadata": {
        "id": "9IPQngdmZkc7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(type(example_tensor))\n",
        "example_tensor"
      ],
      "metadata": {
        "id": "Yu8J9tb6aaVe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# ToPILImage() to convert a tensor to a PIL image\n",
        "example_img = ToPILImage()(example_tensor)\n",
        "print(type(example_img))"
      ],
      "metadata": {
        "id": "XppQAf4cbDYT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "example_img"
      ],
      "metadata": {
        "id": "YPWMOJd4cHRh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.imshow(example_img, cmap='gray')\n",
        "plt.grid(False)"
      ],
      "metadata": {
        "id": "x7fscmiBcIfM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# transforms on image\n",
        "flipper = RandomHorizontalFlip(p=1.0)\n",
        "flipped_img = flipper(example_img)"
      ],
      "metadata": {
        "id": "uwLIDeDtcVgz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.imshow(flipped_img, cmap='gray')\n",
        "plt.grid(False)"
      ],
      "metadata": {
        "id": "-HACZUj3c_hD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# transforms on tensor\n",
        "img_tensor = tensorizer(flipped_img)\n",
        "img_tensor"
      ],
      "metadata": {
        "id": "QnwAdQucdCDm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "$$\n",
        "\\Large\n",
        "\\begin{aligned}\n",
        "\\text{input} = 0 \\implies \\frac{0 - \\text{mean}}{\\text{std}}= \\frac{0 - 0.5}{0.5}&=-1\n",
        "\\\\\n",
        "\\text{input} = 1 \\implies \\frac{1 - \\text{mean}}{\\text{std}}= \\frac{1 - 0.5}{0.5}&=1\n",
        "\\end{aligned}\n",
        "$$"
      ],
      "metadata": {
        "id": "5PIa1NfwdP29"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "normalizer = Normalize(mean=(.5,), std=(.5,))\n",
        "normalized_tensor = normalizer(img_tensor)\n",
        "normalized_tensor"
      ],
      "metadata": {
        "id": "MwuBckLTgzBZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "S7u6wb8Z45q-"
      },
      "source": [
        "### Composing Transforms"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CFFNQATp45q-"
      },
      "outputs": [],
      "source": [
        "composer = Compose([RandomHorizontalFlip(p=1.0),\n",
        "                    Normalize(mean=(.5,), std=(.5,))])"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "example_tensor"
      ],
      "metadata": {
        "id": "m1SQwkufhQB2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hKe7lEVN45q-"
      },
      "outputs": [],
      "source": [
        "composed_tensor = composer(example_tensor)\n",
        "(composed_tensor == normalized_tensor).all()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "composed_tensor"
      ],
      "metadata": {
        "id": "D9fTIcq7hdgr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "p4J6S9Hu45q-"
      },
      "outputs": [],
      "source": [
        "print(example)\n",
        "print(example_tensor)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "P-GnGzOj45q-"
      },
      "outputs": [],
      "source": [
        "example_tensor = torch.as_tensor(example / 255).float()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "example_tensor"
      ],
      "metadata": {
        "id": "OqIaBZZfdnT7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Data Preparation"
      ],
      "metadata": {
        "id": "vM3CObUwhWAV"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Step 1: Features and labels to tensors"
      ],
      "metadata": {
        "id": "D2aZbLFUcRML"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "labels.shape"
      ],
      "metadata": {
        "id": "30MJu5caW3l6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Builds tensors from numpy arrays BEFORE split\n",
        "x_tensor = torch.as_tensor(images / 255).float()\n",
        "y_tensor = torch.as_tensor(labels.reshape(-1, 1)).float()"
      ],
      "metadata": {
        "id": "p9gHn-snWbtw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Step 2: Build a custom dataset that is capable of handling transformations\n"
      ],
      "metadata": {
        "id": "e5suhtzcW8oZ"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wCRx5cWY45rG"
      },
      "outputs": [],
      "source": [
        "class TransformedTensorDataset(Dataset):\n",
        "    def __init__(self, x, y, transform=None):\n",
        "        self.x = x\n",
        "        self.y = y\n",
        "        self.transform = transform\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        x = self.x[index]\n",
        "\n",
        "        if self.transform:\n",
        "            x = self.transform(x)\n",
        "\n",
        "        return x, self.y[index]\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.x)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HvMSCYqj45rG"
      },
      "outputs": [],
      "source": [
        "composer = Compose([RandomHorizontalFlip(p=0.5),\n",
        "                    Normalize(mean=(.5,), std=(.5,))])\n",
        "\n",
        "dataset = TransformedTensorDataset(x_tensor, y_tensor, transform=composer)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dataset[1]"
      ],
      "metadata": {
        "id": "xOWE8nLjiCKg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Step 3: Splitting a dataset"
      ],
      "metadata": {
        "id": "WLSj_Tczczd1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# build a custom spliter\n",
        "def index_splitter(n, splits, seed=13):\n",
        "    idx = torch.arange(n)\n",
        "    # Makes the split argument a tensor\n",
        "    splits_tensor = torch.as_tensor(splits)\n",
        "    # Finds the correct multiplier, so we don't have\n",
        "    # to worry about summing up to N (or one)\n",
        "    multiplier = n / splits_tensor.sum()\n",
        "    splits_tensor = (multiplier * splits_tensor).long()\n",
        "    # If there is a difference, throws at the first split\n",
        "    # so random_split does not complain\n",
        "    diff = n - splits_tensor.sum()\n",
        "    splits_tensor[0] += diff\n",
        "    # Uses PyTorch random_split to split the indices\n",
        "    torch.manual_seed(seed)\n",
        "    return random_split(idx, splits_tensor)"
      ],
      "metadata": {
        "id": "06vTxMDXh87k"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_idx, val_idx = index_splitter(len(x_tensor), [80, 20])\n",
        "train_idx"
      ],
      "metadata": {
        "id": "aEQsgXws7o1-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_idx, val_idx = index_splitter(len(x_tensor), [0.8, 0.2])\n",
        "train_idx"
      ],
      "metadata": {
        "id": "5aDoBUKN7tAI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_idx.indices"
      ],
      "metadata": {
        "id": "7MEF0tTR7xRW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(len(x_tensor),len(train_idx),len(val_idx))"
      ],
      "metadata": {
        "id": "jn6w91XT7ziq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Build a custom sampler"
      ],
      "metadata": {
        "id": "KpnapKF-rd2E"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "The SubsetRandomSampler samples indices from a list, given as argument, without\n",
        "replacement. As in the other samplers, these indices are used to load data from a\n",
        "dataset. If an index is not on the list, the corresponding data point will never be\n",
        "used."
      ],
      "metadata": {
        "id": "l2ErYsDo_-CK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "train_sampler = SubsetRandomSampler(train_idx)\n",
        "val_sampler = SubsetRandomSampler(val_idx)"
      ],
      "metadata": {
        "id": "9VIQTrmH8BdB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Builds a loader of each set\n",
        "train_loader = DataLoader(dataset=dataset, batch_size=16, sampler=train_sampler)\n",
        "val_loader = DataLoader(dataset=dataset, batch_size=16, sampler=val_sampler)"
      ],
      "metadata": {
        "id": "7b5eaxdk8K7L"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(iter(train_loader)), len(iter(val_loader))"
      ],
      "metadata": {
        "id": "yD3Wswmo9GZ3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Leading with an imbalanced dataset: build a weighted sampler"
      ],
      "metadata": {
        "id": "DrATyeZvDelI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "x_train_tensor = x_tensor[train_idx]\n",
        "y_train_tensor = y_tensor[train_idx]\n",
        "\n",
        "x_val_tensor = x_tensor[val_idx]\n",
        "y_val_tensor = y_tensor[val_idx]"
      ],
      "metadata": {
        "id": "dVQJ8vntvGsP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "U8z5_OrK45rH"
      },
      "outputs": [],
      "source": [
        "classes, counts = y_train_tensor.unique(return_counts=True)\n",
        "print(classes, counts)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The class with fewer data points (minority class) should get larger weights, while the class with more data points (majority class) should get smaller weights. This way, on average, we’ll end up with mini-batches containing\n",
        "roughly the same number of data points in each class: **A balanced dataset**."
      ],
      "metadata": {
        "id": "DuuYlJupvRcU"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "scrolled": true,
        "id": "iEKlC9Yz45rH"
      },
      "outputs": [],
      "source": [
        "weights = 1.0 / counts.float()\n",
        "weights"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "y_train_tensor.squeeze().long()"
      ],
      "metadata": {
        "id": "Zz6hIxSsyY2U"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "weights[y_train_tensor.squeeze().long()][:10]"
      ],
      "metadata": {
        "id": "BEs2lwiizI12"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_IXGga1d45rH"
      },
      "outputs": [],
      "source": [
        "sample_weights = weights[y_train_tensor.squeeze().long()]\n",
        "\n",
        "print(sample_weights.shape)\n",
        "print(sample_weights[:10])\n",
        "print(y_train_tensor[:10].squeeze())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KEbNi4mN45rH"
      },
      "outputs": [],
      "source": [
        "generator = torch.Generator()\n",
        "\n",
        "sampler = WeightedRandomSampler(\n",
        "    weights=sample_weights,\n",
        "    num_samples=len(sample_weights),\n",
        "    generator=generator,\n",
        "    replacement=True\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IUK75YDC45rH"
      },
      "outputs": [],
      "source": [
        "def make_balanced_sampler(y):\n",
        "    # Computes weights for compensating imbalanced classes\n",
        "    classes, counts = y.unique(return_counts=True)\n",
        "    weights = 1.0 / counts.float()\n",
        "    sample_weights = weights[y.squeeze().long()]\n",
        "    # Builds sampler with compute weights\n",
        "    generator = torch.Generator()\n",
        "    sampler = WeightedRandomSampler(\n",
        "        weights=sample_weights,\n",
        "        num_samples=len(sample_weights),\n",
        "        generator=generator,\n",
        "        replacement=True\n",
        "    )\n",
        "    return sampler"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rIHbqtgb45rI"
      },
      "outputs": [],
      "source": [
        "sampler = make_balanced_sampler(y_train_tensor)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xskRxx4045rH"
      },
      "source": [
        "#### Data Augmentation Transformations"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OO_7hM5845rH"
      },
      "outputs": [],
      "source": [
        "x_train_tensor = x_tensor[train_idx]\n",
        "y_train_tensor = y_tensor[train_idx]\n",
        "\n",
        "x_val_tensor = x_tensor[val_idx]\n",
        "y_val_tensor = y_tensor[val_idx]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GduME2jg45rH"
      },
      "outputs": [],
      "source": [
        "train_composer = Compose([RandomHorizontalFlip(p=.5),\n",
        "                          Normalize(mean=(.5,), std=(.5,))])\n",
        "\n",
        "val_composer = Compose([Normalize(mean=(.5,), std=(.5,))])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "v80wXdmH45rH"
      },
      "outputs": [],
      "source": [
        "train_dataset = TransformedTensorDataset(x_train_tensor,\n",
        "                                         y_train_tensor,\n",
        "                                         transform=train_composer)\n",
        "val_dataset = TransformedTensorDataset(x_val_tensor,\n",
        "                                       y_val_tensor,\n",
        "                                       transform=val_composer)\n",
        "\n",
        "# Builds a loader of each set\n",
        "train_loader = DataLoader(dataset=train_dataset, batch_size=16, shuffle=True)\n",
        "val_loader = DataLoader(dataset=val_dataset, batch_size=16)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dodSIRZw45rI"
      },
      "source": [
        "#### Putting It Together"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iMl439qp45rI"
      },
      "outputs": [],
      "source": [
        "# Builds tensors from numpy arrays BEFORE split\n",
        "# Modifies the scale of pixel values from [0, 255] to [0, 1]\n",
        "x_tensor = torch.as_tensor(images / 255).float()\n",
        "y_tensor = torch.as_tensor(labels.reshape(-1, 1)).float()\n",
        "\n",
        "# Uses index_splitter to generate indices for training and\n",
        "# validation sets\n",
        "train_idx, val_idx = index_splitter(len(x_tensor), [80, 20])\n",
        "\n",
        "# Uses indices to perform the split\n",
        "x_train_tensor = x_tensor[train_idx]\n",
        "y_train_tensor = y_tensor[train_idx]\n",
        "x_val_tensor = x_tensor[val_idx]\n",
        "y_val_tensor = y_tensor[val_idx]\n",
        "\n",
        "# Builds different composers because of data augmentation on training set\n",
        "train_composer = Compose([RandomHorizontalFlip(p=.5),\n",
        "                          Normalize(mean=(.5,), std=(.5,))])\n",
        "val_composer = Compose([Normalize(mean=(.5,), std=(.5,))])\n",
        "\n",
        "# Uses custom dataset to apply composed transforms to each set\n",
        "train_dataset = TransformedTensorDataset(x_train_tensor,\n",
        "                                         y_train_tensor,\n",
        "                                         transform=train_composer)\n",
        "val_dataset = TransformedTensorDataset(x_val_tensor,\n",
        "                                       y_val_tensor,\n",
        "                                       transform=val_composer)\n",
        "\n",
        "# Builds a weighted random sampler to handle imbalanced classes\n",
        "sampler = make_balanced_sampler(y_train_tensor)\n",
        "\n",
        "# Uses sampler in the training set to get a balanced data loader\n",
        "train_loader = DataLoader(dataset=train_dataset,\n",
        "                          batch_size=16,\n",
        "                          sampler=sampler)\n",
        "val_loader = DataLoader(dataset=val_dataset,\n",
        "                        batch_size=16)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Step 4: Pixels as Features"
      ],
      "metadata": {
        "id": "azAjJxE9ApiZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "dummy_xs, dummy_ys = next(iter(train_loader))\n",
        "dummy_xs.shape"
      ],
      "metadata": {
        "id": "g6ZmrosbteWv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "flattener = nn.Flatten()\n",
        "dummy_xs_flat = flattener(dummy_xs)\n",
        "\n",
        "print(dummy_xs_flat.shape)\n",
        "print(dummy_xs_flat[0])"
      ],
      "metadata": {
        "id": "1BZYy1gLt9dl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f2PsmK4645rI"
      },
      "source": [
        "## Shallow Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ptTTFYq-45rI"
      },
      "source": [
        "$$\n",
        "\\Large \\text{P}(y=1) = \\sigma(z) = \\sigma(w_0x_0+w_1x_1+\\cdots+w_{24}x_{24})\n",
        "$$"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CgUmpoNv45rI"
      },
      "source": [
        "![](https://github.com/dvgodoy/PyTorchStepByStep/blob/master/images/classification.png?raw=1)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_Eaimskx45rI"
      },
      "source": [
        "### Notation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zqTUB3ch45rI"
      },
      "source": [
        "$$\n",
        "\\Large W =\n",
        "\\underset{(25 \\times 1)}{\n",
        "\\begin{bmatrix}\n",
        "w_0 \\\\\n",
        "w_1 \\\\\n",
        "\\vdots \\\\\n",
        "w_{24}\n",
        "\\end{bmatrix}};\n",
        "X =\n",
        "\\underset{(25 \\times 1)}{\n",
        "\\begin{bmatrix}\n",
        "x_0 \\\\\n",
        "x_1 \\\\\n",
        "\\vdots \\\\\n",
        "x_{24}\n",
        "\\end{bmatrix}}\n",
        "$$"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N0_jidTc45rI"
      },
      "source": [
        "$$\n",
        "\\Large\n",
        "\\begin{aligned}\n",
        "z\n",
        "& = W^T \\cdot X\n",
        "=\\underset{(1 \\times 25)}{\n",
        "\\begin{bmatrix}\n",
        "- & w^{T} & -\\\\\n",
        "\\end{bmatrix}}\n",
        "\\underset{(25 \\times 1)}{\n",
        "\\begin{bmatrix}\n",
        "x_0 \\\\\n",
        "x_1 \\\\\n",
        "\\vdots \\\\\n",
        "x_{24}\n",
        "\\end{bmatrix}}\n",
        "=\n",
        "\\underset{(1 \\times 25)}{\n",
        "\\begin{bmatrix}\n",
        "w_0 & w_1 & \\cdots & w_{24}\n",
        "\\end{bmatrix}}\n",
        "\\underset{(25 \\times 1)}{\n",
        "\\begin{bmatrix}\n",
        "x_0 \\\\\n",
        "x_1 \\\\\n",
        "\\vdots \\\\\n",
        "x_{24}\n",
        "\\end{bmatrix}}\n",
        "\\\\\n",
        "& = w_0x_0 + w_1x_1 + \\cdots + w_{24}x_{24}\n",
        "\\end{aligned}\n",
        "$$"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xT43V4p645rI"
      },
      "source": [
        "### Model Configuration"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ycsg6WR845rI"
      },
      "outputs": [],
      "source": [
        "# Sets learning rate - this is \"eta\" ~ the \"n\" like Greek letter\n",
        "lr = 0.1\n",
        "\n",
        "torch.manual_seed(17)\n",
        "# Now we can create a model\n",
        "model_logistic = nn.Sequential()\n",
        "model_logistic.add_module('flatten', nn.Flatten())\n",
        "model_logistic.add_module('output', nn.Linear(25, 1, bias=False))\n",
        "model_logistic.add_module('sigmoid', nn.Sigmoid())\n",
        "\n",
        "# Defines a SGD optimizer to update the parameters\n",
        "optimizer_logistic = optim.SGD(model_logistic.parameters(), lr=lr)\n",
        "\n",
        "# Defines a binary cross entropy loss function\n",
        "binary_loss_fn = nn.BCELoss()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kqfnbPon45rJ"
      },
      "source": [
        "### Model Training"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7mvXGIeh45rJ"
      },
      "outputs": [],
      "source": [
        "n_epochs = 100\n",
        "\n",
        "reg_logistic = Architecture(model_logistic,\n",
        "                            binary_loss_fn,\n",
        "                            optimizer_logistic)\n",
        "reg_logistic.set_loaders(train_loader,\n",
        "                         val_loader)\n",
        "reg_logistic.train(n_epochs)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2YnQAhGs45rJ"
      },
      "outputs": [],
      "source": [
        "fig = reg_logistic.plot_losses()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BuI96Y0z45rJ"
      },
      "source": [
        "## Deep-ish Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yQv1-bUL45rJ"
      },
      "source": [
        "![](https://github.com/dvgodoy/PyTorchStepByStep/blob/master/images/classification_equiv.png?raw=1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E0Dp-V3745rJ"
      },
      "source": [
        "### Model Configuration"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zXneAfBD45rJ"
      },
      "outputs": [],
      "source": [
        "# Sets learning rate - this is \"eta\" ~ the \"n\" like Greek letter\n",
        "lr = 0.1\n",
        "\n",
        "torch.manual_seed(17)\n",
        "# Now we can create a model\n",
        "model_nn = nn.Sequential()\n",
        "model_nn.add_module('flatten', nn.Flatten())\n",
        "model_nn.add_module('hidden0', nn.Linear(25, 5, bias=False))\n",
        "model_nn.add_module('hidden1', nn.Linear(5, 3, bias=False))\n",
        "model_nn.add_module('output', nn.Linear(3, 1, bias=False))\n",
        "model_nn.add_module('sigmoid', nn.Sigmoid())\n",
        "\n",
        "# Defines a SGD optimizer to update the parameters\n",
        "optimizer_nn = optim.SGD(model_nn.parameters(), lr=lr)\n",
        "\n",
        "# Defines a binary cross entropy loss function\n",
        "binary_loss_fn = nn.BCELoss()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YNT_6xv745rJ"
      },
      "source": [
        "### Model Training"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "w-4HThCE45rJ"
      },
      "outputs": [],
      "source": [
        "n_epochs = 100\n",
        "\n",
        "arch_nn = Architecture(model_nn, binary_loss_fn, optimizer_nn)\n",
        "arch_nn.set_loaders(train_loader, val_loader)\n",
        "arch_nn.train(n_epochs)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "q9Djo1Bm45rJ"
      },
      "outputs": [],
      "source": [
        "fig = arch_nn.plot_losses()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def figure5(sbs_logistic, sbs_nn):\n",
        "    fig, axs = plt.subplots(1, 2, figsize=(15, 6))\n",
        "    axs[0].plot(sbs_logistic.losses, 'b--', label='Logistic - Training')\n",
        "    axs[1].plot(sbs_logistic.val_losses, 'r--', label='Logistic - Validation')\n",
        "    axs[0].plot(sbs_nn.losses, 'b', label='3-layer Network - Training', alpha=.5)\n",
        "    axs[1].plot(sbs_nn.val_losses, 'r', label='3-layer Network - Validation', alpha=.5)\n",
        "    axs[0].set_xlabel('Epochs')\n",
        "    axs[0].set_ylabel('Losses')\n",
        "    axs[0].set_ylim([0.45, 0.75])\n",
        "    axs[0].legend()\n",
        "    axs[1].set_xlabel('Epochs')\n",
        "    axs[1].set_ylabel('Losses')\n",
        "    axs[1].set_ylim([0.45, 0.75])\n",
        "    axs[1].legend()\n",
        "    fig.tight_layout()\n",
        "    return fig"
      ],
      "metadata": {
        "id": "660_m17ct8Wh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "scrolled": false,
        "id": "u9OEqRJG45rJ"
      },
      "outputs": [],
      "source": [
        "fig = figure5(reg_logistic, arch_nn)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ru9HYp5z45rJ"
      },
      "source": [
        "### Show Me the Math!"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Xh4sFYpA45rJ"
      },
      "source": [
        "$$\n",
        "\\large\n",
        "\\begin{array}{rcccccccccccc}\n",
        "\\text{Hidden}\\ \\#0 & & & & & & & &\n",
        "\\underset{(5 \\times 1)}{\n",
        "\\begin{bmatrix}\n",
        "z_{00} \\\\\n",
        "z_{01} \\\\\n",
        "z_{02} \\\\\n",
        "z_{03} \\\\\n",
        "z_{04} \\\\\n",
        "\\end{bmatrix}}\n",
        "&\n",
        "=\n",
        "&\n",
        "\\underset{(5 \\times 25)}{\n",
        "\\begin{bmatrix}\n",
        "- & w^{T}_{00} & -\\\\\n",
        "- & w^{T}_{01} & -\\\\\n",
        "- & w^{T}_{02} & -\\\\\n",
        "- & w^{T}_{03} & -\\\\\n",
        "- & w^{T}_{04} & -\n",
        "\\end{bmatrix}}\n",
        "&\n",
        "&\n",
        "\\underset{(25 \\times 1)}{\n",
        "\\begin{bmatrix}\n",
        "x_0 \\\\\n",
        "\\vdots \\\\\n",
        "x_{11} \\\\\n",
        "\\vdots \\\\\n",
        "x_{24}\n",
        "\\end{bmatrix}}\n",
        "\\\\\n",
        "\\text{Hidden}\\ \\#1 & & & &\n",
        "\\underset{(3 \\times 1)}{\n",
        "\\begin{bmatrix}\n",
        "z_{10} \\\\\n",
        "z_{11} \\\\\n",
        "z_{12} \\\\\n",
        "\\end{bmatrix}}\n",
        "&\n",
        "=\n",
        "&\n",
        "\\underset{(3 \\times 5)}{\n",
        "\\begin{bmatrix}\n",
        "- & w^{T}_{10} & -\\\\\n",
        "- & w^{T}_{11} & -\\\\\n",
        "- & w^{T}_{12} & -\\\\\n",
        "\\end{bmatrix}}\n",
        "&\n",
        "&\n",
        "\\underset{(5 \\times 1)}{\n",
        "\\begin{bmatrix}\n",
        "z_{00} \\\\\n",
        "z_{01} \\\\\n",
        "z_{02} \\\\\n",
        "z_{03} \\\\\n",
        "z_{04} \\\\\n",
        "\\end{bmatrix}}\n",
        "\\\\\n",
        "\\text{Output} &\n",
        "\\underset{(1 \\times 1)}{\n",
        "\\begin{bmatrix}\n",
        "z_{2}\n",
        "\\end{bmatrix}}\n",
        "&\n",
        "=\n",
        "&\n",
        "\\underset{(1 \\times 3)}{\n",
        "\\begin{bmatrix}\n",
        "- & w^{T}_{20} & -\\\\\n",
        "\\end{bmatrix}}\n",
        "&\n",
        "\\underset{(3 \\times 1)}{\n",
        "\\begin{bmatrix}\n",
        "z_{10} \\\\\n",
        "z_{11} \\\\\n",
        "z_{12} \\\\\n",
        "\\end{bmatrix}}\n",
        "\\\\\n",
        "\\hline\n",
        "\\text{substituting } z's... &\n",
        "\\underset{(1 \\times 1)}{\n",
        "\\begin{bmatrix}\n",
        "z_{2}\n",
        "\\end{bmatrix}}\n",
        "&\n",
        "=\n",
        "&\n",
        "\\underbrace{\n",
        "\\underset{(1 \\times 3)}{\n",
        "\\begin{bmatrix}\n",
        "- & w^{T}_{20} & -\\\\\n",
        "\\end{bmatrix}}}_{\\text{Output Layer}}\n",
        "&\n",
        "& &\n",
        "\\underbrace{\n",
        "\\underset{(3 \\times 5)}{\n",
        "\\begin{bmatrix}\n",
        "- & w^{T}_{10} & -\\\\\n",
        "- & w^{T}_{11} & -\\\\\n",
        "- & w^{T}_{12} & -\\\\\n",
        "\\end{bmatrix}}}_{\\text{Hidden Layer #1}}\n",
        "& &\n",
        "& &\n",
        "\\underbrace{\n",
        "\\underset{(5 \\times 25)}{\n",
        "\\begin{bmatrix}\n",
        "- & w^{T}_{00} & -\\\\\n",
        "- & w^{T}_{01} & -\\\\\n",
        "- & w^{T}_{02} & -\\\\\n",
        "- & w^{T}_{03} & -\\\\\n",
        "- & w^{T}_{04} & -\n",
        "\\end{bmatrix}}}_{\\text{Hidden Layer #0}}\n",
        "&\n",
        "&\n",
        "\\underset{(25 \\times 1)}{\n",
        "\\begin{bmatrix}\n",
        "x_0 \\\\\n",
        "\\vdots \\\\\n",
        "x_{11} \\\\\n",
        "\\vdots \\\\\n",
        "x_{24}\n",
        "\\end{bmatrix}}\n",
        "\\\\\n",
        "\\text{multiplying...} & &\n",
        "=\n",
        "&\n",
        "\\underbrace{\n",
        "\\underset{(1 \\times 25)}{\n",
        "\\begin{bmatrix}\n",
        "- & w^{T} & -\\\\\n",
        "\\end{bmatrix}}}_{\\text{Matrices Multiplied}}\n",
        "&\n",
        "& & & & & & & &\n",
        "\\underset{(25 \\times 1)}{\n",
        "\\begin{bmatrix}\n",
        "x_0 \\\\\n",
        "\\vdots \\\\\n",
        "x_{11} \\\\\n",
        "\\vdots \\\\\n",
        "x_{24}\n",
        "\\end{bmatrix}}\n",
        "\\end{array}\n",
        "$$"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rI96lFbK45rJ"
      },
      "source": [
        "### Show Me the Code!"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IjoDyIxu45rJ"
      },
      "outputs": [],
      "source": [
        "w_nn_hidden0 = model_nn.hidden0.weight.detach()\n",
        "w_nn_hidden1 = model_nn.hidden1.weight.detach()\n",
        "w_nn_output = model_nn.output.weight.detach()\n",
        "\n",
        "w_nn_hidden0.shape, w_nn_hidden1.shape, w_nn_output.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KjSNYRVZ45rJ"
      },
      "outputs": [],
      "source": [
        "w_nn_equiv = w_nn_output @ w_nn_hidden1 @ w_nn_hidden0\n",
        "w_nn_equiv.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ypjFqnmz45rJ"
      },
      "outputs": [],
      "source": [
        "w_nn_equiv = w_nn_output.mm(w_nn_hidden1.mm(w_nn_hidden0))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "maphzzMZ45rJ"
      },
      "outputs": [],
      "source": [
        "w_logistic_output = model_logistic.output.weight.detach()\n",
        "\n",
        "w_logistic_output.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MrZePLi545rK"
      },
      "outputs": [],
      "source": [
        "reg_logistic.count_parameters(), arch_nn.count_parameters()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "veZgYTgk45rK"
      },
      "source": [
        "### Weights as Pixels"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_ZXqRviW45rK"
      },
      "outputs": [],
      "source": [
        "w_nn_hidden0.shape"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def figure7(weights):\n",
        "    fig, axs = plt.subplots(1, 5, figsize=(15, 4))\n",
        "\n",
        "    for i, m in enumerate(weights):\n",
        "        axs[i].imshow(m.reshape(-1, 5).tolist(), cmap='gray')\n",
        "        axs[i].grid(False)\n",
        "        axs[i].set_xticks([])\n",
        "        axs[i].set_yticks([])\n",
        "        axs[i].set_title(r'$w_{0' + str(i) + '}$')\n",
        "\n",
        "    fig.suptitle('Hidden Layer #0')\n",
        "    fig.subplots_adjust(top=0.6)\n",
        "    fig.tight_layout()\n",
        "    return fig"
      ],
      "metadata": {
        "id": "WeRQUcuq6iJv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Mu8pyRJn45rK"
      },
      "outputs": [],
      "source": [
        "fig = figure7(w_nn_hidden0)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tFzIq07j45rK"
      },
      "source": [
        "## Activation Functions"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iFZoOs8L45rK"
      },
      "source": [
        "### Sigmoid"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_K9LCobu45rK"
      },
      "source": [
        "$$\n",
        "\\Large \\sigma(z) = \\frac{1}{1 + e^{-z}}\n",
        "$$"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def plot_activation(func, name=None):\n",
        "    z = torch.linspace(-5, 5, 1000)\n",
        "    z.requires_grad_(True)\n",
        "    func(z).sum().backward()\n",
        "    sig = func(z).detach()\n",
        "\n",
        "    fig, ax = plt.subplots(1, 1, figsize=(8, 5))\n",
        "\n",
        "    # Move left y-axis and bottim x-axis to centre, passing through (0,0)\n",
        "    if name is None:\n",
        "        try:\n",
        "            name = func.__name__\n",
        "        except AttributeError:\n",
        "            name = ''\n",
        "\n",
        "    if name == 'sigmoid':\n",
        "        ax.set_ylim([0, 1.1])\n",
        "    elif name == 'tanh':\n",
        "        ax.set_ylim([-1.1, 1.1])\n",
        "    elif name == 'relu':\n",
        "        ax.set_ylim([-.1, 5.01])\n",
        "    else:\n",
        "        ax.set_ylim([-1.1, 5.01])\n",
        "\n",
        "    ax.set_xticks(np.arange(-5, 6, 1))\n",
        "    ax.set_xlabel('z')\n",
        "    ax.set_ylabel(r'$\\sigma(z)$')\n",
        "\n",
        "    # Eliminate upper and right axes\n",
        "    ax.spines['right'].set_color('none')\n",
        "    ax.spines['top'].set_color('none')\n",
        "\n",
        "    # Show ticks in the left and lower axes only\n",
        "    ax.xaxis.set_ticks_position('bottom')\n",
        "    ax.yaxis.set_ticks_position('left')\n",
        "\n",
        "    ax.set_title(name, fontsize=16)\n",
        "    ax.plot(z.detach().numpy(), sig.numpy(), c='k', label='Activation')\n",
        "    ax.plot(z.detach().numpy(), z.grad.numpy(), c='r', label='Gradient')\n",
        "    ax.legend(loc=2)\n",
        "\n",
        "    fig.tight_layout()\n",
        "    fig.show()\n",
        "    return fig"
      ],
      "metadata": {
        "id": "uXMRHQz8SRg2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "scrolled": false,
        "id": "o2wkAqrU45rK"
      },
      "outputs": [],
      "source": [
        "fig = plot_activation(torch.sigmoid)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "87-bK5eT45rK"
      },
      "outputs": [],
      "source": [
        "dummy_z = torch.tensor([-3., 0., 3.])\n",
        "torch.sigmoid(dummy_z)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_kPOHsUc45rK"
      },
      "outputs": [],
      "source": [
        "nn.Sigmoid()(dummy_z)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6f-EfeUo45rK"
      },
      "source": [
        "### Hyperbolic Tangent (Tanh)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2EtmWqay45rK"
      },
      "source": [
        "$$\n",
        "\\Large \\sigma(z) = \\frac{e^z - e^{-z}}{e^z + e^{-z}}\n",
        "$$"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yciKLy1445rK"
      },
      "outputs": [],
      "source": [
        "fig = plot_activation(torch.tanh)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2FtjN1od45rK"
      },
      "outputs": [],
      "source": [
        "dummy_z = torch.tensor([-3., 0., 3.])\n",
        "torch.tanh(dummy_z)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9PkAFnGH45rK"
      },
      "outputs": [],
      "source": [
        "nn.Tanh()(dummy_z)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dVkzWTeF45rK"
      },
      "source": [
        "### Rectified Linear Unit (ReLU)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ajxP-fWF45rL"
      },
      "source": [
        "$$\n",
        "\\Large\n",
        "\\begin{aligned}\n",
        "\\sigma(z) &=\n",
        "\\begin{cases}\n",
        "z,\\ \\text{if } z \\ge 0\n",
        "\\\\\n",
        "0,\\ \\text{if } z < 0\n",
        "\\end{cases}\n",
        "\\\\\n",
        "& \\text{or}\n",
        "\\\\\n",
        "\\sigma(z) &= \\text{max}(0, z)\n",
        "\\end{aligned}\n",
        "$$"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PnUsNk1L45rL"
      },
      "outputs": [],
      "source": [
        "fig = plot_activation(torch.relu)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HJGEPFwO45rL"
      },
      "outputs": [],
      "source": [
        "dummy_z = torch.tensor([-3., 0., 3.])\n",
        "F.relu(dummy_z)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lh4ZT0U745rM"
      },
      "outputs": [],
      "source": [
        "nn.ReLU()(dummy_z)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pFApw_h745rM"
      },
      "outputs": [],
      "source": [
        "dummy_z.clamp(min=0)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nBHSeox345rM"
      },
      "source": [
        "### Leaky ReLU"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x4qNcNNB45rM"
      },
      "source": [
        "$$\n",
        "\\Large\n",
        "\\begin{aligned}\n",
        "\\sigma(z) =&\n",
        "\\begin{cases}\n",
        "z,\\ \\text{if } z \\ge 0\n",
        "\\\\\n",
        "0.01z,\\ \\text{if } z < 0\n",
        "\\end{cases}\n",
        "\\\\\n",
        "\\text{or}&\n",
        "\\\\\n",
        "\\sigma(z)=&\\text{max}(0,z)+0.01\\ \\text{min}(0,z)\n",
        "\\end{aligned}\n",
        "$$"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ORm-9ghI45rM"
      },
      "outputs": [],
      "source": [
        "fig = plot_activation(nn.LeakyReLU(), name='Leaky ReLU')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "scrolled": true,
        "id": "D7btzSuU45rM"
      },
      "outputs": [],
      "source": [
        "dummy_z = torch.tensor([-3., 0., 3.])\n",
        "F.leaky_relu(dummy_z, negative_slope=0.01)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lZoWgq2r45rM"
      },
      "outputs": [],
      "source": [
        "nn.LeakyReLU(negative_slope=0.02)(dummy_z)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N-I1q7H145rM"
      },
      "source": [
        "### Parametric ReLU (PReLU)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RBanhd-y45rM"
      },
      "source": [
        "$$\n",
        "\\Large\n",
        "\\begin{aligned}\n",
        "\\sigma(z) =&\n",
        "\\begin{cases}\n",
        "z,\\ \\text{if } z \\ge 0\n",
        "\\\\\n",
        "az,\\ \\text{if } z < 0\n",
        "\\end{cases}\n",
        "\\\\\n",
        "\\text{or}&\n",
        "\\\\\n",
        "\\sigma(z)=&\\text{max}(0,z)+a\\ \\text{min}(0,z)\n",
        "\\end{aligned}\n",
        "$$"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wS-sBECQ45rM"
      },
      "outputs": [],
      "source": [
        "fig = plot_activation(nn.PReLU(), name='Parametric ReLU')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uCatA2ib45rM"
      },
      "outputs": [],
      "source": [
        "dummy_z = torch.tensor([-3., 0., 3.])\n",
        "F.prelu(dummy_z, weight=torch.tensor(0.25))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nKTGuGM545rM"
      },
      "outputs": [],
      "source": [
        "nn.PReLU(init=0.25)(dummy_z)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SkUm_Bmy45rM"
      },
      "source": [
        "## Deep Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1bHeyt4h45rM"
      },
      "source": [
        "![](https://github.com/dvgodoy/PyTorchStepByStep/blob/master/images/classification_relu2.png?raw=1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RBu2Z0Qa45rM"
      },
      "source": [
        "### Model Configuration"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "M5LQr7qL45rM"
      },
      "outputs": [],
      "source": [
        "# Sets learning rate - this is \"eta\" ~ the \"n\" like Greek letter\n",
        "lr = 0.1\n",
        "\n",
        "torch.manual_seed(17)\n",
        "# Now we can create a model\n",
        "model_relu = nn.Sequential()\n",
        "model_relu.add_module('flatten', nn.Flatten())\n",
        "model_relu.add_module('hidden0', nn.Linear(25, 5, bias=False))\n",
        "model_relu.add_module('activation0', nn.ReLU())\n",
        "model_relu.add_module('hidden1', nn.Linear(5, 3, bias=False))\n",
        "model_relu.add_module('activation1', nn.ReLU())\n",
        "model_relu.add_module('output', nn.Linear(3, 1, bias=False))\n",
        "model_relu.add_module('sigmoid', nn.Sigmoid())\n",
        "\n",
        "# Defines a SGD optimizer to update the parameters\n",
        "optimizer_relu = optim.SGD(model_relu.parameters(), lr=lr)\n",
        "\n",
        "# Defines a binary cross entropy loss function\n",
        "binary_loss_fn = nn.BCELoss()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gwAycSLu45rM"
      },
      "source": [
        "### Model Training"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "enMv3YvY45rM"
      },
      "outputs": [],
      "source": [
        "n_epochs = 50\n",
        "\n",
        "arc_relu = Architecture(model_relu, binary_loss_fn, optimizer_relu)\n",
        "arc_relu.set_loaders(train_loader, val_loader)\n",
        "arc_relu.train(n_epochs)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RXbuqda945rM"
      },
      "outputs": [],
      "source": [
        "fig = arc_relu.plot_losses()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def figure5b(sbs_logistic, sbs_nn, sbs_relu):\n",
        "    fig, axs = plt.subplots(1, 2, figsize=(15, 6))\n",
        "    axs[0].plot(sbs_logistic.losses, 'b--', label='Logistic - Training')\n",
        "    axs[1].plot(sbs_logistic.val_losses, 'r--', label='Logistic - Validation')\n",
        "\n",
        "    axs[0].plot(sbs_nn.losses, 'b', label='3-layer Network - Training', alpha=.5)\n",
        "    axs[1].plot(sbs_nn.val_losses, 'r', label='3-layer Network - Validation', alpha=.5)\n",
        "\n",
        "    axs[0].plot(sbs_relu.losses, 'b', label='ReLU Network - Training', alpha=.8)\n",
        "    axs[1].plot(sbs_relu.val_losses, 'r', label='ReLU Network - Validation', alpha=.8)\n",
        "\n",
        "    axs[0].set_xlabel('Epochs')\n",
        "    axs[0].set_ylabel('Losses')\n",
        "    axs[0].legend()\n",
        "    axs[1].set_xlabel('Epochs')\n",
        "    axs[1].set_ylabel('Losses')\n",
        "    axs[1].legend()\n",
        "    fig.tight_layout()\n",
        "    return fig"
      ],
      "metadata": {
        "id": "mRbIxO3TpZOG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BVf7gI9d45rM"
      },
      "outputs": [],
      "source": [
        "fig = figure5b(reg_logistic, arch_nn, arc_relu)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AaEPXPXx45rM"
      },
      "source": [
        "### Show Me the Math Again!"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fMe-Vzey45rN"
      },
      "source": [
        "$$\n",
        "\\large\n",
        "\\begin{array}{rcccccccccccc}\n",
        "\\text{Hidden }\\#0 & & & & & & & &\n",
        "\\underset{(5 \\times 1)}{\n",
        "\\begin{bmatrix}\n",
        "z_{00} \\\\\n",
        "z_{01} \\\\\n",
        "z_{02} \\\\\n",
        "z_{03} \\\\\n",
        "z_{04} \\\\\n",
        "\\end{bmatrix}}\n",
        "&\n",
        "=\n",
        "&\n",
        "\\underset{(5 \\times 25)}{\n",
        "\\begin{bmatrix}\n",
        "- & w^{T}_{00} & -\\\\\n",
        "- & w^{T}_{01} & -\\\\\n",
        "- & w^{T}_{02} & -\\\\\n",
        "- & w^{T}_{03} & -\\\\\n",
        "- & w^{T}_{04} & -\n",
        "\\end{bmatrix}}\n",
        "&\n",
        "&\n",
        "\\underset{(25 \\times 1)}{\n",
        "\\begin{bmatrix}\n",
        "x_0 \\\\\n",
        "\\vdots \\\\\n",
        "x_{11} \\\\\n",
        "\\vdots \\\\\n",
        "x_{24}\n",
        "\\end{bmatrix}}\n",
        "\\\\\n",
        "\\text{Hidden }\\#1 & & & &\n",
        "\\underset{(3 \\times 1)}{\n",
        "\\begin{bmatrix}\n",
        "z_{10} \\\\\n",
        "z_{11} \\\\\n",
        "z_{12} \\\\\n",
        "\\end{bmatrix}}\n",
        "&\n",
        "=\n",
        "&\n",
        "\\underset{(3 \\times 5)}{\n",
        "\\begin{bmatrix}\n",
        "- & w^{T}_{10} & -\\\\\n",
        "- & w^{T}_{11} & -\\\\\n",
        "- & w^{T}_{12} & -\\\\\n",
        "\\end{bmatrix}}\n",
        "&\n",
        "&\n",
        "\\underbrace{\n",
        "f_0\n",
        "\\underset{(5 \\times 1)}{\n",
        "\\left(\n",
        "\\begin{bmatrix}\n",
        "z_{00} \\\\\n",
        "z_{01} \\\\\n",
        "z_{02} \\\\\n",
        "z_{03} \\\\\n",
        "z_{04} \\\\\n",
        "\\end{bmatrix}\n",
        "\\right)}}_{\\text{Activation #0}}\n",
        "\\\\\n",
        "\\text{Output} &\n",
        "\\underset{(1 \\times 1)}{\n",
        "\\begin{bmatrix}\n",
        "z_{2}\n",
        "\\end{bmatrix}}\n",
        "&\n",
        "=\n",
        "&\n",
        "\\underset{(1 \\times 3)}{\n",
        "\\begin{bmatrix}\n",
        "- & w^{T}_{20} & -\\\\\n",
        "\\end{bmatrix}}\n",
        "&\n",
        "\\underbrace{\n",
        "f_1\n",
        "\\underset{(3 \\times 1)}{\n",
        "\\left(\n",
        "\\begin{bmatrix}\n",
        "z_{10} \\\\\n",
        "z_{11} \\\\\n",
        "z_{12} \\\\\n",
        "\\end{bmatrix}\n",
        "\\right)}}_{\\text{Activation #1}}\n",
        "\\\\\n",
        "\\hline\n",
        "\\text{substituting z's...} &\n",
        "\\underset{(1 \\times 1)}{\n",
        "\\begin{bmatrix}\n",
        "z_{2}\n",
        "\\end{bmatrix}}\n",
        "&\n",
        "=\n",
        "&\n",
        "\\underbrace{\n",
        "\\underset{(1 \\times 3)}{\n",
        "\\begin{bmatrix}\n",
        "- & w^{T}_{20} & -\\\\\n",
        "\\end{bmatrix}}}_{\\text{Output Layer}}\n",
        "&\n",
        "f_1\n",
        "& &\n",
        "\\left(\n",
        "\\underbrace{\n",
        "\\underset{(3 \\times 5)}{\n",
        "\\begin{bmatrix}\n",
        "- & w^{T}_{10} & -\\\\\n",
        "- & w^{T}_{11} & -\\\\\n",
        "- & w^{T}_{12} & -\\\\\n",
        "\\end{bmatrix}}}_{\\text{Hidden Layer #1}}\n",
        "\\right.\n",
        "&\n",
        "&\n",
        "f_0\n",
        "& &\n",
        "\\left(\n",
        "\\underbrace{\n",
        "\\underset{(5 \\times 25)}{\n",
        "\\begin{bmatrix}\n",
        "- & w^{T}_{00} & -\\\\\n",
        "- & w^{T}_{01} & -\\\\\n",
        "- & w^{T}_{02} & -\\\\\n",
        "- & w^{T}_{03} & -\\\\\n",
        "- & w^{T}_{04} & -\n",
        "\\end{bmatrix}}}_{\\text{Hidden Layer #0}}\n",
        "\\right.\n",
        "&\n",
        "&\n",
        "\\left.\n",
        "\\left.\n",
        "\\underbrace{\n",
        "\\underset{(25 \\times 1)}{\n",
        "\\begin{bmatrix}\n",
        "x_0 \\\\\n",
        "\\vdots \\\\\n",
        "x_{11} \\\\\n",
        "\\vdots \\\\\n",
        "x_{24}\n",
        "\\end{bmatrix}}}_{\\text{Inputs}}\n",
        "\\right)\n",
        "\\right)\n",
        "\\end{array}\n",
        "$$"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ytcKcFA345rN"
      },
      "source": [
        "## Putting It All Together"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LsIyGGMY45rN"
      },
      "outputs": [],
      "source": [
        "class TransformedTensorDataset(Dataset):\n",
        "    def __init__(self, x, y, transform=None):\n",
        "        self.x = x\n",
        "        self.y = y\n",
        "        self.transform = transform\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        x = self.x[index]\n",
        "\n",
        "        if self.transform:\n",
        "            x = self.transform(x)\n",
        "\n",
        "        return x, self.y[index]\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.x)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Qd3DUvew45rN"
      },
      "outputs": [],
      "source": [
        "def index_splitter(n, splits, seed=13):\n",
        "    idx = torch.arange(n)\n",
        "    # Makes the split argument a tensor\n",
        "    splits_tensor = torch.as_tensor(splits)\n",
        "    # Finds the correct multiplier, so we don't have\n",
        "    # to worry about summing up to N (or one)\n",
        "    multiplier = n / splits_tensor.sum()\n",
        "    splits_tensor = (multiplier * splits_tensor).long()\n",
        "    # If there is a difference, throws at the first split\n",
        "    # so random_split does not complain\n",
        "    diff = n - splits_tensor.sum()\n",
        "    splits_tensor[0] += diff\n",
        "    # Uses PyTorch random_split to split the indices\n",
        "    torch.manual_seed(seed)\n",
        "    return random_split(idx, splits_tensor)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EPDFPIbP45rN"
      },
      "outputs": [],
      "source": [
        "def make_balanced_sampler(y):\n",
        "    # Computes weights for compensating imbalanced classes\n",
        "    classes, counts = y.unique(return_counts=True)\n",
        "    weights = 1.0 / counts.float()\n",
        "    sample_weights = weights[y.squeeze().long()]\n",
        "    # Builds sampler with compute weights\n",
        "    generator = torch.Generator()\n",
        "    sampler = WeightedRandomSampler(\n",
        "        weights=sample_weights,\n",
        "        num_samples=len(sample_weights),\n",
        "        generator=generator,\n",
        "        replacement=True\n",
        "    )\n",
        "    return sampler"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "015vwvDU45rN"
      },
      "outputs": [],
      "source": [
        "# Builds tensors from numpy arrays BEFORE split\n",
        "# Modifies the scale of pixel values from [0, 255] to [0, 1]\n",
        "x_tensor = torch.as_tensor(images / 255).float()\n",
        "y_tensor = torch.as_tensor(labels.reshape(-1, 1)).float()\n",
        "\n",
        "# Uses index_splitter to generate indices for training and\n",
        "# validation sets\n",
        "train_idx, val_idx = index_splitter(len(x_tensor), [80, 20])\n",
        "# Uses indices to perform the split\n",
        "x_train_tensor = x_tensor[train_idx]\n",
        "y_train_tensor = y_tensor[train_idx]\n",
        "x_val_tensor = x_tensor[val_idx]\n",
        "y_val_tensor = y_tensor[val_idx]\n",
        "\n",
        "# Builds different composers because of data augmentation on training set\n",
        "train_composer = Compose([RandomHorizontalFlip(p=.5),\n",
        "                          Normalize(mean=(.5,), std=(.5,))])\n",
        "val_composer = Compose([Normalize(mean=(.5,), std=(.5,))])\n",
        "# Uses custom dataset to apply composed transforms to each set\n",
        "train_dataset = TransformedTensorDataset(x_train_tensor, y_train_tensor, transform=train_composer)\n",
        "val_dataset = TransformedTensorDataset(x_val_tensor, y_val_tensor, transform=val_composer)\n",
        "\n",
        "# Builds a weighted random sampler to handle imbalanced classes\n",
        "sampler = make_balanced_sampler(y_train_tensor)\n",
        "\n",
        "# Uses sampler in the training set to get a balanced data loader\n",
        "train_loader = DataLoader(dataset=train_dataset, batch_size=16, sampler=sampler)\n",
        "val_loader = DataLoader(dataset=val_dataset, batch_size=16)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "P1a0NU8j45rN"
      },
      "outputs": [],
      "source": [
        "# Sets learning rate - this is \"eta\" ~ the \"n\" like Greek letter\n",
        "lr = 0.1\n",
        "\n",
        "torch.manual_seed(11)\n",
        "# Now we can create a model\n",
        "model_relu = nn.Sequential()\n",
        "model_relu.add_module('flatten', nn.Flatten())\n",
        "model_relu.add_module('hidden0', nn.Linear(25, 5, bias=False))\n",
        "model_relu.add_module('activation0', nn.ReLU())\n",
        "model_relu.add_module('hidden1', nn.Linear(5, 3, bias=False))\n",
        "model_relu.add_module('activation1', nn.ReLU())\n",
        "model_relu.add_module('output', nn.Linear(3, 1, bias=False))\n",
        "model_relu.add_module('sigmoid', nn.Sigmoid())\n",
        "\n",
        "# Defines a SGD optimizer to update the parameters\n",
        "# (now retrieved directly from the model)\n",
        "optimizer_relu = optim.SGD(model_relu.parameters(), lr=lr)\n",
        "\n",
        "# Defines a binary cross entropy loss function\n",
        "binary_loss_fn = nn.BCELoss()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_WZ9mDl745rN"
      },
      "outputs": [],
      "source": [
        "n_epochs = 50\n",
        "\n",
        "arch_relu = Architecture(model_relu, binary_loss_fn, optimizer_relu)\n",
        "arch_relu.set_loaders(train_loader, val_loader)\n",
        "arch_relu.train(n_epochs)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "o55vFCra45rN"
      },
      "outputs": [],
      "source": [
        "arch_relu.plot_losses()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "arch_relu.count_parameters()"
      ],
      "metadata": {
        "id": "o2W_dSH5Kw6E"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "V2pdK45uLVL9"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}