lessons/week04/week04c.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Import"
      ],
      "metadata": {
        "id": "CERpGfO--PpT"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Hc7hH4X78OQ9"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import datetime\n",
        "import torch\n",
        "import matplotlib.pyplot as plt\n",
        "from torch.utils.data import Dataset, TensorDataset, DataLoader\n",
        "from torch.utils.data.dataset import random_split\n",
        "import torch.optim as optim\n",
        "import torch.nn as nn\n",
        "from sklearn.datasets import make_moons\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import confusion_matrix, roc_curve, precision_recall_curve, auc\n",
        "from matplotlib.colors import ListedColormap\n",
        "from sklearn.metrics import accuracy_score\n",
        "\n",
        "%matplotlib inline\n",
        "plt.style.use('fivethirtyeight')"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Architecture class"
      ],
      "metadata": {
        "id": "h38OikK_-d4M"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class Architecture(object):\n",
        "    def __init__(self, model, loss_fn, optimizer):\n",
        "        # Here we define the attributes of our class\n",
        "\n",
        "        # We start by storing the arguments as attributes\n",
        "        # to use them later\n",
        "        self.model = model\n",
        "        self.loss_fn = loss_fn\n",
        "        self.optimizer = optimizer\n",
        "        self.device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "        # Let's send the model to the specified device right away\n",
        "        self.model.to(self.device)\n",
        "\n",
        "        # These attributes are defined here, but since they are\n",
        "        # not informed at the moment of creation, we keep them None\n",
        "        self.train_loader = None\n",
        "        self.val_loader = None\n",
        "\n",
        "        # These attributes are going to be computed internally\n",
        "        self.losses = []\n",
        "        self.val_losses = []\n",
        "        self.total_epochs = 0\n",
        "\n",
        "        # Creates the train_step function for our model,\n",
        "        # loss function and optimizer\n",
        "        # Note: there are NO ARGS there! It makes use of the class\n",
        "        # attributes directly\n",
        "        self.train_step_fn = self._make_train_step_fn()\n",
        "        # Creates the val_step function for our model and loss\n",
        "        self.val_step_fn = self._make_val_step_fn()\n",
        "\n",
        "    def to(self, device):\n",
        "        # This method allows the user to specify a different device\n",
        "        # It sets the corresponding attribute (to be used later in\n",
        "        # the mini-batches) and sends the model to the device\n",
        "        try:\n",
        "            self.device = device\n",
        "            self.model.to(self.device)\n",
        "        except RuntimeError:\n",
        "            self.device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "            print(f\"Couldn't send it to {device}, sending it to {self.device} instead.\")\n",
        "            self.model.to(self.device)\n",
        "\n",
        "    def set_loaders(self, train_loader, val_loader=None):\n",
        "        # This method allows the user to define which train_loader (and val_loader, optionally) to use\n",
        "        # Both loaders are then assigned to attributes of the class\n",
        "        # So they can be referred to later\n",
        "        self.train_loader = train_loader\n",
        "        self.val_loader = val_loader\n",
        "\n",
        "    def _make_train_step_fn(self):\n",
        "        # This method does not need ARGS... it can refer to\n",
        "        # the attributes: self.model, self.loss_fn and self.optimizer\n",
        "\n",
        "        # Builds function that performs a step in the train loop\n",
        "        def perform_train_step_fn(x, y):\n",
        "            # Sets model to TRAIN mode\n",
        "            self.model.train()\n",
        "\n",
        "            # Step 1 - Computes our model's predicted output - forward pass\n",
        "            yhat = self.model(x)\n",
        "            # Step 2 - Computes the loss\n",
        "            loss = self.loss_fn(yhat, y)\n",
        "            # Step 3 - Computes gradients for both \"a\" and \"b\" parameters\n",
        "            loss.backward()\n",
        "            # Step 4 - Updates parameters using gradients and the learning rate\n",
        "            self.optimizer.step()\n",
        "            self.optimizer.zero_grad()\n",
        "\n",
        "            # Returns the loss\n",
        "            return loss.item()\n",
        "\n",
        "        # Returns the function that will be called inside the train loop\n",
        "        return perform_train_step_fn\n",
        "\n",
        "    def _make_val_step_fn(self):\n",
        "        # Builds function that performs a step in the validation loop\n",
        "        def perform_val_step_fn(x, y):\n",
        "            # Sets model to EVAL mode\n",
        "            self.model.eval()\n",
        "\n",
        "            # Step 1 - Computes our model's predicted output - forward pass\n",
        "            yhat = self.model(x)\n",
        "            # Step 2 - Computes the loss\n",
        "            loss = self.loss_fn(yhat, y)\n",
        "            # There is no need to compute Steps 3 and 4, since we don't update parameters during evaluation\n",
        "            return loss.item()\n",
        "\n",
        "        return perform_val_step_fn\n",
        "\n",
        "    def _mini_batch(self, validation=False):\n",
        "        # The mini-batch can be used with both loaders\n",
        "        # The argument `validation`defines which loader and\n",
        "        # corresponding step function is going to be used\n",
        "        if validation:\n",
        "            data_loader = self.val_loader\n",
        "            step_fn = self.val_step_fn\n",
        "        else:\n",
        "            data_loader = self.train_loader\n",
        "            step_fn = self.train_step_fn\n",
        "\n",
        "        if data_loader is None:\n",
        "            return None\n",
        "\n",
        "        # Once the data loader and step function, this is the same\n",
        "        # mini-batch loop we had before\n",
        "        mini_batch_losses = []\n",
        "        for x_batch, y_batch in data_loader:\n",
        "            x_batch = x_batch.to(self.device)\n",
        "            y_batch = y_batch.to(self.device)\n",
        "\n",
        "            mini_batch_loss = step_fn(x_batch, y_batch)\n",
        "            mini_batch_losses.append(mini_batch_loss)\n",
        "\n",
        "        loss = np.mean(mini_batch_losses)\n",
        "        return loss\n",
        "\n",
        "    def set_seed(self, seed=42):\n",
        "        torch.backends.cudnn.deterministic = True\n",
        "        torch.backends.cudnn.benchmark = False\n",
        "        torch.manual_seed(seed)\n",
        "        np.random.seed(seed)\n",
        "\n",
        "    def train(self, n_epochs, seed=42):\n",
        "        # To ensure reproducibility of the training process\n",
        "        self.set_seed(seed)\n",
        "\n",
        "        for epoch in range(n_epochs):\n",
        "            # Keeps track of the numbers of epochs\n",
        "            # by updating the corresponding attribute\n",
        "            self.total_epochs += 1\n",
        "\n",
        "            # inner loop\n",
        "            # Performs training using mini-batches\n",
        "            loss = self._mini_batch(validation=False)\n",
        "            self.losses.append(loss)\n",
        "\n",
        "            # VALIDATION\n",
        "            # no gradients in validation!\n",
        "            with torch.no_grad():\n",
        "                # Performs evaluation using mini-batches\n",
        "                val_loss = self._mini_batch(validation=True)\n",
        "                self.val_losses.append(val_loss)\n",
        "\n",
        "    def save_checkpoint(self, filename):\n",
        "        # Builds dictionary with all elements for resuming training\n",
        "        checkpoint = {'epoch': self.total_epochs,\n",
        "                      'model_state_dict': self.model.state_dict(),\n",
        "                      'optimizer_state_dict': self.optimizer.state_dict(),\n",
        "                      'loss': self.losses,\n",
        "                      'val_loss': self.val_losses}\n",
        "\n",
        "        torch.save(checkpoint, filename)\n",
        "\n",
        "    def load_checkpoint(self, filename):\n",
        "        # Loads dictionary\n",
        "        checkpoint = torch.load(filename)\n",
        "\n",
        "        # Restore state for model and optimizer\n",
        "        self.model.load_state_dict(checkpoint['model_state_dict'])\n",
        "        self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n",
        "\n",
        "        self.total_epochs = checkpoint['epoch']\n",
        "        self.losses = checkpoint['loss']\n",
        "        self.val_losses = checkpoint['val_loss']\n",
        "\n",
        "        self.model.train() # always use TRAIN for resuming training\n",
        "\n",
        "    def predict(self, x):\n",
        "        # Set is to evaluation mode for predictions\n",
        "        self.model.eval()\n",
        "        # Takes aNumpy input and make it a float tensor\n",
        "        x_tensor = torch.as_tensor(x).float()\n",
        "        # Send input to device and uses model for prediction\n",
        "        y_hat_tensor = self.model(x_tensor.to(self.device))\n",
        "        # Set it back to train mode\n",
        "        self.model.train()\n",
        "        # Detaches it, brings it to CPU and back to Numpy\n",
        "        return y_hat_tensor.detach().cpu().numpy()\n",
        "\n",
        "    def plot_losses(self):\n",
        "        fig = plt.figure(figsize=(10, 4))\n",
        "        plt.plot(self.losses, label='Training Loss', c='b')\n",
        "        plt.plot(self.val_losses, label='Validation Loss', c='r')\n",
        "        plt.yscale('log')\n",
        "        plt.xlabel('Epochs')\n",
        "        plt.ylabel('Loss')\n",
        "        plt.legend()\n",
        "        plt.tight_layout()\n",
        "        return fig"
      ],
      "metadata": {
        "id": "tYxCEnws8T7L"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R5dip-4Ouv0d"
      },
      "source": [
        "# Data Generation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2RKKUPCLuv0d"
      },
      "outputs": [],
      "source": [
        "# Generate a synthetic two-dimensional dataset with 100 samples\n",
        "# 'make_moons' is used for binary classification problems\n",
        "X, y = make_moons(n_samples=100, noise=0.3, random_state=0)\n",
        "\n",
        "# Split the dataset into training and validation sets\n",
        "# 20% of the data is used for validation\n",
        "X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=.2, random_state=13)\n",
        "\n",
        "# Initialize a StandardScaler instance\n",
        "sc = StandardScaler()\n",
        "\n",
        "# Fit the scaler only on the training data\n",
        "# This computes the mean and standard deviation to be used for later scaling\n",
        "sc.fit(X_train)\n",
        "\n",
        "# Transform both training and validation sets\n",
        "# Scale the training data\n",
        "X_train = sc.transform(X_train)\n",
        "\n",
        "# Apply the same transformation to the validation data\n",
        "X_val = sc.transform(X_val)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Data Visualization"
      ],
      "metadata": {
        "id": "YWE_8lomRIy_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def figure1(X_train, y_train, X_val, y_val, cm_bright=None):\n",
        "    if cm_bright is None:\n",
        "        cm_bright = ListedColormap(['#FF0000', '#0000FF'])\n",
        "\n",
        "    fig, ax = plt.subplots(1, 2, figsize=(12, 6))\n",
        "\n",
        "    ax[0].scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm_bright)#, edgecolors='k')\n",
        "    ax[0].set_xlabel(r'$X_1$')\n",
        "    ax[0].set_ylabel(r'$X_2$')\n",
        "    ax[0].set_xlim([-2.3, 2.3])\n",
        "    ax[0].set_ylim([-2.3, 2.3])\n",
        "    ax[0].set_title('Generated Data - Train')\n",
        "\n",
        "    ax[1].scatter(X_val[:, 0], X_val[:, 1], c=y_val, cmap=cm_bright)#, edgecolors='k')\n",
        "    ax[1].set_xlabel(r'$X_1$')\n",
        "    ax[1].set_ylabel(r'$X_2$')\n",
        "    ax[1].set_xlim([-2.3, 2.3])\n",
        "    ax[1].set_ylim([-2.3, 2.3])\n",
        "    ax[1].set_title('Generated Data - Validation')\n",
        "    fig.tight_layout()\n",
        "\n",
        "    return fig"
      ],
      "metadata": {
        "id": "QJGiUTTePXQW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "fig = figure1(X_train, y_train, X_val, y_val)"
      ],
      "metadata": {
        "id": "_7sTFyEQQali"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Data Preparation"
      ],
      "metadata": {
        "id": "-gvE84bZRltu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "torch.manual_seed(13)\n",
        "\n",
        "# Builds tensors from numpy arrays\n",
        "x_train_tensor = torch.as_tensor(X_train).float()\n",
        "y_train_tensor = torch.as_tensor(y_train.reshape(-1, 1)).float()\n",
        "\n",
        "x_val_tensor = torch.as_tensor(X_val).float()\n",
        "y_val_tensor = torch.as_tensor(y_val.reshape(-1, 1)).float()\n",
        "\n",
        "# Builds dataset containing ALL data points\n",
        "train_dataset = TensorDataset(x_train_tensor, y_train_tensor)\n",
        "val_dataset = TensorDataset(x_val_tensor, y_val_tensor)\n",
        "\n",
        "# Builds a loader of each set\n",
        "train_loader = DataLoader(\n",
        "    dataset=train_dataset,\n",
        "    batch_size=16,\n",
        "    shuffle=True\n",
        ")\n",
        "val_loader = DataLoader(dataset=val_dataset, batch_size=16)"
      ],
      "metadata": {
        "id": "gMlL-TpVRoMg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Model Configuration"
      ],
      "metadata": {
        "id": "WRT5wNXoRrQB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Sets learning rate - this is \"eta\" ~ the \"n\" like Greek letter\n",
        "lr = 0.1\n",
        "\n",
        "torch.manual_seed(42)\n",
        "model = nn.Sequential()\n",
        "model.add_module('linear', nn.Linear(2, 1))\n",
        "\n",
        "# Defines a SGD optimizer to update the parameters\n",
        "optimizer = optim.SGD(model.parameters(), lr=lr)\n",
        "\n",
        "# Defines a BCE loss function\n",
        "loss_fn = nn.BCEWithLogitsLoss()"
      ],
      "metadata": {
        "id": "9F7vIkumRxGq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Training"
      ],
      "metadata": {
        "id": "dJOkyp2oR8lX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "n_epochs = 100\n",
        "\n",
        "arch = Architecture(model, loss_fn, optimizer)\n",
        "arch.set_loaders(train_loader, val_loader)\n",
        "arch.set_seed(42)\n",
        "arch.train(n_epochs)"
      ],
      "metadata": {
        "id": "y7-zNbnwSCcp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "fig = arch.plot_losses()"
      ],
      "metadata": {
        "id": "ICngkj1sSN_F"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(model.state_dict())"
      ],
      "metadata": {
        "id": "QGvMKMeXUtlw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "$$\n",
        "\\large\n",
        "\\begin{array}{ccccccc}\n",
        "z & = & b & + & w_1x_1 & + & w_2x_2\n",
        "\\\\\n",
        "z & = & -0.0591 & + & 1.1806x_1 & - & 1.8693x_2\n",
        "\\end{array}\n",
        "$$"
      ],
      "metadata": {
        "id": "5G-c56owVu7Z"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DfFy5GC4uv0m"
      },
      "source": [
        "## Making Predictions"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "$$\n",
        "\\Large y =\n",
        "\\begin{cases}\n",
        "1,\\ \\text{if } z \\ge 0\n",
        "\\\\\n",
        "0,\\ \\text{if } z < 0\n",
        "\\end{cases}\n",
        "$$"
      ],
      "metadata": {
        "id": "VqN_FSRda6Ra"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "P9PLEyy3uv0m"
      },
      "outputs": [],
      "source": [
        "# prediction logits (z)\n",
        "logits_val = arch.predict(X_val[:4])\n",
        "logits_val"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kS2JSK4suv0m"
      },
      "source": [
        "$$\n",
        "\\Large y =\n",
        "\\begin{cases}\n",
        "1,\\ \\text{if P}(y=1) \\ge 0.5\n",
        "\\\\\n",
        "0,\\ \\text{if P}(y=1) < 0.5\n",
        "\\end{cases}\n",
        "$$"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J_PRmnyouv0m"
      },
      "source": [
        "$$\n",
        "\\Large y =\n",
        "\\begin{cases}\n",
        "1,\\ \\text{if } \\sigma(z) \\ge 0.5\n",
        "\\\\\n",
        "0,\\ \\text{if } \\sigma(z) < 0.5\n",
        "\\end{cases}\n",
        "$$"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# prediction probabilities\n",
        "prob_val = torch.sigmoid(torch.as_tensor(logits_val[:4]).float())\n",
        "prob_val"
      ],
      "metadata": {
        "id": "7IrqnJhFa_r6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1nNagJnpuv0m"
      },
      "outputs": [],
      "source": [
        "classes = (prob_val >= 0.5).to(torch.int8)\n",
        "classes"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pFSWhDwauv0m"
      },
      "source": [
        "## Decision Boundary"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZkKpvZoxuv0m"
      },
      "source": [
        "$$\n",
        "\\Large\n",
        "\\begin{array}{ccccccccc}\n",
        "z & = &   0 & =   & b & + & w_1x_1 & + & w_2x_2\n",
        "\\\\\n",
        "& & -w_2x_2 & = & b & + & w_1x_1 & &\n",
        "\\\\\n",
        "& & x_2 & = & -\\frac{b}{w_2} & - &\\frac{w_1}{w_2}x_1 & &\n",
        "\\end{array}\n",
        "$$"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_Q8vpEcguv0m"
      },
      "source": [
        "$$\n",
        "\\Large\n",
        "\\begin{array}{ccccccccc}\n",
        "& & x_2 & = & -\\frac{0.0591}{1.8693} & + &\\frac{1.1806}{1.8693}x_1 & &\n",
        "\\\\\n",
        "& & x_2 & = & -0.0316 & + &0.6315x_1 & &\n",
        "\\end{array}\n",
        "$$"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Data Visualization"
      ],
      "metadata": {
        "id": "-L95gpV7jImB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def sigmoid(z):\n",
        "    return 1 / (1 + np.exp(-z))\n",
        "\n",
        "\n",
        "def figure7(X, y, model, device, cm=None, cm_bright=None):\n",
        "    if cm is None:\n",
        "        cm = plt.cm.RdBu\n",
        "    if cm_bright is None:\n",
        "        cm_bright = ListedColormap(['#FF0000', '#0000FF'])\n",
        "    fig = plt.figure(figsize=(15, 4.5))\n",
        "\n",
        "    h = .02  # step size in the mesh\n",
        "\n",
        "    # x_min, x_max = X_train[:, 0].min() - .5, X_train[:, 0].max() + .5\n",
        "    # y_min, y_max = X_train[:, 1].min() - .5, X_train[:, 1].max() + .5\n",
        "\n",
        "    x_min, x_max = -2.25, 2.25\n",
        "    y_min, y_max = -2.25, 2.25\n",
        "\n",
        "    xx, yy = np.meshgrid(np.arange(x_min, x_max, h),\n",
        "                         np.arange(y_min, y_max, h))\n",
        "\n",
        "    logits = model(torch.as_tensor(np.c_[xx.ravel(), yy.ravel()]).float().to(device))\n",
        "    logits = logits.detach().cpu().numpy().reshape(xx.shape)\n",
        "\n",
        "    yhat = sigmoid(logits)\n",
        "\n",
        "    # 1st plot\n",
        "    ax = plt.subplot(1, 3, 1)\n",
        "\n",
        "    contour = ax.contourf(xx, yy, logits, 25, cmap=cm, alpha=.8)\n",
        "    # Plot the training points\n",
        "    ax.scatter(X[:, 0], X[:, 1], c=y, cmap=cm_bright)\n",
        "    # Plot the testing points\n",
        "    #ax.scatter(X_val[:, 0], X_val[:, 1], c=y_val, cmap=cm_bright, edgecolors='k', alpha=0.6)\n",
        "\n",
        "    ax.set_xlim(xx.min(), xx.max())\n",
        "    ax.set_ylim(yy.min(), yy.max())\n",
        "    ax.set_xlabel(r'$X_1$')\n",
        "    ax.set_ylabel(r'$X_2$')\n",
        "    ax.set_title(r'$z = b + w_1x_1 + w_2x_2$')\n",
        "    ax.grid(False)\n",
        "    ax_c = plt.colorbar(contour)\n",
        "    ax_c.set_label(\"$z$\", rotation=0)\n",
        "\n",
        "    # 2nd plot\n",
        "    ax = fig.add_subplot(1, 3, 2, projection='3d')\n",
        "\n",
        "    surf = ax.plot_surface(xx, yy, yhat, rstride=1, cstride=1, alpha=.5, cmap=cm, linewidth=0, antialiased=True, vmin=0, vmax=1)\n",
        "    # Plot the training points\n",
        "    ax.scatter(X[:, 0], X[:, 1], c=y, cmap=cm_bright)\n",
        "    # Plot the testing points\n",
        "    #ax.scatter(X_val[:, 0], X_val[:, 1], c=y_val, cmap=cm_bright, edgecolors='k', alpha=0.6)\n",
        "\n",
        "    ax.set_xlim(xx.min(), xx.max())\n",
        "    ax.set_ylim(yy.min(), yy.max())\n",
        "    ax.set_xlabel(r'$X_1$')\n",
        "    ax.set_ylabel(r'$X_2$')\n",
        "    ax.set_title(r'$\\sigma(z) = P(y=1)$')\n",
        "\n",
        "    ax_c = plt.colorbar(surf)\n",
        "    ax_c.set_ticks([0, .25, .5, .75, 1])\n",
        "    ax.view_init(30, 220)\n",
        "\n",
        "    # 3rd plot\n",
        "    ax = plt.subplot(1, 3, 3)\n",
        "\n",
        "    ax.contour(xx, yy, yhat, levels=[.5], cmap=\"Greys\", vmin=0, vmax=1)\n",
        "    contour = ax.contourf(xx, yy, yhat, 25, cmap=cm, alpha=.8, vmin=0, vmax=1)\n",
        "    # Plot the training points\n",
        "    ax.scatter(X[:, 0], X[:, 1], c=y, cmap=cm_bright)\n",
        "    # Plot the testing points\n",
        "    #ax.scatter(X_val[:, 0], X_val[:, 1], c=y_val, cmap=cm_bright, edgecolors='k', alpha=0.6)\n",
        "\n",
        "    ax.set_xlim(xx.min(), xx.max())\n",
        "    ax.set_ylim(yy.min(), yy.max())\n",
        "    ax.set_xlabel(r'$X_1$')\n",
        "    ax.set_ylabel(r'$X_2$')\n",
        "    ax.set_title(r'$\\sigma(z) = P(y=1)$')\n",
        "    ax.grid(False)\n",
        "\n",
        "    ax_c = plt.colorbar(contour)\n",
        "    ax_c.set_ticks([0, .25, .5, .75, 1])\n",
        "\n",
        "    plt.tight_layout()\n",
        "\n",
        "    return fig"
      ],
      "metadata": {
        "id": "RxdES3s4jKvQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JDRPlA47uv0n"
      },
      "outputs": [],
      "source": [
        "# Training set\n",
        "fig = figure7(X_train, y_train, arch.model, arch.device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4O0q4Zliuv0n"
      },
      "outputs": [],
      "source": [
        "# Validation set\n",
        "fig = figure7(X_val, y_val, arch.model, arch.device)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Classification Threshold"
      ],
      "metadata": {
        "id": "xjmZQOPCUaO6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "logits_val = arch.predict(X_val)\n",
        "probabilities_val = sigmoid(logits_val).squeeze()\n",
        "threshold = 0.5"
      ],
      "metadata": {
        "id": "gA6T70VhUc42"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Data Visualization"
      ],
      "metadata": {
        "id": "UEiFZDAUUyxk"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def probability_line(ax, y, probs, threshold, shift=0.0, annot=False, colors=None):\n",
        "    if colors is None:\n",
        "        colors = ['r', 'b']\n",
        "    ax.grid(False)\n",
        "    ax.set_ylim([-.1, .1])\n",
        "    ax.axes.get_yaxis().set_visible(False)\n",
        "    ax.plot([0, 1], [0, 0], linewidth=2, c='k', zorder=1)\n",
        "    ax.plot([0, 0], [-.1, .1], c='k', zorder=1)\n",
        "    ax.plot([1, 1], [-.1, .1], c='k', zorder=1)\n",
        "\n",
        "    tn = (y == 0) & (probs < threshold)\n",
        "    fn = (y == 0) & (probs >= threshold)\n",
        "    tp = (y == 1) & (probs >= threshold)\n",
        "    fp = (y == 1) & (probs < threshold)\n",
        "\n",
        "    ax.plot([threshold, threshold], [-.1, .1], c='k', zorder=1, linestyle='--')\n",
        "    ax.scatter(probs[tn], np.zeros(tn.sum()) + shift, c=colors[0], s=150, zorder=2, edgecolor=colors[0], linewidth=3)\n",
        "    ax.scatter(probs[fn], np.zeros(fn.sum()) + shift, c=colors[0], s=150, zorder=2, edgecolor=colors[1], linewidth=3)\n",
        "\n",
        "    ax.scatter(probs[tp], np.zeros(tp.sum()) - shift, c=colors[1], s=150, zorder=2, edgecolor=colors[1], linewidth=3)\n",
        "    ax.scatter(probs[fp], np.zeros(fp.sum()) - shift, c=colors[1], s=150, zorder=2, edgecolor=colors[0], linewidth=3)\n",
        "\n",
        "    ax.set_xlabel(r'$\\sigma(z) = P(y=1)$')\n",
        "    ax.set_title('Threshold = {}'.format(threshold))\n",
        "\n",
        "    if annot:\n",
        "        ax.annotate('TN', xy=(.20, .03), c='k', weight='bold', fontsize=20)\n",
        "        ax.annotate('FN', xy=(.20, -.08), c='k', weight='bold', fontsize=20)\n",
        "        ax.annotate('FP', xy=(.70, .03), c='k', weight='bold', fontsize=20)\n",
        "        ax.annotate('TP', xy=(.70, -.08), c='k', weight='bold', fontsize=20)\n",
        "    return ax"
      ],
      "metadata": {
        "id": "YQmE3fhiUxwf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def probability_contour(ax, model, device, X, y, threshold, cm=None, cm_bright=None):\n",
        "    if cm is None:\n",
        "        cm = plt.cm.RdBu\n",
        "    if cm_bright is None:\n",
        "        cm_bright = ListedColormap(['#FF0000', '#0000FF'])\n",
        "\n",
        "    h = .02  # step size in the mesh\n",
        "\n",
        "    x_min, x_max = -2.25, 2.25\n",
        "    y_min, y_max = -2.25, 2.25\n",
        "\n",
        "    xx, yy = np.meshgrid(np.arange(x_min, x_max, h),\n",
        "                         np.arange(y_min, y_max, h))\n",
        "\n",
        "    logits = model(torch.as_tensor(np.c_[xx.ravel(), yy.ravel()]).float().to(device))\n",
        "    logits = logits.detach().cpu().numpy().reshape(xx.shape)\n",
        "\n",
        "    yhat = sigmoid(logits)\n",
        "\n",
        "    ax.contour(xx, yy, yhat, levels=[threshold], cmap=\"Greys\", vmin=0, vmax=1)\n",
        "    contour = ax.contourf(xx, yy, yhat, 25, cmap=cm, alpha=.8, vmin=0, vmax=1)\n",
        "    # Plot the training points\n",
        "    ax.scatter(X[:, 0], X[:, 1], c=y, cmap=cm_bright, edgecolors='k')\n",
        "    # Plot the testing points\n",
        "    #ax.scatter(X_val[:, 0], X_val[:, 1], c=y_val, cmap=cm_bright, edgecolors='k', alpha=0.6)\n",
        "\n",
        "    ax.set_xlim(xx.min(), xx.max())\n",
        "    ax.set_ylim(yy.min(), yy.max())\n",
        "    ax.set_xlabel(r'$X_1$')\n",
        "    ax.set_ylabel(r'$X_2$')\n",
        "    ax.set_title(r'$\\sigma(z) = P(y=1)$')\n",
        "    ax.grid(False)\n",
        "\n",
        "    ax_c = plt.colorbar(contour)\n",
        "    ax_c.set_ticks([0, .25, .5, .75, 1])\n",
        "    return ax"
      ],
      "metadata": {
        "id": "_FHVZNJwVo4x"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def figure9(x, y, model, device, probabilities, threshold, shift=0.0, annot=False, cm=None, cm_bright=None):\n",
        "    fig = plt.figure(figsize=(15, 5))\n",
        "    gs = fig.add_gridspec(3, 3)\n",
        "\n",
        "    ax = fig.add_subplot(gs[:, 0])\n",
        "    probability_contour(ax, model, device, x, y, threshold, cm, cm_bright)\n",
        "\n",
        "    if cm_bright is None:\n",
        "        colors = ['r', 'b']\n",
        "    else:\n",
        "        colors = cm_bright.colors\n",
        "\n",
        "    ax = fig.add_subplot(gs[1, 1:])\n",
        "    probability_line(ax, y, probabilities, threshold, shift, annot, colors)\n",
        "\n",
        "    fig.tight_layout()\n",
        "    return fig"
      ],
      "metadata": {
        "id": "yWd6PBS4Vc29"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def figure10(y, probabilities, threshold, shift, annot, colors=None):\n",
        "    fig, ax = plt.subplots(1, 1, figsize=(10, 2))\n",
        "    probability_line(ax, y, probabilities, threshold, shift, annot, colors)\n",
        "    fig.tight_layout()\n",
        "    return fig"
      ],
      "metadata": {
        "id": "VN1Hzs1uUrte"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "fig = figure9(X_val, y_val, arch.model, arch.device, probabilities_val, threshold)"
      ],
      "metadata": {
        "id": "ChncVa1CVUCy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Confusion Matrix"
      ],
      "metadata": {
        "id": "s1x4-ELpVzxv"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "fig = figure10(y_val, probabilities_val, threshold, 0.04, True)"
      ],
      "metadata": {
        "id": "iL_xI6-gWdyA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Metrics"
      ],
      "metadata": {
        "id": "VJM8OYWlWeLd"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def split_cm(cm):\n",
        "    # Actual negatives go in the top row,\n",
        "    # above the probability line\n",
        "    actual_negative = cm[0]\n",
        "    # Predicted negatives go in the first column\n",
        "    tn = actual_negative[0]\n",
        "    # Predicted positives go in the second column\n",
        "    fp = actual_negative[1]\n",
        "\n",
        "    # Actual positives go in the bottow row,\n",
        "    # below the probability line\n",
        "    actual_positive = cm[1]\n",
        "    # Predicted negatives go in the first column\n",
        "    fn = actual_positive[0]\n",
        "    # Predicted positives go in the second column\n",
        "    tp = actual_positive[1]\n",
        "\n",
        "    return tn, fp, fn, tp"
      ],
      "metadata": {
        "id": "PJar7XJP6ybh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### True and False Positive Rates"
      ],
      "metadata": {
        "id": "MYsBSAeG7M3d"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "$$\n",
        "\\Large \\text{TPR} = \\frac{\\text{TP}}{\\text{TP + FN}} \\ \\ \\  \\text{FPR} = \\frac{\\text{FP}}{\\text{FP + TN}}\n",
        "$$"
      ],
      "metadata": {
        "id": "zJ3dBBRw6-pC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def tpr_fpr(cm):\n",
        "    tn, fp, fn, tp = split_cm(cm)\n",
        "\n",
        "    tpr = tp / (tp + fn)\n",
        "    fpr = fp / (fp + tn)\n",
        "\n",
        "    return tpr, fpr"
      ],
      "metadata": {
        "id": "Bc7W7H3B6-4H"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Precision and Recall"
      ],
      "metadata": {
        "id": "sTMQN-bV7Cbm"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "$$\n",
        "\\Large \\text{Recall} = \\frac{\\text{TP}}{\\text{TP + FN}} \\ \\ \\  \\text{Precision} = \\frac{\\text{TP}}{\\text{TP + FP}}\n",
        "$$"
      ],
      "metadata": {
        "id": "kESotky77SES"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def precision_recall(cm):\n",
        "    tn, fp, fn, tp = split_cm(cm)\n",
        "\n",
        "    precision = tp / (tp + fp)\n",
        "    recall = tp / (tp + fn)\n",
        "\n",
        "    return precision, recall"
      ],
      "metadata": {
        "id": "xUsqgpDq7VC4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Accuracy"
      ],
      "metadata": {
        "id": "mbUkpUS97Zpm"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "$$\n",
        "\\Large \\text{Accuracy} = \\frac{\\text{TP+TN}}{\\text{TP+TN+FP+FN}}\n",
        "$$"
      ],
      "metadata": {
        "id": "T-hHitgf7diN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "logits_val = arch.predict(X_val)\n",
        "probabilities_val = sigmoid(logits_val).squeeze()\n",
        "cm_thresh50 = confusion_matrix(y_val, (probabilities_val >= 0.5))\n",
        "cm_thresh50"
      ],
      "metadata": {
        "id": "0xdxxJkO7hVw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "precision_recall(cm_thresh50)"
      ],
      "metadata": {
        "id": "ok_un-pG-lw_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "acc = accuracy_score(y_val, (probabilities_val >= 0.5))\n",
        "acc"
      ],
      "metadata": {
        "id": "oMHt6Lxc-s8d"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tpr_fpr(cm_thresh50)"
      ],
      "metadata": {
        "id": "zShHILGm_FMB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "YLqouqlN_L7d"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}