Bagging_Classifier_from_Scratch.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Bagging Classifier from Scratch.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mXths-CSWX5X",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e0ec1524-e7a8-4771-8e8f-2b9312185322"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/gdrive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount(\"/content/gdrive\")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Extracting the MNIST Dataset"
      ],
      "metadata": {
        "id": "PQITvVbwINgA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "! cp /content/gdrive/MyDrive/SML_Assignment-4/mnist.zip .\n",
        "! unzip mnist.zip -d mnist/"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MxQgWbcovKK1",
        "outputId": "067ea24e-60aa-4785-d49b-bf72966c8263"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Archive:  mnist.zip\n",
            "   creating: mnist/mnist/\n",
            "  inflating: mnist/mnist/readme.txt  \n",
            "  inflating: mnist/mnist/train-labels-idx1-ubyte.gz  \n",
            "  inflating: mnist/mnist/train-images-idx3-ubyte.gz  \n",
            "  inflating: mnist/mnist/t10k-images-idx3-ubyte.gz  \n",
            "  inflating: mnist/mnist/t10k-labels-idx1-ubyte.gz  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import matplotlib.pyplot as plt"
      ],
      "metadata": {
        "id": "z2HOE_ywvKlB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Visualizing one image of the MNIST Dataset"
      ],
      "metadata": {
        "id": "gsxCMRNWIRa8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import gzip\n",
        "def images_file_read(file_name):\n",
        "    with gzip.open(file_name, 'r') as f:\n",
        "        # first 4 bytes is a magic number\n",
        "        magic_number = int.from_bytes(f.read(4), 'big')\n",
        "        # second 4 bytes is the number of images\n",
        "        image_count = int.from_bytes(f.read(4), 'big')\n",
        "        # third 4 bytes is the row count\n",
        "        row_count = int.from_bytes(f.read(4), 'big')\n",
        "        # fourth 4 bytes is the column count\n",
        "        column_count = int.from_bytes(f.read(4), 'big')\n",
        "        # rest is the image pixel data, each pixel is stored as an unsigned byte\n",
        "        # pixel values are 0 to 255\n",
        "        image_data = f.read()\n",
        "        images = np.frombuffer(image_data, dtype=np.uint8)\\\n",
        "            .reshape((image_count, row_count, column_count))\n",
        "        return images\n",
        "def labels_file_read(file_name):\n",
        "    with gzip.open(file_name, 'r') as f:\n",
        "        # first 4 bytes is a magic number\n",
        "        magic_number = int.from_bytes(f.read(4), 'big')\n",
        "        # second 4 bytes is the number of labels\n",
        "        label_count = int.from_bytes(f.read(4), 'big')\n",
        "        # rest is the label data, each label is stored as unsigned byte\n",
        "        # label values are 0 to 9\n",
        "        label_data = f.read()\n",
        "        labels = np.frombuffer(label_data, dtype=np.uint8)\n",
        "        return labels\n",
        "train_x2 = images_file_read(\"mnist/mnist/train-images-idx3-ubyte.gz\")\n",
        "print(train_x2.shape)\n",
        "train_y2 = labels_file_read(\"mnist/mnist/train-labels-idx1-ubyte.gz\")\n",
        "test_x2 = images_file_read(\"mnist/mnist/t10k-images-idx3-ubyte.gz\")\n",
        "print(test_x2.shape)\n",
        "test_y2 = labels_file_read(\"mnist/mnist/t10k-labels-idx1-ubyte.gz\")\n",
        "plt.imshow(train_x2[0])\n",
        "print(\"Label :\" ,  train_y2[0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 319
        },
        "id": "iseIQR2yvP4v",
        "outputId": "87281cad-f361-468c-deb9-3d4429ac36d8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(60000, 28, 28)\n",
            "(10000, 28, 28)\n",
            "Label : 5\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAOZ0lEQVR4nO3dbYxc5XnG8euKbezamMQbB9chLjjgFAg0Jl0ZEBZQobgOqgSoCsSKIkJpnSY4Ca0rQWlV3IpWbpUQUUqRTHExFS+BBIQ/0CTUQpCowWWhBgwEDMY0NmaNWYENIX5Z3/2w42iBnWeXmTMv3vv/k1Yzc+45c24NXD5nznNmHkeEAIx/H+p0AwDag7ADSRB2IAnCDiRB2IEkJrZzY4d5ckzRtHZuEkjlV3pbe2OPR6o1FXbbiyVdJ2mCpH+LiJWl50/RNJ3qc5rZJICC9bGubq3hw3jbEyTdIOnzkk6UtMT2iY2+HoDWauYz+wJJL0TE5ojYK+lOSedV0xaAqjUT9qMk/WLY4621Ze9ie6ntPtt9+7Snic0BaEbLz8ZHxKqI6I2I3kma3OrNAaijmbBvkzRn2ONP1JYB6ELNhP1RSfNsz7V9mKQvSlpbTVsAqtbw0FtE7Le9TNKPNDT0tjoinq6sMwCVamqcPSLul3R/Rb0AaCEulwWSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiCJpmZxRffzxPJ/4gkfm9nS7T/3F8fUrQ1OPVBc9+hjdxTrU7/uYv3Vaw+rW3u893vFdXcOvl2sn3r38mL9uD9/pFjvhKbCbnuLpN2SBiXtj4jeKpoCUL0q9uy/FxE7K3gdAC3EZ3YgiWbDHpJ+bPsx20tHeoLtpbb7bPft054mNwegUc0exi+MiG22j5T0gO2fR8TDw58QEaskrZKkI9wTTW4PQIOa2rNHxLba7Q5J90paUEVTAKrXcNhtT7M9/eB9SYskbayqMQDVauYwfpake20ffJ3bI+KHlXQ1zkw4YV6xHpMnFeuvnPWRYv2d0+qPCfd8uDxe/JPPlMebO+k/fzm9WP/Hf1lcrK8/+fa6tZf2vVNcd2X/54r1j//k0PtE2nDYI2KzpM9U2AuAFmLoDUiCsANJEHYgCcIOJEHYgST4imsFBs/+bLF+7S03FOufmlT/q5jj2b4YLNb/5vqvFOsT3y4Pf51+97K6tenb9hfXnbyzPDQ3tW99sd6N2LMDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKMs1dg8nOvFOuP/WpOsf6pSf1VtlOp5dtPK9Y3v1X+Kepbjv1+3dqbB8rj5LP++b+L9VY69L7AOjr27EAShB1IgrADSRB2IAnCDiRB2IEkCDuQhCPaN6J4hHviVJ/Ttu11i4FLTi/Wdy0u/9zzhCcPL9af+Pr1H7ing67Z+TvF+qNnlcfRB994s1iP0+v/APGWbxZX1dwlT5SfgPdZH+u0KwZGnMuaPTuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJME4exeYMPOjxfrg6wPF+ku31x8rf/rM1cV1F/zDN4r1I2/o3HfK8cE1Nc5ue7XtHbY3DlvWY/sB25tqtzOqbBhA9cZyGH+LpPfOen+lpHURMU/SutpjAF1s1LBHxMOS3nsceZ6kNbX7aySdX3FfACrW6G/QzYqI7bX7r0qaVe+JtpdKWipJUzS1wc0BaFbTZ+Nj6Axf3bN8EbEqInojoneSJje7OQANajTs/bZnS1Ltdkd1LQFohUbDvlbSxbX7F0u6r5p2ALTKqJ/Zbd8h6WxJM21vlXS1pJWS7rJ9qaSXJV3YyibHu8Gdrze1/r5djc/v/ukvPVOsv3bjhPILHCjPsY7uMWrYI2JJnRJXxwCHEC6XBZIg7EAShB1IgrADSRB2IAmmbB4HTrji+bq1S04uD5r8+9HrivWzvnBZsT79e48U6+ge7NmBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnG2ceB0rTJr3/thOK6/7f2nWL9ymtuLdb/8sILivX43w/Xrc35+58V11Ubf+Y8A/bsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEUzYnN/BHpxfrt1397WJ97sQpDW/707cuK9bn3bS9WN+/eUvD2x6vmpqyGcD4QNiBJAg7kARhB5Ig7EAShB1IgrADSTDOjqI4Y36xfsTKrcX6HZ/8UcPbPv7BPy7Wf/tv63+PX5IGN21ueNuHqqbG2W2vtr3D9sZhy1bY3mZ7Q+3v3CobBlC9sRzG3yJp8QjLvxsR82t/91fbFoCqjRr2iHhY0kAbegHQQs2coFtm+8naYf6Mek+yvdR2n+2+fdrTxOYANKPRsN8o6VhJ8yVtl/Sdek+MiFUR0RsRvZM0ucHNAWhWQ2GPiP6IGIyIA5JukrSg2rYAVK2hsNuePezhBZI21nsugO4w6ji77TsknS1ppqR+SVfXHs+XFJK2SPpqRJS/fCzG2cejCbOOLNZfuei4urX1V1xXXPdDo+yLvvTSomL9zYWvF+vjUWmcfdRJIiJiyQiLb266KwBtxeWyQBKEHUiCsANJEHYgCcIOJMFXXNExd20tT9k81YcV67+MvcX6H3zj8vqvfe/64rqHKn5KGgBhB7Ig7EAShB1IgrADSRB2IAnCDiQx6rfekNuBheWfkn7xC+Upm0+av6VubbRx9NFcP3BKsT71vr6mXn+8Yc8OJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kwzj7OufekYv35b5bHum86Y02xfuaU8nfKm7En9hXrjwzMLb/AgVF/3TwV9uxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kATj7IeAiXOPLtZfvOTjdWsrLrqzuO4fHr6zoZ6qcFV/b7H+0HWnFesz1pR/dx7vNuqe3fYc2w/afsb207a/VVveY/sB25tqtzNa3y6ARo3lMH6/pOURcaKk0yRdZvtESVdKWhcR8yStqz0G0KVGDXtEbI+Ix2v3d0t6VtJRks6TdPBayjWSzm9VkwCa94E+s9s+RtIpktZLmhURBy8+flXSrDrrLJW0VJKmaGqjfQJo0pjPxts+XNIPJF0eEbuG12JodsgRZ4iMiFUR0RsRvZM0ualmATRuTGG3PUlDQb8tIu6pLe63PbtWny1pR2taBFCFUQ/jbVvSzZKejYhrh5XWSrpY0sra7X0t6XAcmHjMbxXrb/7u7GL9or/7YbH+px+5p1hvpeXby8NjP/vX+sNrPbf8T3HdGQcYWqvSWD6znyHpy5Kesr2htuwqDYX8LtuXSnpZ0oWtaRFAFUYNe0T8VNKIk7tLOqfadgC0CpfLAkkQdiAJwg4kQdiBJAg7kARfcR2jibN/s25tYPW04rpfm/tQsb5ken9DPVVh2baFxfrjN5anbJ75/Y3Fes9uxsq7BXt2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUgizTj73t8v/2zx3j8bKNavOu7+urVFv/F2Qz1VpX/wnbq1M9cuL657/F//vFjveaM8Tn6gWEU3Yc8OJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0mkGWffcn7537XnT767Zdu+4Y1ji/XrHlpUrHuw3o/7Djn+mpfq1ub1ry+uO1isYjxhzw4kQdiBJAg7kARhB5Ig7EAShB1IgrADSTgiyk+w50i6VdIsSSFpVURcZ3uFpD+R9FrtqVdFRP0vfUs6wj1xqpn4FWiV9bFOu2JgxAszxnJRzX5JyyPicdvTJT1m+4Fa7bsR8e2qGgXQOmOZn327pO21+7ttPyvpqFY3BqBaH+gzu+1jJJ0i6eA1mMtsP2l7te0ZddZZarvPdt8+7WmqWQCNG3PYbR8u6QeSLo+IXZJulHSspPka2vN/Z6T1ImJVRPRGRO8kTa6gZQCNGFPYbU/SUNBvi4h7JCki+iNiMCIOSLpJ0oLWtQmgWaOG3bYl3Szp2Yi4dtjy2cOedoGk8nSeADpqLGfjz5D0ZUlP2d5QW3aVpCW252toOG6LpK+2pEMAlRjL2fifShpp3K44pg6gu3AFHZAEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IIlRf0q60o3Zr0l6ediimZJ2tq2BD6Zbe+vWviR6a1SVvR0dER8bqdDWsL9v43ZfRPR2rIGCbu2tW/uS6K1R7eqNw3ggCcIOJNHpsK/q8PZLurW3bu1LordGtaW3jn5mB9A+nd6zA2gTwg4k0ZGw215s+znbL9i+shM91GN7i+2nbG+w3dfhXlbb3mF747BlPbYfsL2pdjviHHsd6m2F7W21926D7XM71Nsc2w/afsb207a/VVve0feu0Fdb3re2f2a3PUHS85I+J2mrpEclLYmIZ9raSB22t0jqjYiOX4Bh+0xJb0m6NSJOqi37J0kDEbGy9g/ljIi4okt6WyHprU5P412brWj28GnGJZ0v6Svq4HtX6OtCteF968SefYGkFyJic0TslXSnpPM60EfXi4iHJQ28Z/F5ktbU7q/R0P8sbVent64QEdsj4vHa/d2SDk4z3tH3rtBXW3Qi7EdJ+sWwx1vVXfO9h6Qf237M9tJONzOCWRGxvXb/VUmzOtnMCEadxrud3jPNeNe8d41Mf94sTtC938KI+Kykz0u6rHa42pVi6DNYN42djmka73YZYZrxX+vke9fo9OfN6kTYt0maM+zxJ2rLukJEbKvd7pB0r7pvKur+gzPo1m53dLifX+umabxHmmZcXfDedXL6806E/VFJ82zPtX2YpC9KWtuBPt7H9rTaiRPZniZpkbpvKuq1ki6u3b9Y0n0d7OVdumUa73rTjKvD713Hpz+PiLb/STpXQ2fkX5T0V53ooU5fn5T0RO3v6U73JukODR3W7dPQuY1LJX1U0jpJmyT9l6SeLurtPyQ9JelJDQVrdod6W6ihQ/QnJW2o/Z3b6feu0Fdb3jculwWS4AQdkARhB5Ig7EAShB1IgrADSRB2IAnCDiTx/65XcTNOWsh5AAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(train_x2.shape)\n",
        "print(test_x2.shape)\n",
        "print(train_y2.shape)\n",
        "print(test_y2.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fIX4b0_Rvi3j",
        "outputId": "4821a62a-c174-4ae2-f2fc-22c246d8c691"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(60000, 28, 28)\n",
            "(10000, 28, 28)\n",
            "(60000,)\n",
            "(10000,)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Normalizing and Centralizing the dataset"
      ],
      "metadata": {
        "id": "Buwul_P2IXLS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "train_x = (train_x2 / 255) - 0.5\n",
        "test_x = (test_x2 / 255) - 0.5"
      ],
      "metadata": {
        "id": "SGpSHOozz3c7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_x = train_x.reshape((-1, 784))\n",
        "test_x = test_x.reshape((-1, 784))"
      ],
      "metadata": {
        "id": "DgPcrngU0F7d"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from random import randint"
      ],
      "metadata": {
        "id": "6O_fOVW7w0O1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def create_subset(train_x,train_y,rf):\n",
        "  subset_x = list()\n",
        "  subset_y = list()\n",
        "  n = int(len(train_x)*rf)\n",
        "  for i in range(n):\n",
        "    index = randint(0,len(train_x)-1)\n",
        "    subset_x.append(train_x[index])\n",
        "    subset_y.append(train_y[index])\n",
        "  return [subset_x,subset_y]"
      ],
      "metadata": {
        "id": "RXz97BwsvoT1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.tree import DecisionTreeClassifier\n",
        "from scipy import stats as s"
      ],
      "metadata": {
        "id": "zzE_XmlKx-B2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def bagging_classifier(bags,train_x,train_y,test_x):\n",
        "  clf2 = bagging(bags,train_x,train_y)\n",
        "  ans = predictor(clf2,test_x)\n",
        "  maxLi = list()\n",
        "  for i in range(len(ans[0])):\n",
        "      temp = list()\n",
        "      for j in ans:\n",
        "        temp.append(j[i])\n",
        "      res = int(s.mode(temp)[0])\n",
        "      maxLi.append(res)\n",
        "  return maxLi"
      ],
      "metadata": {
        "id": "IbQKbmsWESWs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Method defined for Bagging Classifier"
      ],
      "metadata": {
        "id": "nSkzYsT6IdAl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def bagging(bags,train_x,train_y):\n",
        "  classifier_big_bag = list()\n",
        "  for i in range(bags):\n",
        "    d1 = create_subset(train_x,train_y,0.6)\n",
        "    d_x = np.array(d1[0])\n",
        "    d_y = np.array(d1[1])\n",
        "    clf = DecisionTreeClassifier(max_depth = 4)\n",
        "    clf.fit(d_x,d_y)\n",
        "    classifier_big_bag.append(clf)\n",
        "\n",
        "  return classifier_big_bag"
      ],
      "metadata": {
        "id": "2QtBKr5i13l0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def predictor(big_bag,test_x):\n",
        "  answers = list()\n",
        "  for i in range(len(big_bag)):\n",
        "    answers.append(big_bag[i].predict(test_x))\n",
        "  return answers"
      ],
      "metadata": {
        "id": "4ShkWk_D6cGp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import accuracy_score\n",
        "accuracy_score(test_y2,bagging_classifier(3,train_x,train_y2,test_x))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MOtIfIlsCj6c",
        "outputId": "e784c795-6a71-4dfa-c786-2c22603e79df"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.6975"
            ]
          },
          "metadata": {},
          "execution_count": 82
        }
      ]
    }
  ]
}