Complete Code - Neural Networks in python.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "obvious-looking",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "interracial-chess",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = np.loadtxt('train_X.csv', delimiter = ',').T\n",
    "Y_train = np.loadtxt('train_label.csv', delimiter = ',').T\n",
    "\n",
    "X_test = np.loadtxt('test_X.csv', delimiter = ',').T\n",
    "Y_test = np.loadtxt('test_label.csv', delimiter = ',').T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "continental-paragraph",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of X_train : (784, 1000)\n",
      "shape of Y_train : (10, 1000)\n",
      "shape of X_test : (784, 350)\n",
      "shape of Y_test : (10, 350)\n"
     ]
    }
   ],
   "source": [
    "print(\"shape of X_train :\", X_train.shape)\n",
    "print(\"shape of Y_train :\", Y_train.shape)\n",
    "print(\"shape of X_test :\", X_test.shape)\n",
    "print(\"shape of Y_test :\", Y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "certain-elizabeth",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAN6klEQVR4nO3dX6xV5Z3G8efRab0ALgAjHuzJ2KlHk2IyVogxGTNxbNoweIEkRkr8L+Y0WhM0JiN2TIqRSdCZzlx40XiwWhgdSImSGtS2DBJxjGkE4ij+KToGU8gR4nBRIUYUfnNxFpOjnP2uw15r7bXh/X6Sk7P3+u291o8ND2vt9e61X0eEAJz+zmi7AQC9QdiBTBB2IBOEHcgEYQcy8Re93JhtTv0DDYsIT7S80p7d9nzbf7T9ge3lVdYFoFnudpzd9pmSdkv6gaS9kl6XtCQi3kk8hz070LAm9uyXSfogIj6MiCOS1ktaWGF9ABpUJeznSfrTuPt7i2VfYXvY9nbb2ytsC0BFjZ+gi4gRSSMSh/FAm6rs2fdJGhx3/1vFMgB9qErYX5c0ZPvbtr8p6UeSnqunLQB16/owPiK+tH2XpN9JOlPSExHxdm2dAahV10NvXW2M9+xA4xr5UA2AUwdhBzJB2IFMEHYgE4QdyARhBzLR0+vZgfFuuOGGZP2BBx5I1oeGhpL1OXPmdKy99957yeeejtizA5kg7EAmCDuQCcIOZIKwA5kg7EAmGHpDJTNnzkzWH3nkkY61W265Jfncsisyy+oDAwMdawy9AThtEXYgE4QdyARhBzJB2IFMEHYgE4QdyATj7Ei6+OKLk/UNGzYk62WXoVbx0ksvJeuvvvpqY9s+FbFnBzJB2IFMEHYgE4QdyARhBzJB2IFMEHYgE4yzI+nQoUPJ+tSpUxvb9ujoaLL+1FNPJetHjhyps51TXqWw294j6VNJRyV9GRHz6mgKQP3q2LP/XUR8UsN6ADSI9+xAJqqGPST93vYO28MTPcD2sO3ttrdX3BaACqoexl8REftsnyNps+33ImLb+AdExIikEUmynf6GQACNqbRnj4h9xe8DkjZKuqyOpgDUr+uw255ie9rx25J+KGlXXY0BqFeVw/hZkjbaPr6e/4iI39bSFXpm4cKFyfrixYuT9dmzZ3e97WPHjiXrIyMjyfratWu73naOug57RHwo6a9r7AVAgxh6AzJB2IFMEHYgE4QdyARhBzLBJa6ZK/uq5+uuuy5ZL5s2OWXTpk3J+kMPPdT1unEi9uxAJgg7kAnCDmSCsAOZIOxAJgg7kAnCDmSCcfbTwODgYMfaHXfckXzufffdl6xXGUeX0mPlDz74YKV14+SwZwcyQdiBTBB2IBOEHcgEYQcyQdiBTBB2IBOMs58C5s6dm6y/+OKLHWszZsyou52vKLvmfOXKlY1uH5PHnh3IBGEHMkHYgUwQdiAThB3IBGEHMkHYgUwwzt4HUtejS9LTTz+drM+cObPrbZdNm7xjx45kvWxa5aNHj550T2hG6Z7d9hO2D9jeNW7ZDNubbb9f/J7ebJsAqprMYfyvJM3/2rLlkrZExJCkLcV9AH2sNOwRsU3Swa8tXihpTXF7jaRr6m0LQN26fc8+KyJGi9sfS5rV6YG2hyUNd7kdADWpfIIuIsJ2x28ljIgRSSOSlHocgGZ1O/S23/aAJBW/D9TXEoAmdBv25yTdXNy+WdJv6mkHQFNKD+Ntr5N0paSzbe+V9DNJqyT92vZSSR9JSk/inbnZs2cn6w8//HCyfsEFFyTrTc6RvmjRoq7Xjf5SGvaIWNKh9P2aewHQID4uC2SCsAOZIOxAJgg7kAnCDmSCS1x7YPHixZXqVYbW1q1bl6zfeOONXa8bpxb27EAmCDuQCcIOZIKwA5kg7EAmCDuQCcIOZIJx9h649NJLG13/7t27O9buv//+Rre9ZEmniyLHnHvuuR1r99xzT/K5VT5fUObw4cPJ+urVq5P19evXJ+ujo6PJehvYswOZIOxAJgg7kAnCDmSCsAOZIOxAJgg7kAk3OZZ5wsZO0xlhHn300WT9zjvvTNbPOCP9f27ZtMqpr3u2nXzu1VdfnawvXbo0Wa+irLcm/202ve2bbropWS+bhruKiJjwD8eeHcgEYQcyQdiBTBB2IBOEHcgEYQcyQdiBTDDOPkkXXnhhx9qGDRuSz50zZ06yfjqPN+/cubNj7Ysvvkg+t+zzBWXmzp3bsXbWWWcln1v1z71t27Zk/aqrrqq0/pSux9ltP2H7gO1d45atsL3P9hvFz4I6mwVQv8kcxv9K0vwJlv9bRFxS/LxQb1sA6lYa9ojYJulgD3oB0KAqJ+jusv1mcZg/vdODbA/b3m57e4VtAaio27D/QtJ3JF0iaVTSzzs9MCJGImJeRMzrclsAatBV2CNif0QcjYhjklZLuqzetgDUrauw2x4Yd3eRpF2dHgugP5R+b7ztdZKulHS27b2SfibpStuXSApJeyT9uLkW+0PqmvGycfRT2WuvvZasb926NVlfuXJlx9qRI0e66mmyUt/XXzbOvWrVqrrbaV1p2CNiolkAftlALwAaxMdlgUwQdiAThB3IBGEHMkHYgUwwZXNh2rRpyfrQ0FCPOjl5qSmbn3/++eRzX3nllWR98+bNyfpnn32WrLcpdXnt9ddf3+i2y6Z8bgN7diAThB3IBGEHMkHYgUwQdiAThB3IBGEHMsE4e2HFihXJ+q233trYtg8fPpysL1u2LFl/8skn62ynZwYHB5P1gYGBZH3x4sXJ+u23396xVva5is8//zxZL3vN161bl6y3gT07kAnCDmSCsAOZIOxAJgg7kAnCDmSCsAOZYJy9MDo6mqw3OW3ylClTkvXbbrstWZ8/f6J5N8ekrumWpJdffjlZL3PRRRcl6wsWdJ7g99prr00+t83poteuXZusP/bYY1311Cb27EAmCDuQCcIOZIKwA5kg7EAmCDuQCcIOZMJNjh+fsDG7dxs7SZdffnlj6964cWOyfs455yTrTf4d2W5026nvra/6nfPr169P1l944YWOtaani25TREz4l1q6Z7c9aHur7Xdsv217WbF8hu3Ntt8vfk+vu2kA9ZnMYfyXku6NiO9KulzST2x/V9JySVsiYkjSluI+gD5VGvaIGI2IncXtTyW9K+k8SQslrSketkbSNQ31CKAGJ/XZeNvnS/qepD9ImhURxz9Q/rGkWR2eMyxpuEKPAGow6bPxtqdKekbS3RHx5/G1GDuLM+GZnIgYiYh5ETGvUqcAKplU2G1/Q2NBfzoini0W77c9UNQHJB1opkUAdSgdevPY2MwaSQcj4u5xy/9Z0v9GxCrbyyXNiIh/KFlX3w69NansK5HvvffeZL3Jobeyr7F+/PHHK60/denw0aNHK60bE+s09DaZ9+x/I+lGSW/ZfqNY9lNJqyT92vZSSR9Juq6GPgE0pDTsEfFfkjp98uL79bYDoCl8XBbIBGEHMkHYgUwQdiAThB3IBJe4AqeZri9xBXB6IOxAJgg7kAnCDmSCsAOZIOxAJgg7kAnCDmSCsAOZIOxAJgg7kAnCDmSCsAOZIOxAJgg7kAnCDmSCsAOZIOxAJgg7kAnCDmSCsAOZIOxAJgg7kInSsNsetL3V9ju237a9rFi+wvY+228UPwuabxdAt0onibA9IGkgInbaniZph6RrNDYf+6GI+JdJb4xJIoDGdZokYjLzs49KGi1uf2r7XUnn1dsegKad1Ht22+dL+p6kPxSL7rL9pu0nbE/v8Jxh29ttb6/WKoAqJj3Xm+2pkl6W9E8R8aztWZI+kRSSHtLYof5tJevgMB5oWKfD+EmF3fY3JG2S9LuI+NcJ6udL2hQRF5esh7ADDet6YkfblvRLSe+OD3px4u64RZJ2VW0SQHMmczb+CkmvSHpL0rFi8U8lLZF0icYO4/dI+nFxMi+1LvbsQMMqHcbXhbADzWN+diBzhB3IBGEHMkHYgUwQdiAThB3IBGEHMkHYgUwQdiAThB3IBGEHMkHYgUwQdiAThB3IROkXTtbsE0kfjbt/drGsH/Vrb/3al0Rv3aqzt7/sVOjp9ewnbNzeHhHzWmsgoV9769e+JHrrVq964zAeyARhBzLRdthHWt5+Sr/21q99SfTWrZ701up7dgC90/aeHUCPEHYgE62E3fZ823+0/YHt5W300IntPbbfKqahbnV+umIOvQO2d41bNsP2ZtvvF78nnGOvpd76YhrvxDTjrb52bU9/3vP37LbPlLRb0g8k7ZX0uqQlEfFOTxvpwPYeSfMiovUPYNj+W0mHJK09PrWW7UckHYyIVcV/lNMj4r4+6W2FTnIa74Z66zTN+C1q8bWrc/rzbrSxZ79M0gcR8WFEHJG0XtLCFvroexGxTdLBry1eKGlNcXuNxv6x9FyH3vpCRIxGxM7i9qeSjk8z3uprl+irJ9oI+3mS/jTu/l7113zvIen3tnfYHm67mQnMGjfN1seSZrXZzARKp/Hupa9NM943r103059XxQm6E10REZdK+ntJPykOV/tSjL0H66ex019I+o7G5gAclfTzNpspphl/RtLdEfHn8bU2X7sJ+urJ69ZG2PdJGhx3/1vFsr4QEfuK3wckbdTY245+sv/4DLrF7wMt9/P/ImJ/RByNiGOSVqvF166YZvwZSU9HxLPF4tZfu4n66tXr1kbYX5c0ZPvbtr8p6UeSnmuhjxPYnlKcOJHtKZJ+qP6bivo5STcXt2+W9JsWe/mKfpnGu9M042r5tWt9+vOI6PmPpAUaOyP/P5L+sY0eOvT1V5L+u/h5u+3eJK3T2GHdFxo7t7FU0kxJWyS9L+k/Jc3oo97+XWNTe7+psWANtNTbFRo7RH9T0hvFz4K2X7tEXz153fi4LJAJTtABmSDsQCYIO5AJwg5kgrADmSDsQCYIO5CJ/wMOV3dsB75sEwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "index = random.randrange(0, X_train.shape[1])\n",
    "plt.imshow(X_train[:, index].reshape(28, 28), cmap = 'gray')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "medical-wagner",
   "metadata": {},
   "source": [
    "# Model\n",
    "\n",
    "![nn](Images/nn.png)\n",
    "\n",
    "## Initialize parameters Randomly\n",
    "$ W_1 = np.random.randn(n_1, n_0) $\n",
    "\n",
    "$ b_1 = np.zeros((n_1, 1))$\n",
    "\n",
    "$ W_2 = np.random.randn(n_2, n_1) $\n",
    "\n",
    "$ b_2 = np.zeros((n_2, 1))$\n",
    "\n",
    "\n",
    "## *Repeat Below Steps for many times : *\n",
    "\n",
    "\n",
    "## Forward Propagation \n",
    "\n",
    "$ Z_1 = W_1 * X + B_1 $\n",
    "\n",
    "$ A_1 = f ( Z_1 ) $  \n",
    "\n",
    "$ Z_2 = W2 * A_1 + B_2 $\n",
    "\n",
    "$ A_2 = Softmax( Z_2 ) $\n",
    "\n",
    "## Softmax \n",
    "\n",
    "$ a_i = \\frac{e^{z_i}}{\\sum_{i=k}^ne^{z_k}}$\n",
    "\n",
    "\n",
    "## Cost Function \n",
    "\n",
    "$Loss = - \\sum_{i=k}^{n}[ y_k*log(a_k) ]$\n",
    "\n",
    "$Cost = - \\frac{1}{m}\\sum_{i=1}^{m}\\sum_{k=1}^{n}[ y_k*log(a_k) ]$\n",
    "\n",
    "\n",
    "\n",
    "## Backward Propagation\n",
    "$dZ_2 = ( A_2 - Y )$\n",
    "\n",
    "$ dW_2 = \\frac{1}{m}. dZ_2 . A_1^T$\n",
    "\n",
    "$ dB_2 = \\frac{1}{m}.sum(dZ_2, 1)$\n",
    "\n",
    "\n",
    "\n",
    "$dZ_1 = W_2^T . dZ_2 * f_1^|(Z_1) $\n",
    "\n",
    "$dW_1 = \\frac{1}{m}.dZ_1.X^T$\n",
    "\n",
    "$dB_1 = \\frac{1}{m}.sum(dZ_1, 1)$\n",
    "\n",
    "\n",
    "## Updating Parameters\n",
    "\n",
    "$ W_2 = W_2 -  \\alpha * \\frac{\\partial Cost }{\\partial W_2}$ \n",
    "\n",
    "$ B_2 = B_2 -  \\alpha * \\frac{\\partial Cost }{\\partial B_2}$ \n",
    "\n",
    "$ W_1 = W_1 -  \\alpha * \\frac{\\partial Cost }{\\partial W_1}$ \n",
    "\n",
    "$ B_1 = B_1 -  \\alpha * \\frac{\\partial Cost }{\\partial B_1}$ \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "centered-tunnel",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tanh(x):\n",
    "    return np.tanh(x)\n",
    "\n",
    "def relu(x):\n",
    "    return np.maximum(x, 0)\n",
    "\n",
    "def softmax(x):\n",
    "    expX = np.exp(x)\n",
    "    return expX/np.sum(expX, axis = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "functional-monthly",
   "metadata": {},
   "outputs": [],
   "source": [
    "def derivative_tanh(x):\n",
    "    return (1 - np.power(np.tanh(x), 2))\n",
    "\n",
    "def derivative_relu(x):\n",
    "    return np.array(x > 0, dtype = np.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aggregate-collar",
   "metadata": {},
   "source": [
    "# Initialize Parameters\n",
    "\n",
    "### *Note I multiplied 0.01 with weights W. But, this is still large, and that is the reason we saw increase in Cost value at the beginning, while training the model with ReLU function.*\n",
    "\n",
    "### *So, instead, you can multiply with 0.001, and it will solve the problem.*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "prepared-heritage",
   "metadata": {},
   "outputs": [],
   "source": [
    "def initialize_parameters(n_x, n_h, n_y):\n",
    "    w1 = np.random.randn(n_h, n_x)*0.01\n",
    "    b1 = np.zeros((n_h, 1))\n",
    "    \n",
    "    w2 = np.random.randn(n_y, n_h)*0.01\n",
    "    b2 = np.zeros((n_y, 1))\n",
    "    \n",
    "    parameters = {\n",
    "        \"w1\" : w1,\n",
    "        \"b1\" : b1,\n",
    "        \"w2\" : w2,\n",
    "        \"b2\" : b2\n",
    "    }\n",
    "    \n",
    "    return parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cubic-thailand",
   "metadata": {},
   "source": [
    "# Forward Propagation\n",
    "\n",
    "$ Z_1 = W_1 * X + B_1 $\n",
    "\n",
    "$ A_1 = f ( Z_1 ) $  \n",
    "\n",
    "$ Z_2 = W2 * A_1 + B_2 $\n",
    "\n",
    "$ A_2 = Softmax( Z_2 ) $"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "voluntary-sigma",
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward_propagation(x, parameters):\n",
    "    \n",
    "    w1 = parameters['w1']\n",
    "    b1 = parameters['b1']\n",
    "    w2 = parameters['w2']\n",
    "    b2 = parameters['b2']\n",
    "    \n",
    "    z1 = np.dot(w1, x) + b1\n",
    "    a1 = tanh(z1)\n",
    "    \n",
    "    z2 = np.dot(w2, a1) + b2\n",
    "    a2 = softmax(z2)\n",
    "    \n",
    "    forward_cache = {\n",
    "        \"z1\" : z1,\n",
    "        \"a1\" : a1,\n",
    "        \"z2\" : z2,\n",
    "        \"a2\" : a2\n",
    "    }\n",
    "    \n",
    "    return forward_cache"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "lined-charger",
   "metadata": {},
   "source": [
    "# Cost Function\n",
    "\n",
    "$Cost = - \\frac{1}{m}\\sum_{i=1}^{m}\\sum_{k=1}^{n}[ y_k*log(a_k) ]$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "legendary-playback",
   "metadata": {},
   "outputs": [],
   "source": [
    "def cost_function(a2, y):\n",
    "    m = y.shape[1]\n",
    "    \n",
    "    cost = -(1/m)*np.sum(y*np.log(a2))\n",
    "    \n",
    "    #cost = -(1/m)*np.sum(np.sum(y*np.log(a2, 0), 1))\n",
    "    \n",
    "    return cost"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "present-easter",
   "metadata": {},
   "source": [
    "# Backpropagation\n",
    "\n",
    "$dZ_2 = ( A_2 - Y )$\n",
    "\n",
    "$ dW_2 = \\frac{1}{m}. dZ_2 . A_1^T$\n",
    "\n",
    "$ dB_2 = \\frac{1}{m}.sum(dZ_2, 1)$\n",
    "\n",
    "\n",
    "\n",
    "$dZ_1 = W_2^T . dZ_2 * f_1^|(Z_1) $\n",
    "\n",
    "$dW_1 = \\frac{1}{m}.dZ_1.X^T$\n",
    "\n",
    "$dB_1 = \\frac{1}{m}.sum(dZ_1, 1)$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "weird-lyric",
   "metadata": {},
   "outputs": [],
   "source": [
    "def backward_prop(x, y, parameters, forward_cache):\n",
    "    \n",
    "    w1 = parameters['w1']\n",
    "    b1 = parameters['b1']\n",
    "    w2 = parameters['w2']\n",
    "    b2 = parameters['b2']\n",
    "    \n",
    "    a1 = forward_cache['a1']\n",
    "    a2 = forward_cache['a2']\n",
    "    \n",
    "    m = x.shape[1]\n",
    "    \n",
    "    dz2 = (a2 - y)\n",
    "    dw2 = (1/m)*np.dot(dz2, a1.T)\n",
    "    db2 = (1/m)*np.sum(dz2, axis = 1, keepdims = True)\n",
    "    \n",
    "    dz1 = (1/m)*np.dot(w2.T, dz2)*derivative_tanh(a1)\n",
    "    dw1 = (1/m)*np.dot(dz1, x.T)\n",
    "    db1 = (1/m)*np.sum(dz1, axis = 1, keepdims = True)\n",
    "    \n",
    "    gradients = {\n",
    "        \"dw1\" : dw1,\n",
    "        \"db1\" : db1,\n",
    "        \"dw2\" : dw2,\n",
    "        \"db2\" : db2\n",
    "    }\n",
    "    \n",
    "    return gradients"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "capital-hostel",
   "metadata": {},
   "source": [
    "# Update Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "written-sullivan",
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_parameters(parameters, gradients, learning_rate):\n",
    "    \n",
    "    w1 = parameters['w1']\n",
    "    b1 = parameters['b1']\n",
    "    w2 = parameters['w2']\n",
    "    b2 = parameters['b2']\n",
    "    \n",
    "    dw1 = gradients['dw1']\n",
    "    db1 = gradients['db1']\n",
    "    dw2 = gradients['dw2']\n",
    "    db2 = gradients['db2']\n",
    "    \n",
    "    w1 = w1 - learning_rate*dw1\n",
    "    b1 = b1 - learning_rate*db1\n",
    "    w2 = w2 - learning_rate*dw2\n",
    "    b2 = b2 - learning_rate*db2\n",
    "    \n",
    "    parameters = {\n",
    "        \"w1\" : w1,\n",
    "        \"b1\" : b1,\n",
    "        \"w2\" : w2,\n",
    "        \"b2\" : b2\n",
    "    }\n",
    "    \n",
    "    return parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "intelligent-julian",
   "metadata": {},
   "source": [
    "# Complete Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "dietary-capacity",
   "metadata": {},
   "outputs": [],
   "source": [
    "def model(x, y, n_h, learning_rate, iterations):\n",
    "    \n",
    "    n_x = x.shape[0]\n",
    "    n_y = y.shape[0]\n",
    "    \n",
    "    cost_list = []\n",
    "    \n",
    "    parameters = initialize_parameters(n_x, n_h, n_y)\n",
    "    \n",
    "    for i in range(iterations):\n",
    "        \n",
    "        forward_cache = forward_propagation(x, parameters)\n",
    "        \n",
    "        cost = cost_function(forward_cache['a2'], y)\n",
    "        \n",
    "        gradients = backward_prop(x, y, parameters, forward_cache)\n",
    "        \n",
    "        parameters = update_parameters(parameters, gradients, learning_rate)\n",
    "        \n",
    "        cost_list.append(cost)\n",
    "        \n",
    "        if(i%(iterations/10) == 0):\n",
    "            print(\"Cost after\", i, \"iterations is :\", cost)\n",
    "        \n",
    "    return parameters, cost_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "integral-credit",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cost after 0 iterations is : 5.554932437641482\n",
      "Cost after 10 iterations is : 2.99219213554902\n",
      "Cost after 20 iterations is : 2.0780609204989977\n",
      "Cost after 30 iterations is : 1.586854977299809\n",
      "Cost after 40 iterations is : 1.288589509627695\n",
      "Cost after 50 iterations is : 1.087678774358446\n",
      "Cost after 60 iterations is : 0.941718027730125\n",
      "Cost after 70 iterations is : 0.8298448671990124\n",
      "Cost after 80 iterations is : 0.7406886294167034\n",
      "Cost after 90 iterations is : 0.667502642788124\n"
     ]
    }
   ],
   "source": [
    "iterations = 100\n",
    "n_h = 1000\n",
    "learning_rate = 0.02\n",
    "Parameters, Cost_list = model(X_train, Y_train, n_h = n_h, learning_rate = learning_rate, iterations = iterations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "italian-graph",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAD4CAYAAADFAawfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAde0lEQVR4nO3deZhcZYHv8e9bVV1VvVTv1XsnnX0PSQiQkIQliAYEUcQZUBRkNFcvI47XuTrcee71Ye69zvDc0RkXrooE0VHRK6IgCoIQMIkkobMSstPZutP7vq/v/aMqoQMJ6ZCuPqeqfp/nqadro/t3ntP88vZ73nPKWGsRERH38jgdQERE3p2KWkTE5VTUIiIup6IWEXE5FbWIiMv5YvFN8/PzbUVFRSy+tYhIQtq2bVuTtTZ8ttdiUtQVFRVUVlbG4luLiCQkY8yxc72mqQ8REZdTUYuIuJyKWkTE5VTUIiIup6IWEXE5FbWIiMupqEVEXM41RT0yYvnuS4f488FGp6OIiLiKa4ra4zE8/Ocq/rSv3ukoIiKu4pqiBijJTuVkW6/TMUREXMVVRV2anUpNW5/TMUREXMVVRa0RtYjIO7mqqEtzUmnvHaSrf8jpKCIiruGqoi7JTgWgVqNqEZHTXFXUpdlBAKpV1CIip7mqqE+NqDVPLSLyFlcVdUEoiNdjVNQiIqO4qqi9HkNRZpCTWqInInKaq4oaIis/ajSiFhE5zX1FnZ1KTauKWkTkFNcVdUl2kLqOPoZHrNNRRERcwYVFncrwiKWhU/PUIiLg0qIGLdETETnFdUVdGi1qXZxJRCTCdUV9akStA4oiIhGuK+qMgI+s1BRNfYiIRLmuqEGXOxURGc03ljcZY44CncAwMGStXRrLUKXZQao19SEiAoyxqKOutdY2xSzJKCXZqWw90jIRP0pExPVcO/XR0TdEZ9+g01FERBw31qK2wPPGmG3GmLVne4MxZq0xptIYU9nY2HhRod5aS60leiIiYy3qldbaJcANwL3GmKve/gZr7cPW2qXW2qXhcPiiQpXqpBcRkdPGVNTW2pro1wbgN8DlsQz11kkvKmoRkfMWtTEm3RgTOnUfeD+wJ5ahwqEAPn2AgIgIMLZVH4XAb4wxp97/c2vtc7EM5fUYSnNSOdLUHcsfIyISF85b1NbaKuCSCchyhvmlWew83jbRP1ZExHVcuTwPYGFpFjVtvTR39TsdRUTEUe4t6rJsAHbXtDsbRETEYa4t6gVlWRgDu0+oqEUkubm2qDMCPqaFM9hd3eZ0FBERR7m2qCEyT727ph1r9fmJIpK83F3UZVk0dvZT16FTyUUkebm7qMuzAdileWoRSWKuLuq5xZn4PIbXa9qcjiIi4hhXF3UwxcvMwhC7qzWiFpHk5eqiBrikPIvd1TqgKCLJy/VFvbAsm/beQY639DgdRUTEEa4v6gWlWQDs0vSHiCQp1xf1rKIQAZ+H3SfanI4iIuII1xd1itfD/NIsKo+1Oh1FRMQRri9qgBXT8thd3UZ7rz7sVkSST1wU9coZYUYsvPpms9NRREQmXFwU9aLybNL8XjYdbnI6iojIhIuLovb7PFwxJVdFLSJJKS6KGiLTH1VN3fpkchFJOvFT1NPzAdh0SKNqEUkucVPUMwszCIcCbND0h4gkmbgpamMMK6fn85fDTYyM6LofIpI84qaoAVZMz6e5e4B9dR1ORxERmTBxVdSn56k1/SEiSSSuirooK8j0ggw26ICiiCSRuCpqgNWzC9hc1UxHn04nF5HkEHdF/YF5hQwOW9bvb3A6iojIhIi7ol5cnkM4FOCPb9Q5HUVEZELEXVF7PIbr5xby8oFG+gaHnY4jIhJzcVfUAGvmFdEzMMxGHVQUkSQQl0W9bGoeoaCP5zT9ISJJYMxFbYzxGmN2GGOeiWWgsfD7PFw3u4AX99UzNDzidBwRkZi6kBH1F4F9sQpyoT4wr4jWnkG2Hm1xOoqISEyNqaiNMWXAB4FHYhtn7K6eFSbg8/D8G/VORxERiamxjqj/HfgKcM55BmPMWmNMpTGmsrGxcTyyvas0v4+rZoZ5dk8tw7pIk4gksPMWtTHmJqDBWrvt3d5nrX3YWrvUWrs0HA6PW8B3c8uiEuo7+tlcpc9SFJHENZYR9QrgQ8aYo8AvgNXGmJ/GNNUYvW9OIaGAjye31zgdRUQkZs5b1Nba+621ZdbaCuB24CVr7Z0xTzYGwRQvNywo4rk9tfQO6OQXEUlMcbmOerSPLC6je2CY5/dqTbWIJKYLKmpr7cvW2ptiFea9uGJKLiVZQX6zQ9MfIpKY4n5E7fEYbllcyoZDTTR29jsdR0Rk3MV9UQPcuriU4RHL73addDqKiMi4S4iinlEYYl5JJk/uqHY6iojIuEuIoga47dIy9tR0sKem3ekoIiLjKmGK+tbFZQRTPPxsy3Gno4iIjKuEKeqstBRuXljCUztr6NTnKYpIAkmYogb4+BWT6BkY5qmdOqgoIokjoYp6UXk2c4sz+dmW41irCzWJSGJIqKI2xvCJZZPYV9vBjhNtTscRERkXCVXUALcsKiXd7+XnOqgoIgki4Yo6I+DjlsWl/G7XSVq6B5yOIyJy0RKuqAHuvrKC/qERfrb5mNNRREQuWkIW9czCEFfPDPPjV4/RN6jLn4pIfEvIogb47KqpNHX187Su/yEicS5hi3rF9DxmF4VYt+GIluqJSFxL2KI2xvCZVVM5UN/JhkNNTscREXnPEraoAW6+pJhwKMAPN1Q5HUVE5D1L6KIO+LzcfWUFGw416ap6IhK3ErqoAe5cNplQ0Md3XzrsdBQRkfck4Ys6KzWFT6+YwnNv1LG/rsPpOCIiFyzhixrgnhUVZAR8fEejahGJQ0lR1Nlpfj61fDJ/eL2Www2dTscREbkgSVHUAJ9ZNZXUFK/mqkUk7iRNUeem+/nkssk8veskhxu6nI4jIjJmSVPUAGuvmkqa38c3nj/gdBQRkTFLqqLOywjw2VVTeXZPHTuOtzodR0RkTJKqqAE+s2oK+Rl+Hnxuv64BIiJxIemKOj3g4wurZ7C5qoVXDjY6HUdE5LySrqgB7rh8EuW5qTz43AFGRjSqFhF3S8qi9vs8/P37Z7GvtoMnd9Q4HUdE5F0lZVED3LywhEXl2Tz43H66+oecjiMick7nLWpjTNAYs9UYs8sY84Yx5oGJCBZrHo/hazfPpbGzXyfBiIirjWVE3Q+sttZeAiwC1hhjlsU01QRZPCmHjy4p49GNRzja1O10HBGRszpvUduIU6fypURvCXME7qtrZpHiNfyv3+9zOoqIyFmNaY7aGOM1xuwEGoAXrLVbzvKetcaYSmNMZWNj/Cx7K8gM8rerZ/CnffWsP9DgdBwRkXcYU1Fba4ettYuAMuByY8z8s7znYWvtUmvt0nA4PM4xY+uelRVMC6fzP57aQ+/AsNNxRETOcEGrPqy1bcB6YE1M0jgk4PPy9Y8s4ERLL9968ZDTcUREzjCWVR9hY0x29H4qcD2wP8a5JtwVU/P4q6Vl/HBDFftq9UkwIuIeYxlRFwPrjTG7gdeIzFE/E9tYzvhvN84hOzWF+598XWcsiohrjGXVx25r7WJr7UJr7Xxr7T9NRDAnZKf5+e83zWXniTZ+/OpRp+OIiABJfGbiudyyqITVswt48Ln9VDXqAwZExHkq6rcxxvDPty7A7/Xw97/axbCmQETEYSrqsyjMDPLALfPYfryNRzZUOR1HRJKcivocPryolPfPLeQbLxzkYL0+uVxEnKOiPgdjDP/7IwsIBXzc9/gO+gZ1IoyIOENF/S7CoQD/+rFL2F/Xyb88m3BLx0UkTqioz+Pa2QXcs2IKj/3lKH/aW+90HBFJQirqMfjqDbOYW5zJf31iF3XtfU7HEZEko6Ieg4DPy3c+vpi+wRHue3wHg8MjTkcSkSSioh6jaeEM/uWjC9h6tEXz1SIyoVTUF+CWRaXctXwy6zYe4ZndJ52OIyJJQkV9gf7xg3NZMimbrzyxm8MNWl8tIrGnor5Afp+Hhz6xhDS/l8/+ZBttPQNORxKRBKeifg+Ks1L5/p2XUtPay70/366DiyISUyrq92hpRS5fv3UBmw4388Dv3nA6jogkMJ/TAeLZbZeWcai+kx/8uYrp4QzuXjHF6UgikoBU1BfpK2tm82ZjNw88s5eirFTWzC9yOpKIJBhNfVwkr8fwnTsWc0lZNvf9YgevHW1xOpKIJBgV9ThI9Xt59O7LKM1O5TM/rtSyPREZVyrqcZKb7ufHn76cFK+HT67byomWHqcjiUiCUFGPo0l5afzknsvp7h/iznVbqO/QBZxE5OKpqMfZ3JJMHrvncpo6+7nzkS20dOuEGBG5OCrqGFgyKYdH7rqM4y093PnIFlpV1iJyEVTUMbJ8Wh4Pf2ophxu7uOOHm2nu6nc6kojEKRV1DF09M8y6u5ZypKmbj/9wC00qaxF5D1TUMbZqRpgf3X0Zx1q6uf3hzdS29zodSUTijIp6Alw5PZ/HPn05de193Pa9V6lq7HI6kojEERX1BFk2NY9frF1G3+AwH/v+q+ypaXc6kojECRX1BJpfmsWvPrecYIqX2x/ezCsHG52OJCJxQEU9waaGM/j156+kPDeNex57jce3Hnc6koi4nIraAUVZQX71ueWsnJ7P/U++zoPP7WdkxDodS0Rc6rxFbYwpN8asN8bsNca8YYz54kQES3QZAR+P3LWUOy6fxPdefpP/9NNtdPUPOR1LRFxoLCPqIeDL1tq5wDLgXmPM3NjGSg4pXg9f/8h8vnbzXF7a38Ct/3cTx5q7nY4lIi5z3qK21tZaa7dH73cC+4DSWAdLFsYYPr1iCj+553LqO/r50Hc3sf5Ag9OxRMRFLmiO2hhTASwGtpzltbXGmEpjTGVjo1YzXKgV0/N5+m9XUJwV5J7HXuObzx9gWPPWIsIFFLUxJgP4NfB31tqOt79urX3YWrvUWrs0HA6PZ8akMTkvnd/eu4LblpTx7ZcOc9ejW2no1KVSRZLdmIraGJNCpKR/Zq19MraRklswxcv/+dglPPjRBbx2tIUbv7WBlzUVIpLUxrLqwwDrgH3W2m/GPpIA/PVlk/jdF1aSnxHg7h+9xv98Zi99g8NOxxIRB4xlRL0C+CSw2hizM3q7Mca5BJhZGOK3967gruWTWbfxCB/67kadei6ShIy143/AaunSpbaysnLcv28yW3+gga8+sZuW7gHuu24Gn79mGilena8kkiiMMdustUvP9pr+T48T184q4PkvXcWNC4r55gsH+fBDmzS6FkkSKuo4kp3m59t3LOb7dy6hobOfWx7axD8/u09z1yIJTkUdh9bML+ZPX7qa25aU8YNXqrj+317hpf31TscSkRhRUceprLQUHrxtIb9Yu4yAz8s9j1Wy9ieVVLf2OB1NRMaZijrOLZuaxx/uW8VX18xmw6EmrvvGK3zzhYP0DOgCTyKJQkWdAPw+D5+/ZhovfvlqPjCviG+/eIjrvvEKv9lRrcuniiQAFXUCKclO5dt3LOZXn1tOfkaAL/1yFzd/dyMbDzU5HU1ELoKKOgFdVpHLU/eu4Fu3L6KtZ5A7123hk+u2sOtEm9PRROQ90AkvCa5vcJifbj7GQ+sP09ozyAfmFfJfrp/FrKKQ09FEZJR3O+FFRZ0kOvsGeXTjUR7ZUEVn/xA3LijiC6tnMKc40+loIoKKWkZp6xlg3cYjPLbpKJ39Q7x/biH/+drpLCrPdjqaSFJTUcs7tPcMsm7TER7bdISOviGunJbH56+Zxsrp+UQumCgiE0lFLefU1T/E41uO88jGKuo7+pldFOIzq6byoUtK8Pt0rFlkoqio5bz6h4Z5eudJHtlwhAP1nYRDAe68YjIfv2IS4VDA6XgiCU9FLWNmreXPh5p4dOMRXjnYSIrX8MEFxXxy+WSWTMrRtIhIjLxbUfsmOoy4mzGGq2eGuXpmmDcbu/iPV4/xxLZqfrvzJHOKM/nEFZO4ZVEJoWCK01FFkoZG1HJe3f1DPLXzJD/dfIy9tR2kpni5aWExt19erlG2yDjR1IeMC2stu6rb+eVrJ3h6Zw3dA8NMzU/no5eWceuSUoqzUp2OKBK3VNQy7rr7h/j967U8sa2arUdaMAaWT83jw4tLuWF+kaZGRC6Qilpi6lhzN7/eXsNTO2s41txDwOfhujkF3LywhGtnFxBM8TodUcT1VNQyIay17DjRxlM7avj967U0dQ2QEfBx3ZwCblxQzNUzwyptkXNQUcuEGxoeYcuRFn636yR/fKOO1p5B0v1erpldwJp5RVw7u4CMgBYdiZyiohZHDQ6PsLmqmT+8XssLe+tp6hrA7/Vw5fQ83jenkOvnFlKYGXQ6poijVNTiGsMjlu3HW/njnjpe2FfPsebIZzzOL81k9exCVs8uYGFpFh6PlvxJclFRiytZaznc0MXze+t5aX8DO463MmIhL93PVTPDXDMrzKoZYXLT/U5HFYk5FbXEhdbuAV452Mj6Aw38+WAjrT2DGAPzS7JYOSOfVdPzWTI5RwckJSGpqCXuDI9YXq9pZ8PBRjYcamL78VaGRizBFA+XVeRy5bR8rpyWx7ySTHxeXeVP4p+KWuJeZ98gW4+0sPFwE5sON3GwvguAUMDHZVNyuWJKLldMzWO+ilvilC7KJHEvFEzhujmFXDenEIDGzn42VzXzalUzm99s5qX9DQCk+70smZzDZRW5XFaRy6LybFL9miqR+KYRtSSEho4+th5tYUtVC68dbeFAfSfWgs9jmFeSyaWTc1kyOZslk3IoydY1ScR9NPUhSae9Z5Btx1uoPNpK5bFWdp1oo39oBICizCCLJ2WzqDybxZNymF+aSZpff1yKsy5q6sMY8yhwE9BgrZ0/3uFEYiErLSW6LjsyVTIwNMK+2g62H29l+/E2dp5o5dk9dQB4DMwsDHFJWTYLy7NYWJrNrKKQPopMXOO8I2pjzFVAF/CTsRa1RtQSD5q6+tl5vI3d1W3srG5nd3UbbT2DAPi9HmYVhZhfmsX80kzml2QxqyikpYESMxc99WGMqQCeUVFLIrPWUt3ay+7qdnbXtLGnpp3Xq9vp6BsCwOsxTA9nMLckkznFIeYWZzGnOERehj5TUi7ehKz6MMasBdYCTJo0aby+rciEMcZQnptGeW4aH1xYDETK+0RLL3tr23njZAdvnOzg1Teb+c2OmtP/XUEowOziTOYUhZgVvU0vyCDg0+hbxodG1CLvQUv3AHtPdrC/roO9tR3sq+3kzYYuBoYjByy9HkNFXhqzikLMKAgxszDEzMIMKvLTSdE6bzkLraMWGWe56X5Wzshn5Yz8088NDo9wtKmb/XWdHKrv5EB9J/tqO3l2Tx2nxkM+j6EiP53p4QymF7x1mxpO18oTOSf9ZoiMkxSvhxmFIWYUhs54vm9wmMMNXRxq6ORgfReHG7o4WN/J83vrGBn1B21JVpCp4UhpT81PZ2o4gyn56ZRkp+LV1QST2liW5z0OXAPkG2Oqga9Za9fFOphIogimeKOrR7LOeL5/aJhjzT282RAp76qmbqoau3hyew1d/UOn3+f3eajIS6MiL50p+elU5KczOfq4KDOoS8ImgfMWtbX2jokIIpJsAj5vdO76zBG4tZbGrn6qGrupauzmaHM3R5q6qWrq5uUDjafnwSFS4pNz05icl8ak3EiBT8pLY1JuGmU5qTqgmSA09SHiMsYYCkJBCkJBlk3NO+O14RHLybZejjZ3c6y5h2PN3Rxt7uF4cw+bDjfTOzg86vtEzsIsz0mjLDeV8pzIipbynFTKctMoygxqSiVOqKhF4ojX89YSwlUzznzt1Ej8REsPx1t6ONbcw4mWXk609PCXw83Ud9YwepGXz2Mozg5Smp1KaXZkBF6ak0pZdiol2akUZwc1IncJFbVIghg9Er90cu47Xu8fGuZkWx8nWnqoaeulurWH6tZeqlt72XS4ifrOPt6+WjccClCSnUpJVjBS3m/7mp8R0Kh8AqioRZJEwOdlSn7kgOTZDAyNUNveS01rLzVtkVttWx8n23s5UN/Jywcaz5hagciovDAzSFFW5FYcvX/6ucwgBZkBjcwvkopaRIDogcm8dCbnnb3IrbW09Qxysr2XuvY+Trb3UdvWS11HH3Xtfew92cFL+xreUeYQWXdemBmkKDNAYWaQgswgBaHI/cLMAAWhIPkZfn3owzmoqEVkTIwx5KT7yUn3M68k66zvsdbS0TtEbUekzOs7+qhr76e+s4/69j7qOvrYc7KDpq7+d0yzGAN56QEKQgEKMgOEM976Gg4FCYcCp2/pfi/GJM+Ui4paRMaNMYastBSy0lKYXZR5zvcNDY/Q1DVAQ2cf9R391Hf00dDZT2NnHw0d/TR09rO/tpOmrn6GRt55mYvUFC/5IT/5GYHTt3CGn/xQ5H5eevR+eoDMVF/cl7qKWkQmnM/rOT2v/W5GRiytPQM0dvXT2Bm5NZ1xf4DjzT1sP9ZKS8/AO0bpACleQ156gLwMP7npfvLS/eRlBMhN95Of4Sc3PXD6+dwMP6GA+4pdRS0iruXxGPIyAuRlBJhd9O7vHRoeoaVngKbOAZq7+2nuGqCpq5/m7gGauyKPm7sHONrcTVPnwFnn0iFS7DlpkVIffTv1XE66n9w0P9lpKaefj/XncqqoRSQh+Lye08sTx6J3YJjm7n5auiMF3tI1cPp+66mvPZGrJDZ3D9DeO3jO7xVM8ZCT5qc8J43/97nl47VJp6moRSQppfq9lPnTKMtJG9P7h4ZHaO8dpKV7gNaeyNe2ngFaegZo6xmktXsAnzc2UyYqahGRMfB5PaenYSaaFi2KiLicilpExOVU1CIiLqeiFhFxORW1iIjLqahFRFxORS0i4nIqahERlzP2bFcxudhvakwjcOw9/uf5QNM4xokHybjNkJzbnYzbDMm53Re6zZOtteGzvRCTor4YxphKa+1Sp3NMpGTcZkjO7U7GbYbk3O7x3GZNfYiIuJyKWkTE5dxY1A87HcABybjNkJzbnYzbDMm53eO2za6boxYRkTO5cUQtIiKjqKhFRFzONUVtjFljjDlgjDlsjPkHp/PEijGm3Biz3hiz1xjzhjHmi9Hnc40xLxhjDkW/5jiddbwZY7zGmB3GmGeij6cYY7ZE9/kvjTF+pzOON2NMtjHmCWPMfmPMPmPM8kTf18aYL0V/t/cYYx43xgQTcV8bYx41xjQYY/aMeu6s+9ZEfDu6/buNMUsu5Ge5oqiNMV7gIeAGYC5whzFmrrOpYmYI+LK1di6wDLg3uq3/ALxorZ0BvBh9nGi+COwb9fhB4N+stdOBVuBvHEkVW98CnrPWzgYuIbL9CbuvjTGlwH3AUmvtfMAL3E5i7uvHgDVve+5c+/YGYEb0thb43gX9JGut4zdgOfDHUY/vB+53OtcEbftTwPXAAaA4+lwxcMDpbOO8nWXRX9zVwDOAIXLWlu9svwOJcAOygCNED9qPej5h9zVQCpwAcol81N8zwAcSdV8DFcCe8+1b4AfAHWd731hurhhR89bOPaU6+lxCM8ZUAIuBLUChtbY2+lIdUOhUrhj5d+ArwEj0cR7QZq0dij5OxH0+BWgEfhSd8nnEGJNOAu9ra20N8K/AcaAWaAe2kfj7+pRz7duL6ji3FHXSMcZkAL8G/s5a2zH6NRv5Jzdh1k0aY24CGqy125zOMsF8wBLge9baxUA3b5vmSMB9nQPcQuQfqRIgnXdODySF8dy3binqGqB81OOy6HMJyRiTQqSkf2atfTL6dL0xpjj6ejHQ4FS+GFgBfMgYcxT4BZHpj28B2cYYX/Q9ibjPq4Fqa+2W6OMniBR3Iu/r9wFHrLWN1tpB4Eki+z/R9/Up59q3F9Vxbinq14AZ0SPDfiIHH552OFNMGGMMsA7YZ6395qiXngbuit6/i8jcdUKw1t5vrS2z1lYQ2bcvWWs/AawHbou+LaG2GcBaWwecMMbMij51HbCXBN7XRKY8lhlj0qK/66e2OaH39Sjn2rdPA5+Krv5YBrSPmiI5P6cn40dNrt8IHATeBP7R6Twx3M6VRP4c2g3sjN5uJDJn+yJwCPgTkOt01hht/zXAM9H7U4GtwGHgV0DA6Xwx2N5FQGV0f/8WyEn0fQ08AOwH9gD/AQQScV8DjxOZhx8k8tfT35xr3xI5eP5QtN9eJ7IqZsw/S6eQi4i4nFumPkRE5BxU1CIiLqeiFhFxORW1iIjLqahFRFxORS0i4nIqahERl/v/YqmlBVDgP30AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "t = np.arange(0, iterations)\n",
    "plt.plot(t, Cost_list)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "french-design",
   "metadata": {},
   "outputs": [],
   "source": [
    "def accuracy(inp, labels, parameters):\n",
    "    forward_cache = forward_propagation(inp, parameters)\n",
    "    a_out = forward_cache['a2']   # containes propabilities with shape(10, 1)\n",
    "    \n",
    "    a_out = np.argmax(a_out, 0)  # 0 represents row wise \n",
    "    \n",
    "    labels = np.argmax(labels, 0)\n",
    "    \n",
    "    acc = np.mean(a_out == labels)*100\n",
    "    \n",
    "    return acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "mexican-jamaica",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of Train Dataset 100.0 %\n",
      "Accuracy of Test Dataset 85.14 %\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy of Train Dataset\", accuracy(X_train, Y_train, Parameters), \"%\")\n",
    "print(\"Accuracy of Test Dataset\", round(accuracy(X_test, Y_test, Parameters), 2), \"%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "finished-superintendent",
   "metadata": {},
   "outputs": [],
   "source": [
    "idx = int(random.randrange(0,X_test.shape[1]))\n",
    "plt.imshow(X_test[:, idx].reshape((28,28)),cmap='gray')\n",
    "plt.show()\n",
    "\n",
    "cache = forward_prop(X_test[:, idx].reshape(X_test[:, idx].shape[0], 1), Parameters)\n",
    "a_pred = cache['a2']  \n",
    "a_pred = np.argmax(a_pred, 0)\n",
    "\n",
    "print(\"Our model says it is :\", a_pred[0])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}