Call volume prediction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "fd373498",
   "metadata": {},
   "source": [
    "<h2> Import Packages </h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "id": "f1281eac",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import sklearn\n",
    "from sklearn.model_selection import train_test_split\n",
    "from datetime import datetime as dt\n",
    "import os\n",
    "from termcolor import colored\n",
    "import lightgbm\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "import warnings\n",
    "import dill\n",
    "import datetime\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6ad277d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = r'C:\\Users\\gaurav.gund\\OneDrive - Grant Thornton Advisory Private Limited\\Documents\\d_drive\\C Drive Backup 2021\\Desktop\\hackathon\\niologic'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff97db7f",
   "metadata": {},
   "source": [
    "<h2> Read Data </h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "59df62ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.chdir(path)\n",
    "data = pd.read_csv('Seattle_Real_Time_Fire_911_Calls.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5c11715c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1659898, 7)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9dcbd697",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Address</th>\n",
       "      <th>Type</th>\n",
       "      <th>Datetime</th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "      <th>Report Location</th>\n",
       "      <th>Incident Number</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>904 7th Av</td>\n",
       "      <td>Car Fire</td>\n",
       "      <td>05/22/2019 12:55:00 PM</td>\n",
       "      <td>47.606900</td>\n",
       "      <td>-122.329051</td>\n",
       "      <td>POINT (-122.329051 47.6069)</td>\n",
       "      <td>F190051945</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9610 53rd Av S</td>\n",
       "      <td>Aid Response</td>\n",
       "      <td>05/22/2019 12:55:00 PM</td>\n",
       "      <td>47.515984</td>\n",
       "      <td>-122.266529</td>\n",
       "      <td>POINT (-122.266529 47.515984)</td>\n",
       "      <td>F190051946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>106 Pike St</td>\n",
       "      <td>Aid Response</td>\n",
       "      <td>05/22/2019 12:57:00 PM</td>\n",
       "      <td>47.608849</td>\n",
       "      <td>-122.339945</td>\n",
       "      <td>POINT (-122.339945 47.608849)</td>\n",
       "      <td>F190051950</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Address          Type                Datetime   Latitude  \\\n",
       "0      904 7th Av      Car Fire  05/22/2019 12:55:00 PM  47.606900   \n",
       "1  9610 53rd Av S  Aid Response  05/22/2019 12:55:00 PM  47.515984   \n",
       "2     106 Pike St  Aid Response  05/22/2019 12:57:00 PM  47.608849   \n",
       "\n",
       "    Longitude                Report Location Incident Number  \n",
       "0 -122.329051    POINT (-122.329051 47.6069)      F190051945  \n",
       "1 -122.266529  POINT (-122.266529 47.515984)      F190051946  \n",
       "2 -122.339945  POINT (-122.339945 47.608849)      F190051950  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "50948934",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1659898 entries, 0 to 1659897\n",
      "Data columns (total 7 columns):\n",
      " #   Column           Non-Null Count    Dtype  \n",
      "---  ------           --------------    -----  \n",
      " 0   Address          1659822 non-null  object \n",
      " 1   Type             1659898 non-null  object \n",
      " 2   Datetime         1659898 non-null  object \n",
      " 3   Latitude         1658959 non-null  float64\n",
      " 4   Longitude        1658959 non-null  float64\n",
      " 5   Report Location  1658995 non-null  object \n",
      " 6   Incident Number  1659898 non-null  object \n",
      "dtypes: float64(2), object(5)\n",
      "memory usage: 88.6+ MB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "557a16ff",
   "metadata": {},
   "source": [
    "<h2> Exploratory Analysis </h2>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ec04022",
   "metadata": {},
   "source": [
    "<h3> Unique Incident Number </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "id": "12cfc07e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34mIncident Number has length :1659898Hence, they are unique.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print(colored('Incident Number has length :' + str(len(data['Incident Number'].unique()))+'Hence, they are unique.', 'blue'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a2e2a8f3",
   "metadata": {},
   "source": [
    "<h3> Observation by Year </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 323,
   "id": "cd9d5ea0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZsAAAEoCAYAAACOxlwjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAArzUlEQVR4nO3de7xc0/3/8de7cUm1+lUSoYJEUKXaIr3xQyj9UqXSr5S0VXEpdYmipRolqoS2ighxv4S2lEjRqn7dEurSfgXVutQ1QYKKaxIVJPn8/ljrNGN6zpl9zpx9JmfO+/l47MfMrDVrr7X37JnP7L3XXlsRgZmZWZne1+gGmJlZ83OwMTOz0jnYmJlZ6RxszMysdA42ZmZWOgcbMzMrnYON9WiSpkkKSaMa3ZZ6SRqWl2VmA9sQeRrUqDZYc3KwsaWWpJ0l3S5prqS3JP1d0uGSevx220aQnAWMBy5uTKu6lqSPSXo7L+dXK9L3yWnzJK3VyDZa9+nxX1prTpIOBK4HtgLuBKYAQ4DTgMsb2K5ly5p3RDwZEYdFxAll1dGdIuJRYFx+eYakD0haGfhpTjsmIp7t6nolLdPV87QuEBGePC1VE7AiMBcI4MSK9B1yWgBb57Rp+fWPgTuAN4GpwKCcvxxwAfAi8DbwHPC7inmuBVwJzAZeB24CPl6RPzPP/xjgYWBRriuACRXvG5vTzsqvf53n+TYwD7gN2LiqzZXT8cCw/HxmxXw/AfwReBmYA/wO+Ggr7TsaeCAv/x+AD+f8DwNX5/ILgBnAee2s+5b27A88kdfJRcD78/R6XgcDK9bvG8DilrSq+S2X11sAP8ufRQD3kP7srgKcl5djHnAXsGVF+e/ndryZ1+WDwG4V+Zfm+Z0H3Ay8Awxr9DbsqZVtq9EN8OSpegL+u+JHb+2qvJYf15Py65Yf7rdJezx/z6/vzfn75dcPAROBa4FXct4KwJP5h/JG4LL8Y/US0K+qvoWkoHQ1aQ8rgOeB9+X3tfygfia/vgv4FXA2cHvOezTnHUI6ZBak4HYGKZAOoyLYAKsDr+W03+f3BvACS4JJS/v+BUwCXsmvf5LzT6z4cT+HFIj+3s66b1nvL5IO572UX/8i55+TXx+VX38xv57Wzjw/TwpQ7+Z1/Q7wcVKwuTOXv4MU1OYCb5EDal5/U/LjVflzWMCSPxOXVrR5Wm7zpo3ehj395+TDaLY06lfx/MWqvBfyY/+q9IkRsSewDekHaaikjYCWw15/J/347w2smtN2IgWO54HHgFeBZ/O8d6ua/7iI2CMiRkTEU8DdpGCwVa5nQ+AfEfF/+f1fA/5M+rf+t5y2gaSPRMRZpCAH8OtIh87+2Mp62BNYifRD/uWI+CLwV2A1YETVe8dGxF7AWfn1JvmxZfn/AlyS2/WpVuqqdkBE7AN8O7/+Vn68MD9+Mz/u0rIcbc0oIloC3TKAgJ9GxEPAZsAWpHV0f358EuhL+pwAjiL9QXiVtKc4B1ge2LyqmjsiYlhE7BMR9xdYPutmPrZpS6OXK54PIAWAFqu18h6ARwEi4mVJL+f3DSTtrQwDvgLsQfoHfIuk4cCgXHYN4LtV81u36vVdVa8nkX7wdif9+yfXhaT1SD+eH2xl2fqTglsRLe17tCLtH6RgsXbVex/Ij6/nx5a6zwA+CRxEWsZFwG8k7RkRi9upu6XOf+THfpKWj4j7JP0V+JSkTwA7k/ZUrq6xLKcDB1e0CZYs34q0sv4lLUcK2B9vZX7VfzburlG/NZj3bGxpdA8wPz/fryVR0vYs+YH636oyH8vv6ceSPaNZwMKI2B34UH7PLcD2wFdJh6AA7iMdDlNEiHSe46Sq+b9d9foq0uGc/yEFnMXAL3PeTqQf+7+S9kwGVJRTflyUH9v7Dra0b4OKtI/mx2eq3rswP1YP4/5qROxA+kH/JOlw39dJexTt+VhV3S9HRMs6uCg/nko65/XHiHitxvwWtfJ8Zn58Aehbsf5XIB1q3JAUaBaS9kDfBzySy4j3qv58bCnjPRtb6kTEXEk/BCYAx0rajHTuYnh+y28i4vaqYgflQPMp0nZ9P+mHaS9JPwCmkwLYxvn9r5NO2s8gHc65S9LfSD+ew4Avkc4BtNXG1yX9jnQ4qz9wW0Q8l7P/mR/XJ3Vl/lQrs2h573fzHsIlrbznl8AYYBtJ15NOtm+S5z+5rbZVOVrSLqTDiO+wJFi/UaPcebnczvl1ZQ/AXwI/JwVtaOcQWg33kf5YfB64V9LdpD3SrYHDSX8MFpM+z1+QAuZ6nazLGsx7NrZUyuc1hpNOIG9FOocyAzgS+EYrRU4h/ZAOIZ2QHxERQToX8zIpeOxL+sE9Efh9RLwJbAtcQQoye5H2HH6Zy9UyqeL5ZRXPryL9+18EbAec3ErZX5DO5WxIOoT0Hz+iEfE86RzUTaQ9kaHADcA2EfFqgfZBCroLgV1J513+CRwaEX9rrxBwHGm9L09azh9VtOt14Jr8cj6pi3qH5cN4XwHOJe15jiIF0xuAP0fELGB0bvO2pODkw2U9lNL30cysOEm7k3rnXR4R36r1fjMfRjOzwiR9iHQebc+cdE4Dm2M9iPdszKywPGbaDNI5r59GxCkNbZD1GA42ZmZWOncQMDOz0vmcTRv69esXgwYNanQzzMx6lPvuu+/liKi+6NbBpi2DBg1i+vTpjW6GmVmPIqn6gmPAh9HMzKwbONiYmVnpHGzMzKx0DjZmZlY6BxszMyudg42ZmZXOwcbMzErnYGNmZqVzsDEzs9J5BIESDDr6hg69f+YpO5XUEjOzpYP3bMzMrHQONmZmVjoHGzMzK52DjZmZlc7BxszMSudgY2ZmpXOwMTOz0vk6m6VMR6/RAV+nY2ZLPwebJuMLSs1saVRasJF0JrA7sCpwQ0R8Oad/DLgI2AyYCYyOiJty3hbAOcBHgYeB/SLi/py3K3AqMBD4M7B3RMzIeQcAxwKrADcB+0TEKzlvLHAI8H5gMvCdiFhQ1nL3ZPUEKu+RmVl7yj5nc2UraVcAGwBHAO8CV0v6L0l9gWuAFYHDgQHAZEl9JK2W5zUXOJIUqCYBSNoEOBd4FBgL7AScnvOGA8cDtwJnAnsBY8pYUDMza1tpwSYiDiX/6LfIgeGTwBURcTZwGvAhYDdgR1KAmRgRE0l7P4OBYcBIYHng5IiYAPwW2FLSEGBUnv2YiPgZcDcwMgevlrzRETEGeA7Yu4zlNTOztnV3b7TB+XF2fpyVH9fp4rxlgDVz3rsRMacibw1Jy7XWOEn7S5ouafqcOXNae4uZmXVCo7s+aynKIyLOj4ihETG0f//+7b3VzMw6oLuDzYz8ODA/rpEfn+7ivIWkQ2YzgGUlrVqRNzsi3qlvMczMrCPK7I22E/Dx/HJNSfsBtwN/A/aQ9DBwIDCP1DFgAfAScKCkecC+pN5q04BHgFOAH0gaAAwH7oyIpyRdBhwKnCTpZmBz0jmhBZImAbsA4yXNIB1aO7GsZbbOc5dts+ZW5p7NkaQAAfAJ4AJgC+DrwGOkzgHLAV+LiNdzd+QRwHxgPCnwjIiIRRHxAqmTwEqk7s8PkE/+R8R9wMHAhsAJwI2k3mxExBTgJ8D2pIB0OTCuxGU2M7NWlLZnExHD2sn+fBtl7gA2biNvCjCljbyJwMQ28o4DjmuvrWZmVq5GdxAwM7NewMHGzMxK52BjZmalc7AxM7PSOdiYmVnpHGzMzKx0vp+N9Xi+INRs6edgY72eg5VZ+XwYzczMSudgY2ZmpXOwMTOz0jnYmJlZ6RxszMysdA42ZmZWOgcbMzMrnYONmZmVzhd1mtXBF4SaFeM9GzMzK52DjZmZla5DwUbSepL+X1mNMTOz5lQz2Ei6XdJlkjYDHgVulzS+/KaZmVmzKLJnszFwJ7AL8BDwW2BkmY0yM7PmUiTY9AWWBYYC1wLXASuW2CYzM2syRYLN34EzgR2A24G1gFllNsrMzJpLkets9gQOBh6PiKmS1gdOKrdZZs2vo9fogK/TsZ6rZrCJiMclfQ8YIGkt4Mbym2VmZs2kZrCRNBo4hXTupkUUKWtmZgbFAsaPgQXAHcDCcptjZkV5qBzrSYoEm2eB8yLinLIbY2ZmzalIsHkEOFbSR4DXclpExOnlNcvMyuS9IutuRYLNHvnxmIq0AOoKNpIOAw4DVgeeB06LiAmStgDOAT4KPAzsFxH35zK7AqcCA4E/A3tHxIycdwBwLLAKcBOwT0S8kvPGAocA7wcmA9+JiAX1tN+st3IvOuuMItfZ7N3KtE89lUpajxSsFgNHkC4aPVPSmsA1pItGDwcGAJMl9ZG0GnAlMBc4EtgMmJTntwlwLmk4nbHATnn+SBoOHA/cSrpeaC9gTD3tNzOzjqkZbCJiEnA98FKers9pXVHvbOAW4EXgbeBzpAAzMSImAhcBg4FhpCFylgdOjogJpGFztpQ0BBiV5zcmIn4G3A2MlNS3Im90RIwBniMFTDMz6yZFuj5vQRqi5sM56VVJu0TEPZ2tNCIek3Q0cDLwD9Iezt7Amvkts/Njy0gF65CCTkfylsnzGwy8GxFzKvI+J2m5iHinaln3B/YHWGuttTq7eGbWjnrOF3Vn2eryVp8i52xOA94lBQZIh9BOJ+2FdIqk/sBo4K+krtVjgbNI52Pe89b2ZtPVeRFxPnA+wNChQ6OdeZhZL+NOFfUpEmw2Ag6PiAsAJD0L/KLOeocBawDnRsR1kjYGfkI65wKpAwD5PQBPAx9sJ29GRd7zOW8h6ZDZDGBjSatGxEs5b3b1Xo2ZWVm8V1Us2DwPfEvSU/n1njmtHi3B4ZuSXgC+kV8/TjovdKCkecC+wExgGqkL9inADyQNAIYDd0bEU5IuAw4FTpJ0M7A5cEVELJA0iXR7hPGSZpAOrZ1YZ/vNzKwDivRG+zmwBXBznrYAflZPpRExHfge6YT/2fnxkIh4EBgBzAfGkwLPiIhYFBEvkDoJrEQ63PYA+eR/RNxHGix0Q+AE0vhth+e8KaS9pu1JAelyYFw97Tczs44pMhDnBZKeBL6Uk/4QEVPrrTgiTiOdD6pOv4N0w7bWykwBprSRNxGY2EbeccBxnW6smZnVpc1gI2lT4ClgCPAGcEVlXsuFlmZmZrW0t2dzL+mw1ZWkEQOq9SmlRWZm1nTaCzaXkU7O13sBp5mZ9XJtBpuI2BsgX4X/SES8XPF65e5pnpmZNYMivdGmAttUvP4K6foVMzOzQtrrILAV6eJLASMkfSxnbUUaUcDMzKyQ9s7ZbEMaRiaA3fLU4pYyG2VmZs2lvWBzFel+MlcBZwB3kQLPa8CdpbfMzMyaRnsdBB4FHpU0GHgpIt7qvmaZmVkzKTI22grAtXmwzL45LSJilfKaZWZmzaRIb7TzSLcTWI00ZtlKLLmXjJmZWU1Fgs0mpIE3g3SDsxNJowuYmZkVUiTYwJJbCuxAumfMHuU0x8zMmlGRczZPkG44dg/ptgCB92zMzKwDigSbLwKLgYuA75KCzYQyG2VmZs2lSLC5kHTDsd9FxNElt8fMzJpQkXM2O5Iu7PynpAskbV1ym8zMrMkUCTarAvuQRg34JnCbpJllNsrMzJpLkdtCvwFMkvQ48CIp8KxZdsPMzKx51Aw2ks4AvkrqkRakWw78qtxmmZlZMynSQeBQ4K/AeODXEfFCqS0yM7Om026wkdQHOAa4JSJ8bY2ZmXVKux0EImIRcDCwUfc0x8zMmlGRw2jXA0dImgP8+xBaRNxfWqvMzKypFAk23yF1DLi+Kr1P1zfHzMyaUZFgcxkp2JiZmXVKketsRnVDO8zMrInVHEFA0iqSrpb0mqTt8vODu6NxZmbWHIoMV3MO6T42HyKN/jyTdB7HzMyskCLBZnvg1IrXjwCDy2mOmZk1oyLB5k1gQH7eB9gOeKXeiiWtJOkySa9Lmi/pjpy+haS/SXpb0v2SNq0os6ukJyUtkDRN0uCKvAMkzZL0lqTrJK1SkTdW0pxcz6WS+tbbfjMzK65IsLmSJYfNfk+6JfQVXVD3xcA3SDdlOwx4MgeBa4AVgcNJQW6ypD6SVsttmQscCWwGTAKQtAlwLvAoMBbYCTg95w0HjgduBc4E9gLGdEH7zcysoCJdn38IzCP9gEMKOCfXU6mkdYDhpAE9fwgsiogLc2AYABwVERNzgDkWGAZ8AlgeODkirpb0aWBPSUOAUXnWYyLiXklfBkZK2r8ib3REzJH0TWBv4Lh6lsHMzIqruWcTEe9GxI8j4jPk+9lExLt11rthfvw06TDdm5J+ypJzQbPz46z8uE4n8pYh3QphMPBuRMypyFtD0nLVjZK0v6TpkqbPmTOnOtvMzDqpSNfn2/O5lc1InQNulzS+znqXz48fAHYH7gKO4j/3tNRe07o6LyLOj4ihETG0f//+7czCzMw6osg5m41Jd+ncBXgI+C0wss56Z+THP0XEFNJtp2FJIBiYH9fIj09XlCmatxB4LuctK2nVirzZEfFOnctgZmYFFQk2fYFlgaHAtcB1pBP49XgA+DvwBUnfJp1DWQTcALwEHCjpQGBf0nU900idA94BfiBpNOmcz50R8RRpSB2AkyQdBWwOXBkRC8idCIDxksaRDq1dWmf7zcysA4oEm7+TenHtANwOrMWS8yWdEhFB2jt6CpgArAx8KyIeAkYA80k3a3sJGBERi/JN20YCK5Gu+3mAfPI/Iu4j3QphQ+AE4EZSbzbyntNPSNcLHQpcDoyrp/1mZtYxRXqj7QkcBDwREVMlrQ+cVG/FEfEw8PlW0u8gHbprrcwUYEobeROBiW3kHYd7n5mZNUyRgTgfl3QS8NncpXhKRc8uMzOzmmoGG0lfBy4gnbsBWCBp34i4stSWmZlZ0yhyzuZU4DXSobOT8/PTymyUmZk1lyLnbN4GToqICwEkPQN8v9RWmZlZU2kz2Ej6an56A3CopIWkPaHRwM3d0DYzM2sS7e3ZTGbJ7aBFGjCz5flGwCEltsvMzJpIe8HmBJYEGzMzs05rM9hExPEtzyX1AdbPLx+PiEUlt8vMzJpIkYE4NyTdJ+ahPD0iaYOyG2ZmZs2jSNfns4DVSTdMuyI/P6vMRpmZWXMp0vV5KPDDiDgLQNIheGwxMzPrgCJ7Nq8C20laJ99hc3vglXKbZWZmzaTIns2FpJ5pO1ekHVtOc8zMrBkVGYjzREnPAzvmpBsi4tJSW2VmZk2lyJ4NEXExcHHJbTEzsyZV5JyNmZlZXRxszMysdG0GG0n7SBokaStJ/bqzUWZm1lza27O5ANgcmAps0z3NMTOzZtReB4G5pBumCRgr6cCKvIiIL5TaMjMzaxrtBZsLSDdJC2DDPLXwaNBmZlZYm4fRIuIoYEXgWeAbQP+KadVuaZ2ZmTWFdq+ziYg3gcGS1gO+lpNviYgnSm+ZmZk1jZoXdUraGbgaWJZ0/uYdSSMi4ndlN87MzJpDketsxgEzgAOA/YGnSB0HzMzMCikyXM1g4LCIuBBAEsAZJbbJzMyaTJFg8zRwhKTF+fXhpL0bMzOzQooEm2OBq0hdoQW8C+xWZqPMzKy5FLnFwHWSNibdNA3g5oh4vNxmmZlZMyk0EGdEPB4RZ+epywKNpL6SHpMUklpuO/0xSXdLejvnfbHi/VtI+lvOu1/SphV5u0p6UtICSdMkDa7IO0DSLElvSbpO0ipdtQxmZlZbo0d9Pg4YWJV2BbABcATpkN3Vkv5LUl/gGtKFpocDA4DJkvpIWg24kjTEzpHAZsAkAEmbAOcCjwJjgZ2A00teLjMzq9CwYCPpE6SgMbYibRPgk8AVEXE2cBrwIdI5oh1JAWZiREwELiL1lBsGjASWB06OiAnAb4EtJQ0BRuXZj4mInwF3AyNz8DIzs25QM9hI+paktSteryJp83oqlfQ+4ELgbGB6RVbLoa/Z+XFWflyni/OWAdZspV37S5ouafqcOXM6skhmZtaOIns2lwCfqXi9PfCnOuvdGxgEXAaskdP+izRKQSW1M48uz4uI8yNiaEQM7d+/fzuzMDOzjmizN5qkXYBdST/OB0naMWdtAiyos941SQN6PliR9k3gI/l5y3mclkD0NPBqO3kfbCdvRkXe8zlvIfBcXUtgZmaFtdf1eRPS+Y4Ats5Ti1/VWe9VwEP5+UbA8cAfgR8BFwN7SHoYOBCYR+oYsAB4CThQ0jxgX2AmMA14BDgF+IGkAcBw4M6IeErSZcChwEmSbibdEO6KiKg3YJqZWUHtHUY7n3T4TMAxwKeBocCQiNiznkoj4pGImBwRk4Hbc/JTEXEf8HXgMVLngOWAr0XE6zk4jADmA+NJgWdERCyKiBdInQRWAk4FHiB3DMjzPJh0P54TgBtJHRPMzKybtLlnk3/AXwDeJ2kZUk+wPgCS1oqIZ7uiARExjYrzKBHxMPD5Nt57B7BxG3lTgClt5E0EJtbbVjMz65witxgYTTpEVdlVOIqUNTMzg2IB48ek8yV3kE6sm5mZdUiRYPMscF5EnFN2Y8zMrDkVCTaPAMdK+gjwWk6LiPCQL2ZmVkiRYLNHfjymIi3w+GJmZlZQkWCzDym4mJmZdUqR+9lc2g3tMDOzJlak6/PTrSRHRAwpoT1mZtaEihxGW5Ulh9HeTxp14F+ltcjMzJpOkcNoLYNcImk50gWejb7pmpmZ9SBFDqNtWvFyGdJ4ZV8HDiupTWZm1mSKHEabznt7own4v3KaY2ZmzahIsLmMJcFmEWlY/wvKapCZmTWfIudsRknqA6yfkx6PiEXlNsvMzJpJzRP9kj4GPEq62dlDwCM5zczMrJAivcrOBlYHrsjT6sCEMhtlZmbNpcg5m6HADyPiLABJhwDjSm2VmZk1lSJ7Nq8C20laR9I6wPbAK+U2y8zMmkmRPZsLgROAnSvSji2nOWZmVm3Q0Td06P0zT9mppJZ0XpHeaCdKeh7YMSfd4ME5zcysI4qMILA2cHtEXJxfD5G0dkQ8U3rrzMysKRQ5Z3MLMKri9aicZmZmVkiRYLMGadSAFs/kNDMzs0KKdBB4Gvi+pNmkcdG+l9PMzMwKKRJsfgpMAlq6QwjYs7QWmZlZ0ynSG+1ySc8AX85Jv4+IO8ptlpmZNZMiezbk4OIAY2ZmneI7bpqZWekcbMzMrHQONmZmVrqGBBtJ60maKukVSfMk3SxpSM7bVdKTkhZImiZpcEW5AyTNkvSWpOskrVKRN1bSHEnzJV0qqW9OX0bS2ZLekPSapJ9LcpA1M+tGjfrRXSPXPRa4BNgOuFDSasCVwFzgSGAzUrdrJG0CnEu6kdtYYCfg9Jw3HDgeuBU4E9gLGJPrGg0cRLq99WTg+7x3RAQzMytZo4LN3RGxdUScFRGHkm5jsBEwElgeODkiJgC/BbbMez2jctkxEfEz4G5gZN6DackbHRFjgOeAvXPaKGAecBgp8LxTkWdmZt2gIcEmIt5peS5pKLAyqWt1yyGz2flxVn5cp428ZYA1c967ETGnIm8NScvlvBcjYlFELCDdi2ed1tolaX9J0yVNnzNnTmtvMTOzTmjouQtJGwDXk8ZeG93aW9or3tV5EXF+RAyNiKH9+/dvZxZmZtYRDQs2kjYEppEOa20bES8AM3L2wPzYMuDn023kLSQdMpsBLCtp1Yq82XkPagawuqQ++ZDbKnhsNzOzbtWo3mhrAlOBfqST/p+VtAepc8A7wA8kjQaGA3dGxFOkE/wAJ0k6CtgcuDIfGpuU88ZLGkc6tHZpTpsEfBA4g9R5YNmKPDMz6waFhqspwRCgZS/k5JbEiJCkkcDPgVOBv5BP5kfEfZIOBo4BtgRuBA7PeVMk/YTU66wvcDkwLs92ArAe8C0gSD3YLilz4czM7L0aEmwiYhptnDuJiCnAlDbyJgIT28g7DjiulfR3gQPzZGZmDeCLG83MrHQONmZmVjoHGzMzK52DjZmZlc7BxszMSudgY2ZmpXOwMTOz0jnYmJlZ6RxszMysdA42ZmZWukaNjWZmZt1g0NE3dOj9M0/ZqZR2eM/GzMxK52BjZmalc7AxM7PSOdiYmVnpHGzMzKx0DjZmZlY6BxszMyudg42ZmZXOwcbMzErnYGNmZqVzsDEzs9I52JiZWekcbMzMrHQONmZmVjoHGzMzK52DjZmZlc7BxszMSudgY2ZmpXOwMTOz0vWKYCNpC0l/k/S2pPslbdroNpmZ9SZNH2wk9QWuAVYEDgcGAJMl9Wlow8zMepGmDzbAjqQAMzEiJgIXAYOBYY1slJlZb6KIaHQbSiXpCOAXwDci4teS9gfOA/aPiAuq3rs/sH9++VHgsS5uTj/g5QaU7a11u929p263u/vrbsvaEdG/OnGZEipa2qmtjIg4Hzi/tIql6RExtLvL9ta63e7eU7fb3f11d1RvOIw2Iz8OzI9r5MenG9AWM7NeqTfs2dwIvAQcKGkesC8wE5jWwDaZmfUqTb9nExELgBHAfGA8KfCMiIhFDWhOPYfo6j281xvrdrt7T91ud/fX3SFN30HAzMwar+n3bMzMrPEcbMzMrHQONmZmVrre0ButV5P0YWBBRLzV6Lb0BJKWAYiIhY1uS2dJWi4i3ml0O8omaTCwTn75dETMaO/9SxtJywObAY9GxGuNbk/ZvGfTzSTtK6ndnnCSFkm6N7/3/Z2o44OSxkt6nXSF8HxJMyQd1Il5rStpe0n9Cr6/jyTl5++XtJmklTtaby7fV9J6LQGgg2VXkjRG0voF3rucpOMlPQMsABZIeianLV+g/EaS/ijpBknrS7pQ0lxJ90gaUqPsKpIukvRnST+VtEJO/7KkmteCSRot6eD8fGtJzwL/knR3/jFur+wnJN0s6SpJq0u6RtK8vO1tXKPsbZJ+Lmm9Wm1sZx7DJV0r6UFJf5F0saSPFyg3SNI04Engpjw9KWlarWVuZ56rSdqqwPsOlbRufr5uXn+z87r7j6vmq8puWjkB/w38Cdin6ODAkvaUdIqkHSvSPiPp4hrlPiDpaElHSlohz+d6SScrjR9ZvojwVMIEfLWNaSKwqEbZxXlaBLwGnAls2IG6r8zlnwfeAeYC/8jzO7ZG2cuB0/Lz0cDCXO4NYNsaZb8GvAm8AGwLPJvLvgV8tUC7pwCb5+c75ToX5fltVqPsEVXTSbnsJcDhNcpektfXM8BdwN35+WLgkgLtnlbxmbUs88z8+toaZSdXlF0M/AX4MLB7re0kl38BOCw/n5k/67uBfwF/qFH2zop6n87byhO5/VMLbKOL8nRT3rb7dGAbPb5qud+teNy1RtmpeTkn5c95HHBZTmu33e3Mc9+C63sR8LX8/J7c5pfz4y87sM5apn+nFaj7p1Vlzs7pNbcV4NcV5W6rms9ZnVlnHV7H3VFJb5za2LD+/QEXKHsg8H3g8Yp53Q6MLFD3fOCH+fmnc9l1SUFoVo2yrwPfzs/nAg/mL/NjwP01yj6ayzxHCjozgAvyD+JDBddZyxd5JinYTM7zmlpwfS9uZaq1vucBJ7aSPg6YW6Ddc4FTgB1yfWNy+unAnBplXwN+CXwQ2C0v83TgOwV/gBbkbaVv1fo7BHijwHZyIvCFXHZUTj+q1nLn958F/J4lf0hmAz8GBhZo90ukALE+cHBuy6fysj9co+xbwMGtpB8CvFWj7JltTLcWXN+LSX+qls/L/JOcPhF4qUDZeXm5L2HJH43bKPan5nnSn5Ev53W/mDSw8MgC2/jLwLXA2Fzul8Ag4AbguVp1d8VUegW9dcpfiP/NH27ldF2BDePfPxr59bbAVcDbBb8Qz+UN+XPAD/OXYlNgrwJfxvnAsRVfpp1z+j7Avwos88GkQUwXA1/J6fvVKlux3COAFfLzfXP60dT+4Xw5f5F/lJfzqDyPCcBeNcq+BPwKWLkirR/p32C7PyAV62zPXGYxsF1OH1Vgfb9GDu759Rak4DW/4Gd9Lymobws8nH9EticFgXZ/RIBX8nIfntv9M+Aj+QesVpCsDGxrAicAs3L6OwXa/QpLfqiH5W3ts6Q9jAU1yj5BCkpfzdv450iB+l7giQLt7tSfkoryY0kBehHwpYrPel6Nstvmtt8LfBIYkudXc68/l38T2Kfi9Xdz+cdrtZ30p+SbpBHwFwM75PS9aq3vrppKr6C3TqRDFJNaSd8PWFyj7HuCTUX6AOAHBeoeU/WlepjUGWQc8EiNstflH7oTgEeAv5IOVfwDeLBG2dn5i9RyGO8PwNdJx6Vr/nvKZX5DOsSyENglp38beLVG2Zbg8Abpn/56Rb/ILPm3tyh/KRewZE/0+ALlH8jrSPlHZIWcfgbwWI2yd1F1qA3YKv+wFPnx2yp/Xq3tQR9Qo+w1Fcv9p6ryV3V0GwX6AMOBGwu0e1KeR8u6foV0z6nvU3vve1fSH5vq5V1A7UNwrwIXAltXTT8vuL6rg9X3cvrp1Njzz+/rSwrqbwKX5nkVDTYPkQ+dVaR9jwKBknR+6yjSn8jvAmvk9ONqre+umkqvoLdOpH9767aS/gHSENztlR1LB87RtDGP/yENz/MjYJWctjowoEa5gaR/jdX/+l4EtqhR9uSKL+ME0r/2li/myQXaXF3nuJz+K+DWgsv9JdIhuIc7+EUeTgq0D+bpWmB4wbL9gH6tpI8CdqpR9nPAt4BlqtI3ocYeWcV7B5D+HFwNXJ8/900LlFuJJec8VgK+QTqscmbLNtNO2anAsDq2zxXzNvIgcEvLtkXaY/jvgst8eF7W8cBhwGoFyk0GLm8l/WvAjALlq4PUBjn9WOB/OrD8mwB3kPZKay5vLrMvcG4r28p3qHEYLq/XLVtJPw34UWc/x45MHq6mB8k9lfpFxLNll5e0DbAh6Z/QDNK/1QUFym0HvC8ibpL0UWBnUrfUKZ1pc57njsAzEfFIwfd/gHTobSAwISLu72zd9cjtWKUzn1dHP2tJy5H+4EDaiyzc9blRZbuifNW86vp+dLCuhq2zejSy7tKjWW+dSD/Sx5IOKR3IknHotgFu6+Q8C/WYKaM86TzKWt1dNpf/QB111yxbxmfVBeu7aO+oD5MOx7zNkkNKb+e0lZfGsl1RvoT1XWgbbfA66/R2Wsb67ujkizrLMxHYOz8fAewoaTdgVdLud5uU7i7ams8Wqbje8m0YSRoltk83lwXYo47yRcp2+rOC+tZ3F3xWl5C6id9NOkEv0h7dN0iHxnZdCsvWVb7B23cj11k922m9ddfNh9FKImkO6dzHkaQusSeRulf+Brg4ItrcqCUtBoLW7yoa7ZWtt3yNL/JuZZVtZN31fFa5fD3ru97P+l/AcRFxalX6UaTODSssbWW7oO6GbN9d0O5611k9vyl11d0VvGdTnuWByRHxEPCQpBnAFaThKWp5k3TTtxuq0rcinXQus/yptPNFLrFsI+uu57OC+tZ3vZ/1K8B2km4jXYcB6R/rdjlvaSxbb/lGbd/Q2HVWz3Zab911c7ApzxOkXduLACLiGknfJPWsquX/UpGYVJkoaVmW7EaXVb6RP5yNqruezwrqW9/1ftYnAueQrq15zyyAWsMTNapsveUbtX1DY9dZPdtpvXXXzYfRSiJpZ9I1F6dExaCOknYChkbEj9sp25c09Mebnay70+Ul3Qq8HBG7V6XvB5wfEW2Op1dP2UbWXc9nld9Xz/qu67PO89iE9EM5KCfNAC6LAr3wGlW2nvKN2r7rbXcXlK13O63r86qXg03JVMfItPWU7Wz5Rv5wLgU/2t2+vnt63T2t3V2xnXS27q4o2+i669IdXd564wSsTRqgsfrK7mnA4BplB3W2bFeUz/MYTLoQ7AtFy3RF2UbUXc9nVe/6buRn3RvbXe821uB11rDflK6YvGdTEklTSSfufst7uxruCtwXEduUUbYL6h5E6nu/ZVXWn4C9o51/QvWUbWTdDV7fPbLuHtzuQdS3jfa6ddZluiOi9caJ+kam7XTZLqh7Kp0cvr2eso2su8Hru0fW3YPbXe822uvWWVdN7o1WnlnA3pJe4L1dDffKeWWVrbf854DvR8TZlYmSDiENVlhW2UbW3cj13VPr7qntrncb7Y3rrGt0R0TrjRPtj0zb7gCP9ZTtgrrrGb6902UbWXeD13ePrLsHt7vebbTXrbOumnzOpkSSBpCGwRick2YAV0bEi2WWrae8pF1JF4otV5lMuovjyIj4bRlll4K6G7K+e3LdPbHd9W4njWr30lB33bojovXGKX+gt5KG5r+KPLQ/xW7h2umyXVR+AGnI9vF0YPj2ess2qu5Gru+eWndPbXcXbGO9cp11xVR6Bb11It2XvfL+5E8CaxXcMDpdtgvq7nVfpgav7x5Zdw9ud73baK9bZ1011bxa1jrtc8CZEdEP+AxpCPOppL7yZZatt/x5wDakoYx2A+6StFbBeusp28i6G7m+e2rdPbXd9W6jvXGddY3uiGi9cSL9gzio4vUGwAukY8O1/oV0umwX1D0XOCM/H0rqufIU6ZaypZVtZN0NXt89su4e3O56t9Fet866aiq9gt46kf41TK1K2xD4Z4ENo9Nlu6DuXvdlavD67pF19+B217uN9rp11lVT6RX01glYlzRCa/X9wgcCW5dVtgvq7nVfpgav7x5Zdw9ud73baK9bZ101ueuzvYekdYE1gLvivSPLDgSGRMTtZZRtdN3WO3g7aRwHGzMzK517o5mZWekcbMzMrHQONmYNJmlZSY9IWiRp05y2rqS3JT0n6QONbqNZvRxszBosIt4lDfX+PuCMnHwaafyuI6L+u0p6dHdrOAcbs6VARNwG/AbYUtI5wM7AzcAsSfdImi/pcUkjAST1l/RATp8v6U+SNsp5oySFpN9Iepg0LItZQ7k3mtlSQtIawD+AD5IuMtwK+APwEukGX9uQbmO8GfAccBDpCvjVSVfA/yUitpc0CrgEeAM4Dng2Iq7tzmUxq+bda7OlRETMlvRz4MekMbz6ASvnaVzFW7cFrgR2AD5PGiIfYOOqWV4cEWeW2mizghxszJYuz1Y9Qtqrubzi9UzgUGBz4HTgRuBiYMWqeT2P2VLC52zMll73AK+S9mA2AD4OHE26Ar7Fh0mH2wZ2e+vMOsDBxmwpFRGvAl8m3XvkFOAY4F+kPZsJpFsZ704KPg81ppVmxbiDgJmZlc57NmZmVjoHGzMzK52DjZmZlc7BxszMSudgY2ZmpXOwMTOz0jnYmJlZ6f4/2Bu1fqtqmJwAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "abc1 = data['Datetime'].apply(lambda x:x.split('/')[2].split(' ')[0] ).value_counts().reset_index()\n",
    "abc1.columns=['Year', 'count of observations']\n",
    "plt.bar(abc1.Year, abc1['count of observations'])\n",
    "plt.title('Observations by Year')\n",
    "plt.xlabel('Year')\n",
    "plt.ylabel( 'count of observations')\n",
    "plt.xticks(rotation= 'vertical')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "af6fc8a0",
   "metadata": {},
   "source": [
    "<h3> Type of calls </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 320,
   "id": "545a6683",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34mType has length :245\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "print(colored('Type has length :' + str(len(data['Type'].unique())), 'blue'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c8347621",
   "metadata": {},
   "source": [
    "<h3> Selecting data for last 5 year </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 314,
   "id": "7f6c0122",
   "metadata": {},
   "outputs": [],
   "source": [
    "data['Year'] = data['Datetime'].apply(lambda x:x.split('/')[2].split(' ')[0] )\n",
    "data1 = data[data.Year.isin(['2016', '2017','2018', '2019', '2020', '2021'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 364,
   "id": "fe1f55c1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34mShape of data is :(609306, 8)\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "print(colored('Shape of data is :' + str(data1.shape), 'blue'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 316,
   "id": "e3d6d138",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Address</th>\n",
       "      <th>Type</th>\n",
       "      <th>Datetime</th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "      <th>Report Location</th>\n",
       "      <th>Incident Number</th>\n",
       "      <th>Year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>904 7th Av</td>\n",
       "      <td>Car Fire</td>\n",
       "      <td>05/22/2019 12:55:00 PM</td>\n",
       "      <td>47.606900</td>\n",
       "      <td>-122.329051</td>\n",
       "      <td>POINT (-122.329051 47.6069)</td>\n",
       "      <td>F190051945</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9610 53rd Av S</td>\n",
       "      <td>Aid Response</td>\n",
       "      <td>05/22/2019 12:55:00 PM</td>\n",
       "      <td>47.515984</td>\n",
       "      <td>-122.266529</td>\n",
       "      <td>POINT (-122.266529 47.515984)</td>\n",
       "      <td>F190051946</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Address          Type                Datetime   Latitude  \\\n",
       "0      904 7th Av      Car Fire  05/22/2019 12:55:00 PM  47.606900   \n",
       "1  9610 53rd Av S  Aid Response  05/22/2019 12:55:00 PM  47.515984   \n",
       "\n",
       "    Longitude                Report Location Incident Number  Year  \n",
       "0 -122.329051    POINT (-122.329051 47.6069)      F190051945  2019  \n",
       "1 -122.266529  POINT (-122.266529 47.515984)      F190051946  2019  "
      ]
     },
     "execution_count": 316,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data1.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba985163",
   "metadata": {},
   "source": [
    "<h2> Data Preparation </h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "2abb6f13",
   "metadata": {},
   "outputs": [],
   "source": [
    "data2 = data1.drop(['Address', 'Report Location', 'Incident Number'], axis= 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "4838dd43",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Type</th>\n",
       "      <th>Datetime</th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "      <th>Year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Car Fire</td>\n",
       "      <td>05/22/2019 12:55:00 PM</td>\n",
       "      <td>47.606900</td>\n",
       "      <td>-122.329051</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Aid Response</td>\n",
       "      <td>05/22/2019 12:55:00 PM</td>\n",
       "      <td>47.515984</td>\n",
       "      <td>-122.266529</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Type                Datetime   Latitude   Longitude  Year\n",
       "0      Car Fire  05/22/2019 12:55:00 PM  47.606900 -122.329051  2019\n",
       "1  Aid Response  05/22/2019 12:55:00 PM  47.515984 -122.266529  2019"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data2.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ec38c96",
   "metadata": {},
   "source": [
    "<h3> Missing Data Check  </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 365,
   "id": "fe5b5c13",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Type           0\n",
      "Datetime       0\n",
      "Latitude     852\n",
      "Longitude    852\n",
      "Year           0\n",
      "dtype: int64\n",
      " \n",
      "\u001b[34mThere are 852 cases where Latitude and Longitude are missing\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "print(data2.isnull().sum(axis= 0))\n",
    "print(' ')\n",
    "print(colored('There are 852 cases where Latitude and Longitude are missing','blue') )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 366,
   "id": "466c7230",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Type</th>\n",
       "      <th>Datetime</th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "      <th>Year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>767</th>\n",
       "      <td>Automatic Medical Alarm</td>\n",
       "      <td>04/25/2019 09:49:00 AM</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>768</th>\n",
       "      <td>Aid Response</td>\n",
       "      <td>04/25/2019 09:51:00 AM</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        Type                Datetime  Latitude  Longitude  \\\n",
       "767  Automatic Medical Alarm  04/25/2019 09:49:00 AM       NaN        NaN   \n",
       "768             Aid Response  04/25/2019 09:51:00 AM       NaN        NaN   \n",
       "\n",
       "     Year  \n",
       "767  2019  \n",
       "768  2019  "
      ]
     },
     "execution_count": 366,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data2[data2.Latitude.isnull()].head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7dd4451e",
   "metadata": {},
   "source": [
    "<h1> Experiment 1: Daily predictions </h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "52ba0b7d",
   "metadata": {},
   "source": [
    "<h3> <font color=\"blue\"> This involves predicting calls on everyday basis by Locations  </font></h3>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fc841c59",
   "metadata": {},
   "source": [
    "<h2> Experiment 1: Data Split </h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "id": "d63f4383",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34mShape of training data for Experiment 1 is (502269, 9)\u001b[0m\n",
      "\u001b[34mShape of test data for Experiment 1 is (107037, 9)\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "test= data2[data2.Year.isin(['2021'])]\n",
    "train = data2[~data2.Year.isin(['2021'])]\n",
    "\n",
    "print(colored('Shape of training data for Experiment 1 is ' + str(train1.shape), 'blue'))\n",
    "print(colored('Shape of test data for Experiment 1 is ' + str(test1.shape), 'blue'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d66ad62f",
   "metadata": {},
   "source": [
    "<h2> Experiment 1: Feature Engineering </h2>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea5a4465",
   "metadata": {},
   "source": [
    "<h3> Extract date features </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "id": "187f7b50",
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert year into it\n",
    "train1 = train.copy(deep=True)\n",
    "train1['Year'] = train1.Year.astype('int')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "id": "94751ac0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# extract Month\n",
    "train1['Month'] = train1.Datetime.apply(lambda x: int(x.split('/')[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "id": "f66378b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# extract day\n",
    "train1['Day'] = train1.Datetime.apply(lambda x: int(x.split('/')[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "id": "43a74944",
   "metadata": {},
   "outputs": [],
   "source": [
    "# extract hour\n",
    "train1['hour'] = train1.Datetime.apply(lambda x: int(x.split('/')[2].split(' ')[1].split(':')[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15015352",
   "metadata": {},
   "source": [
    "<h5> <font color='blue'> Observation : Tried usig pd.to_datetime but it is very slow </font></h5>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "81c55857",
   "metadata": {},
   "source": [
    "<h3> Extract Season </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "id": "2222492d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# extract season \n",
    "def season(month):\n",
    "    '''Fuction to extract season based on month value '''\n",
    "    \n",
    "    if (month == 12 or month ==1 or month == 2):\n",
    "        return(\"winter\")\n",
    "        \n",
    "    elif(month ==3 or month == 4 or month ==  5):\n",
    "        return \"spring\"\n",
    "        \n",
    "    elif(month ==6 or month==7 or month == 8 ):\n",
    "        return \"summer\"\n",
    "        \n",
    "    elif(month ==9 or month ==10 or month==11):\n",
    "        return \"Autumn\" "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "id": "f439d88e",
   "metadata": {},
   "outputs": [],
   "source": [
    "train1['season'] = train1.Month.apply(lambda x: season(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "273d1deb",
   "metadata": {},
   "source": [
    "<h3> Aggregate Data for prediction: daily predictions </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 369,
   "id": "1c6d8134",
   "metadata": {},
   "outputs": [],
   "source": [
    "train1_agg= train1.groupby(['Latitude', 'Longitude', 'Year', 'Month', 'Day', 'season']).size().reset_index()\n",
    "train1_agg.columns = ['Latitude', 'Longitude', 'Year', 'Month', 'Day','season', 'calls']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 370,
   "id": "eb04b420",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>season</th>\n",
       "      <th>calls</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>47.251232</td>\n",
       "      <td>-122.122934</td>\n",
       "      <td>2020</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>summer</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>47.261880</td>\n",
       "      <td>-122.225729</td>\n",
       "      <td>2019</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>winter</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Latitude   Longitude  Year  Month  Day  season  calls\n",
       "0  47.251232 -122.122934  2020      6    3  summer      1\n",
       "1  47.261880 -122.225729  2019     12   12  winter      1"
      ]
     },
     "execution_count": 370,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train1_agg.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 371,
   "id": "ed7e967f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# one hot encode seasons\n",
    "train1_agg = pd.get_dummies(train1_agg, columns= ['season'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 372,
   "id": "17641c79",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>calls</th>\n",
       "      <th>season_Autumn</th>\n",
       "      <th>season_spring</th>\n",
       "      <th>season_summer</th>\n",
       "      <th>season_winter</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>47.251232</td>\n",
       "      <td>-122.122934</td>\n",
       "      <td>2020</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>47.261880</td>\n",
       "      <td>-122.225729</td>\n",
       "      <td>2019</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Latitude   Longitude  Year  Month  Day  calls  season_Autumn  \\\n",
       "0  47.251232 -122.122934  2020      6    3      1              0   \n",
       "1  47.261880 -122.225729  2019     12   12      1              0   \n",
       "\n",
       "   season_spring  season_summer  season_winter  \n",
       "0              0              1              0  \n",
       "1              0              0              1  "
      ]
     },
     "execution_count": 372,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train1_agg.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "id": "061ff8fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "train1_agg1 = train1_agg.drop(['Year'], axis = 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1431ead9",
   "metadata": {},
   "source": [
    "<h3> Prepare test data </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 327,
   "id": "766d5a13",
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_process_first(test1, loc_filter = True):\n",
    "    '''Function to prepare test data '''\n",
    "    test1['Year'] = test1['Datetime'].apply(lambda x:x.split('/')[2].split(' ')[0] )\n",
    "    test1['Year'] = test1.Year.astype('int')\n",
    "    test1['Month'] =test1.Datetime.apply(lambda x: int(x.split('/')[0]))\n",
    "    test1['Day'] = test1.Datetime.apply(lambda x: int(x.split('/')[1]))\n",
    "    test1['hour'] = test1.Datetime.apply(lambda x: int(x.split('/')[2].split(' ')[1].split(':')[0]))\n",
    "    test1['season'] = test1.Month.apply(lambda x: season(x))\n",
    "    if loc_filter == True:\n",
    "        test1_agg= test1.groupby(['Latitude', 'Longitude', 'Year', 'Month', 'Day', 'season']).size().reset_index()\n",
    "        test1_agg.columns = ['Latitude', 'Longitude', 'Year', 'Month', 'Day','season', 'calls']\n",
    "    else:\n",
    "        test1_agg= test1.groupby([ 'Year', 'Month', 'Day', 'season']).size().reset_index()\n",
    "        test1_agg.columns = [ 'Year', 'Month', 'Day','season', 'calls']\n",
    "        \n",
    "    test1_agg = pd.get_dummies(test1_agg, columns= ['season'])\n",
    "    test1_agg1 = test1_agg.drop(['Year'], axis = 1)\n",
    "    return(test1_agg1)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 349,
   "id": "06bcf8a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def results_prep_first(test1, loc_filter = True):\n",
    "    '''Function to prepare results for dashboard'''\n",
    "    \n",
    "    test1['Year'] = test1['Datetime'].apply(lambda x:x.split('/')[2].split(' ')[0] )\n",
    "    test1['Year'] = test1.Year.astype('int')\n",
    "    test1['Month'] =test1.Datetime.apply(lambda x: int(x.split('/')[0]))\n",
    "    test1['Day'] = test1.Datetime.apply(lambda x: int(x.split('/')[1]))\n",
    "    test1['hour'] = test1.Datetime.apply(lambda x: int(x.split('/')[2].split(' ')[1].split(':')[0]))\n",
    "    test1['season'] = test1.Month.apply(lambda x: season(x))\n",
    "    if loc_filter == True:\n",
    "        test1_agg= test1.groupby(['Latitude', 'Longitude', 'Year', 'Month', 'Day', 'season']).size().reset_index()\n",
    "        test1_agg.columns = ['Latitude', 'Longitude', 'Year', 'Month', 'Day','season', 'calls']\n",
    "    else:\n",
    "        test1_agg= test1.groupby([ 'Year', 'Month', 'Day', 'season']).size().reset_index()\n",
    "        test1_agg.columns = [ 'Year', 'Month', 'Day','season', 'calls']\n",
    "        \n",
    "    \n",
    "    return(test1_agg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 328,
   "id": "f31dbcd7",
   "metadata": {},
   "outputs": [],
   "source": [
    "test1_agg1 = data_process_first(test1, loc_filter = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 329,
   "id": "a8cdc5ae",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>calls</th>\n",
       "      <th>season_Autumn</th>\n",
       "      <th>season_spring</th>\n",
       "      <th>season_summer</th>\n",
       "      <th>season_winter</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>47.259911</td>\n",
       "      <td>-121.999790</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>47.272963</td>\n",
       "      <td>-122.294079</td>\n",
       "      <td>5</td>\n",
       "      <td>23</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>47.275077</td>\n",
       "      <td>-122.254463</td>\n",
       "      <td>7</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Latitude   Longitude  Month  Day  calls  season_Autumn  season_spring  \\\n",
       "0  47.259911 -121.999790      2    1      1              0              0   \n",
       "1  47.272963 -122.294079      5   23      1              0              1   \n",
       "2  47.275077 -122.254463      7   15      1              0              0   \n",
       "\n",
       "   season_summer  season_winter  \n",
       "0              0              1  \n",
       "1              0              0  \n",
       "2              1              0  "
      ]
     },
     "execution_count": 329,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test1_agg1.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "76a0f722",
   "metadata": {},
   "source": [
    "<h2> Experiment 1: Modeling </h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "id": "4761de08",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train1 = train1_agg1.calls\n",
    "x_train1 = train1_agg1.drop(['calls'], axis= 1)\n",
    "\n",
    "y_test1 = test1_agg1.calls\n",
    "x_test1 = test1_agg1.drop(['calls'], axis= 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89385d7d",
   "metadata": {},
   "source": [
    "<h3> Lightgbm Regressor training and prediction </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "id": "da7a0e2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "lgbm1 = lightgbm.LGBMRegressor(n_estimators = 300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "id": "3fa5804b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LGBMRegressor(n_estimators=300)"
      ]
     },
     "execution_count": 232,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lgbm1.fit(x_train1, y_train1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 374,
   "id": "4c20658c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34mMean Absolute Error on test data is :0.16365566571900106\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "pred1 = lgbm1.predict(x_test1)\n",
    "print(colored('Mean Absolute Error on test data is :' +  str(mean_absolute_error(y_test1, pred1)), 'blue'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 376,
   "id": "e059bacd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34mMean Absolute Error on train data is :0.15779555802557307\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "print(colored('Mean Absolute Error on train data is :' +  str(mean_absolute_error(y_train1,lgbm1.predict(x_train1) )), 'blue'))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "13359bee",
   "metadata": {},
   "source": [
    "<h2> Experiment 1: Lightgbm Error Analysis </h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 297,
   "id": "dcca1409",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>actual</th>\n",
       "      <th>predicted</th>\n",
       "      <th>change_percent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   actual  predicted  change_percent\n",
       "0       1        1.0             0.0\n",
       "1       1        1.0             0.0\n",
       "2       1        1.0             0.0"
      ]
     },
     "execution_count": 297,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_p1 = pd.DataFrame()\n",
    "df_p1['actual'] = y_test1\n",
    "df_p1['predicted']= list(pred1)\n",
    "df_p1['predicted'] = round(df_p1['predicted'])\n",
    "df_p1['change_percent'] = abs(df_p1['actual']- df_p1['predicted'])\n",
    "df_p1.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 377,
   "id": "2422a7ba",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34mUnderstanding distribution of predictions in test data\u001b[0m\n",
      " \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1.0    96152\n",
       "2.0      803\n",
       "3.0       33\n",
       "Name: predicted, dtype: int64"
      ]
     },
     "execution_count": 377,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(colored('Understanding distribution of predictions in test data', 'blue'))\n",
    "print(' ')\n",
    "df_p1.predicted.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 378,
   "id": "dc534886",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34mUnderstanding distribution of predictions in training data\u001b[0m\n",
      " \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1     88940\n",
       "2      6753\n",
       "3      1008\n",
       "4       197\n",
       "5        65\n",
       "6        14\n",
       "7         9\n",
       "10        1\n",
       "8         1\n",
       "Name: actual, dtype: int64"
      ]
     },
     "execution_count": 378,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(colored('Understanding distribution of predictions in training data', 'blue'))\n",
    "print(' ')\n",
    "df_p1.actual.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf862477",
   "metadata": {},
   "source": [
    "<h3> <font color=\"blue\">  Observations: The model mostly predict 1,2 and 3 since those are the majority of cases in the training data </font></h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 379,
   "id": "53549395",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34mUnderstanding distribution of errors in test data\u001b[0m\n",
      " \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.25    0.0\n",
       "0.50    0.0\n",
       "0.75    0.0\n",
       "0.85    0.0\n",
       "0.95    1.0\n",
       "0.99    2.0\n",
       "Name: change_percent, dtype: float64"
      ]
     },
     "execution_count": 379,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(colored('Understanding distribution of errors in test data', 'blue'))\n",
    "print(' ')\n",
    "df_p1.change_percent.quantile([0.25, 0.5, 0.75, 0.85, 0.95, 0.99])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4f13022d",
   "metadata": {},
   "source": [
    "<h3> <font color=\"blue\">  Observations: The model predicts correctly for 95% of the data </font></h3>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f91eb13",
   "metadata": {},
   "source": [
    "<h3> Generating results for visualisations </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 354,
   "id": "d7e5e049",
   "metadata": {},
   "outputs": [],
   "source": [
    "# generate data for visualisation \n",
    "results_1 = results_prep_first(test1, loc_filter = True)\n",
    "results_1['predicted'] = list(pred1)\n",
    "results_1['predicted'] = round(results_1['predicted'])\n",
    "results_1['Datetime'] = results_1.apply(lambda x: datetime.date(x['Year'], x['Month'], x['Day']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 355,
   "id": "b43880be",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>season</th>\n",
       "      <th>calls</th>\n",
       "      <th>predicted</th>\n",
       "      <th>Datetime</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>47.259911</td>\n",
       "      <td>-121.999790</td>\n",
       "      <td>2021</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>winter</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2021-02-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>47.272963</td>\n",
       "      <td>-122.294079</td>\n",
       "      <td>2021</td>\n",
       "      <td>5</td>\n",
       "      <td>23</td>\n",
       "      <td>spring</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2021-05-23</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Latitude   Longitude  Year  Month  Day  season  calls  predicted  \\\n",
       "0  47.259911 -121.999790  2021      2    1  winter      1        1.0   \n",
       "1  47.272963 -122.294079  2021      5   23  spring      1        1.0   \n",
       "\n",
       "     Datetime  \n",
       "0  2021-02-01  \n",
       "1  2021-05-23  "
      ]
     },
     "execution_count": 355,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_1.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 356,
   "id": "33183d58",
   "metadata": {},
   "outputs": [],
   "source": [
    "#results_1.to_csv('Results1.csv', index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6a4813c0",
   "metadata": {},
   "source": [
    "<h1> Experiment 1: Daily predictions without location </h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15b94dfb",
   "metadata": {},
   "source": [
    "<h3> <font color=\"blue\"> This involves predicting calls on everyday basis without Locations  </font></h3>\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "921df35b",
   "metadata": {},
   "source": [
    "<h3> Data Preparation </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "id": "04fbab02",
   "metadata": {},
   "outputs": [],
   "source": [
    "test1_agg2 = data_process_first(test1, loc_filter = False)\n",
    "train1_agg2 = data_process_first(train1, loc_filter = False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "id": "883c9f0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train12 = train1_agg2.calls\n",
    "x_train12 = train1_agg2.drop(['calls'], axis= 1)\n",
    "\n",
    "y_test12 = test1_agg2.calls\n",
    "x_test12 = test1_agg2.drop(['calls'], axis= 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64f53a0f",
   "metadata": {},
   "source": [
    "<h3> Lightgbm: Model training and prediction </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "id": "957f611f",
   "metadata": {},
   "outputs": [],
   "source": [
    "lgbm12 = lightgbm.LGBMRegressor(n_estimators = 300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "id": "458f3872",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LGBMRegressor(n_estimators=300)"
      ]
     },
     "execution_count": 335,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lgbm12.fit(x_train12, y_train12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 380,
   "id": "01dcfa5c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34mMean Absolute Error on test data is :31.923202796480215\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "pred12 = lgbm12.predict(x_test12)\n",
    "\n",
    "print(colored('Mean Absolute Error on test data is :' +  str(mean_absolute_error(y_test12, pred12)), 'blue'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 381,
   "id": "82706094",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34mMean Absolute Error on train data is :19.986378984910008\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "print(colored('Mean Absolute Error on train data is :' +  str(mean_absolute_error(y_train12,lgbm12.predict(x_train12) )), 'blue'))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "abb8f507",
   "metadata": {},
   "source": [
    "<h2> Experiment 1: Lightgbm (without location) Error Analysis </h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "id": "b1e9f6af",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>actual</th>\n",
       "      <th>predicted</th>\n",
       "      <th>change_percent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>264</td>\n",
       "      <td>306.0</td>\n",
       "      <td>42.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>225</td>\n",
       "      <td>272.0</td>\n",
       "      <td>47.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>243</td>\n",
       "      <td>281.0</td>\n",
       "      <td>38.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   actual  predicted  change_percent\n",
       "0     264      306.0            42.0\n",
       "1     225      272.0            47.0\n",
       "2     243      281.0            38.0"
      ]
     },
     "execution_count": 337,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_p12 = pd.DataFrame()\n",
    "df_p12['actual'] = y_test12\n",
    "df_p12['predicted']= list(pred12)\n",
    "df_p12['predicted'] = round(df_p12['predicted'])\n",
    "df_p12['change_percent'] = abs(df_p12['actual']- df_p12['predicted'])\n",
    "df_p12.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 382,
   "id": "753d9b77",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34mUnderstanding distribution of errors in test data\u001b[0m\n",
      " \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.25     13.00\n",
       "0.50     26.00\n",
       "0.75     42.00\n",
       "0.85     52.00\n",
       "0.95     76.80\n",
       "0.99    153.08\n",
       "Name: change_percent, dtype: float64"
      ]
     },
     "execution_count": 382,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(colored('Understanding distribution of errors in test data', 'blue'))\n",
    "print(' ')\n",
    "df_p12.change_percent.quantile([0.25, 0.5, 0.75, 0.85, 0.95, 0.99])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7140cae9",
   "metadata": {},
   "source": [
    "<h5> <font color=\"blue\"> Observations: Wwithout location gives poor performance in terms of MAE </font></h5>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5106b045",
   "metadata": {},
   "source": [
    "<h1> Experiment 2: Monthly Predictions </h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bfb93949",
   "metadata": {},
   "source": [
    "<h3> <font color=\"blue\"> This involves predicting calls on monthly basis by Locations  </font></h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "id": "27ea05f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_process_monthly(test1):\n",
    "    '''Function to prepare train and test data '''\n",
    "    \n",
    "    if 'Type' in test1.columns:\n",
    "        test1 = test1.drop(['Type'], axis = 1)\n",
    "    test1['Year'] = test1['Datetime'].apply(lambda x:x.split('/')[2].split(' ')[0] )\n",
    "    test1['Year'] = test1.Year.astype('int')\n",
    "    test1['Month'] =test1.Datetime.apply(lambda x: int(x.split('/')[0]))\n",
    "  \n",
    "    test1['season'] = test1.Month.apply(lambda x: season(x))\n",
    "    test1 = test1.drop(['Datetime'], axis = 1)\n",
    "    #print(test1.head(2))\n",
    "    test1_agg= test1.groupby(['Latitude', 'Longitude', 'Year', 'Month', 'season']).size().reset_index()\n",
    "    \n",
    "    test1_agg.columns = ['Latitude', 'Longitude', 'Year', 'Month','season', 'calls']\n",
    "    #print(test1_agg.head(2))\n",
    "    test1_agg = pd.get_dummies(test1_agg, columns= ['season'])\n",
    "    test1_agg1 = test1_agg.drop(['Year'], axis = 1)\n",
    "    return(test1_agg1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 358,
   "id": "51e682f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def result_process_monthly(test1):\n",
    "    '''Function to generate results for dashboard visualisations''' \n",
    "    \n",
    "    if 'Type' in test1.columns:\n",
    "        test1 = test1.drop(['Type'], axis = 1)\n",
    "    test1['Year'] = test1['Datetime'].apply(lambda x:x.split('/')[2].split(' ')[0] )\n",
    "    test1['Year'] = test1.Year.astype('int')\n",
    "    test1['Month'] =test1.Datetime.apply(lambda x: int(x.split('/')[0]))\n",
    "  \n",
    "    test1['season'] = test1.Month.apply(lambda x: season(x))\n",
    "    test1 = test1.drop(['Datetime'], axis = 1)\n",
    "    #print(test1.head(2))\n",
    "    test1_agg= test1.groupby(['Latitude', 'Longitude', 'Year', 'Month', 'season']).size().reset_index()\n",
    "    \n",
    "    test1_agg.columns = ['Latitude', 'Longitude', 'Year', 'Month','season', 'calls']\n",
    "    \n",
    "    return(test1_agg)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8df1009",
   "metadata": {},
   "source": [
    "<h3> Train and test preparation </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "id": "ba45f717",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    Latitude   Longitude  Year  Month  season\n",
      "0  47.606900 -122.329051  2019      5  spring\n",
      "1  47.515984 -122.266529  2019      5  spring\n",
      "    Latitude   Longitude  Year  Month  season  calls\n",
      "0  47.251232 -122.122934  2020      6  summer      1\n",
      "1  47.261880 -122.225729  2019     12  winter      1\n",
      "        Latitude   Longitude  Year  Month  season\n",
      "43660  47.602203 -122.298914  2021      1  winter\n",
      "43786  47.649847 -122.344754  2021      7  summer\n",
      "    Latitude   Longitude  Year  Month  season  calls\n",
      "0  47.259911 -121.999790  2021      2  winter      1\n",
      "1  47.272963 -122.294079  2021      5  spring      1\n"
     ]
    }
   ],
   "source": [
    "train2_agg1 = data_process_monthly(train)\n",
    "test2_agg1 = data_process_monthly(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "id": "e15922c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "      <th>Month</th>\n",
       "      <th>calls</th>\n",
       "      <th>season_Autumn</th>\n",
       "      <th>season_spring</th>\n",
       "      <th>season_summer</th>\n",
       "      <th>season_winter</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>47.251232</td>\n",
       "      <td>-122.122934</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>47.261880</td>\n",
       "      <td>-122.225729</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Latitude   Longitude  Month  calls  season_Autumn  season_spring  \\\n",
       "0  47.251232 -122.122934      6      1              0              0   \n",
       "1  47.261880 -122.225729     12      1              0              0   \n",
       "\n",
       "   season_summer  season_winter  \n",
       "0              1              0  \n",
       "1              0              1  "
      ]
     },
     "execution_count": 245,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train2_agg1.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "id": "d6388b70",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((279376, 8), (502269, 5))"
      ]
     },
     "execution_count": 246,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train2_agg1.shape, train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "id": "e918cde0",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train2 = train2_agg1.calls\n",
    "x_train2 = train2_agg1.drop(['calls'], axis= 1)\n",
    "\n",
    "y_test2 = test2_agg1.calls\n",
    "x_test2 = test2_agg1.drop(['calls'], axis= 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c15e383e",
   "metadata": {},
   "source": [
    "<h2> Experiment 2: Model training & Prediction </h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "id": "1b0fb2e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "lgbm2 = lightgbm.LGBMRegressor(n_estimators = 300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "id": "e060ea68",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LGBMRegressor(n_estimators=300)"
      ]
     },
     "execution_count": 249,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lgbm2.fit(x_train2, y_train2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 383,
   "id": "07ce9333",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34mMean Absolute Error on test data is :0.9313283223229918\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "pred2 = lgbm2.predict(x_test2)\n",
    "print(colored('Mean Absolute Error on test data is :' +  str(mean_absolute_error(y_test2, pred2)), 'blue'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 385,
   "id": "bee2a798",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34mMean Absolute Error on train data is :0.8612342212033189\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "print(colored('Mean Absolute Error on train data is :' +  str(mean_absolute_error(y_train2,lgbm2.predict(x_train2) )), 'blue'))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ffa11242",
   "metadata": {},
   "source": [
    "<h2> Experiment 2: LightGBM Error Analysis </h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 293,
   "id": "7d7a8ecc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>actual</th>\n",
       "      <th>predicted</th>\n",
       "      <th>change_percent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   actual  predicted  change_percent\n",
       "0       1        1.0             0.0\n",
       "1       1        1.0             0.0\n",
       "2       1        1.0             0.0"
      ]
     },
     "execution_count": 293,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_p2 = pd.DataFrame()\n",
    "df_p2['actual'] = y_test2\n",
    "df_p2['predicted']= list(pred2)\n",
    "df_p2['predicted']=round(df_p2['predicted'])\n",
    "df_p2['change_percent'] = abs(df_p1['actual']- df_p1['predicted'])\n",
    "df_p2.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 386,
   "id": "c4b83ce8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34mUnderstanding distribution of errors in test data\u001b[0m\n",
      " \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.25    0.0\n",
       "0.50    0.0\n",
       "0.75    0.0\n",
       "0.85    0.0\n",
       "0.95    1.0\n",
       "0.99    2.0\n",
       "Name: change_percent, dtype: float64"
      ]
     },
     "execution_count": 386,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(colored('Understanding distribution of errors in test data', 'blue'))\n",
    "print(' ')\n",
    "df_p2.change_percent.quantile([0.25, 0.5, 0.75, 0.85, 0.95, 0.99])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "id": "33095d1b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1     42220\n",
       "2      7753\n",
       "3      2969\n",
       "4      1488\n",
       "5       911\n",
       "6       582\n",
       "7       427\n",
       "8       313\n",
       "9       229\n",
       "10      204\n",
       "11      150\n",
       "12      126\n",
       "13      110\n",
       "14      100\n",
       "15       70\n",
       "16       52\n",
       "17       45\n",
       "19       41\n",
       "18       39\n",
       "20       32\n",
       "21       31\n",
       "22       24\n",
       "23       22\n",
       "25       20\n",
       "27       14\n",
       "24       13\n",
       "29       12\n",
       "26       11\n",
       "28       11\n",
       "30       10\n",
       "31        9\n",
       "32        7\n",
       "33        5\n",
       "39        4\n",
       "34        3\n",
       "38        3\n",
       "35        3\n",
       "36        3\n",
       "43        2\n",
       "37        2\n",
       "40        2\n",
       "45        1\n",
       "61        1\n",
       "46        1\n",
       "Name: actual, dtype: int64"
      ]
     },
     "execution_count": 265,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "df_p2.actual.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "id": "ab2285ac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0     34337\n",
       "2.0     18346\n",
       "3.0      2581\n",
       "4.0       953\n",
       "5.0       607\n",
       "6.0       457\n",
       "8.0       207\n",
       "7.0       182\n",
       "9.0       120\n",
       "12.0       73\n",
       "10.0       66\n",
       "11.0       50\n",
       "21.0       15\n",
       "13.0       15\n",
       "31.0       12\n",
       "14.0        9\n",
       "20.0        8\n",
       "18.0        7\n",
       "43.0        5\n",
       "17.0        5\n",
       "47.0        4\n",
       "23.0        3\n",
       "16.0        2\n",
       "50.0        2\n",
       "32.0        2\n",
       "0.0         2\n",
       "45.0        1\n",
       "22.0        1\n",
       "39.0        1\n",
       "19.0        1\n",
       "52.0        1\n",
       "Name: predicted, dtype: int64"
      ]
     },
     "execution_count": 266,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_p2.predicted.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be84db2e",
   "metadata": {},
   "source": [
    "<h3> Generating results for Visualisation </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 361,
   "id": "a7db4a0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "results_2 = result_process_monthly(test1)\n",
    "results_2['predicted'] = list(pred2)\n",
    "results_2['predicted'] = round(results_2['predicted'])\n",
    "results_2['Datetime'] = results_2.apply(lambda x: datetime.date(x['Year'], x['Month'], 1), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 362,
   "id": "6a8ae134",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>season</th>\n",
       "      <th>calls</th>\n",
       "      <th>predicted</th>\n",
       "      <th>Datetime</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>47.259911</td>\n",
       "      <td>-121.999790</td>\n",
       "      <td>2021</td>\n",
       "      <td>2</td>\n",
       "      <td>winter</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2021-02-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>47.272963</td>\n",
       "      <td>-122.294079</td>\n",
       "      <td>2021</td>\n",
       "      <td>5</td>\n",
       "      <td>spring</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2021-05-01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Latitude   Longitude  Year  Month  season  calls  predicted    Datetime\n",
       "0  47.259911 -121.999790  2021      2  winter      1        1.0  2021-02-01\n",
       "1  47.272963 -122.294079  2021      5  spring      1        1.0  2021-05-01"
      ]
     },
     "execution_count": 362,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_2.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 363,
   "id": "a84d26f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "#results_2.to_csv('Results2.csv', index= False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d016fb9c",
   "metadata": {},
   "source": [
    "<h1> Experiment 3: Monthly sales based on previous month data </h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef20f5ee",
   "metadata": {},
   "source": [
    "<h3> <font color=\"blue\"> This involves predicting calls on monthly based on entire data except last 2 months  </font></h3>\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 281,
   "id": "31457c60",
   "metadata": {},
   "outputs": [],
   "source": [
    "def same_train_test(cols_train, cols_test, test):\n",
    "    '''Function to ensure train and test has same features'''\n",
    "    \n",
    "    # get list of columns not in test\n",
    "    cols_not_in_test = [i for i in cols_train if i not in cols_test]\n",
    "    \n",
    "    # make a column for them with value 0\n",
    "    if cols_not_in_test:\n",
    "        for i in cols_not_in_test:\n",
    "            test[i] = 0\n",
    "    \n",
    "    # re-arrange columns based on training data\n",
    "    test_final = test[cols_train]\n",
    "    return(test_final)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7405f02",
   "metadata": {},
   "source": [
    "<h2> Data Preparation </h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "id": "06adb6a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "data3= data2.copy(deep = True)\n",
    "data3['Month'] = data3.Datetime.apply(lambda x: int(x.split('/')[0]))\n",
    "test3= data3[(data3.Year.isin(['2021']) & data3.Month.isin([11, 12]))]\n",
    "train3 = data3[~(data3.Year.isin(['2021']) & data3.Month.isin([11, 12]))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "id": "0eb3083b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((590538, 6), (18768, 6))"
      ]
     },
     "execution_count": 272,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train3.shape, test3.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "id": "12d12524",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    Latitude   Longitude  Year  Month  season\n",
      "0  47.606900 -122.329051  2019      5  spring\n",
      "1  47.515984 -122.266529  2019      5  spring\n",
      "    Latitude   Longitude  Year  Month  season  calls\n",
      "0  47.251232 -122.122934  2020      6  summer      1\n",
      "1  47.259911 -121.999790  2021      2  winter      1\n",
      "         Latitude   Longitude  Year  Month  season\n",
      "602895  47.617249 -122.347115  2021     11  Autumn\n",
      "652706  47.545638 -122.362183  2021     12  winter\n",
      "    Latitude   Longitude  Year  Month  season  calls\n",
      "0  47.377633 -122.238763  2021     12  winter      1\n",
      "1  47.445253 -122.291809  2021     12  winter      1\n"
     ]
    }
   ],
   "source": [
    "train3_agg1 = data_process_monthly(train3)\n",
    "test3_agg1 = data_process_monthly(test3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 282,
   "id": "1c931ff1",
   "metadata": {},
   "outputs": [],
   "source": [
    "test3_agg1= same_train_test(train3_agg1.columns, test3_agg1.columns, test3_agg1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "id": "b9decaeb",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train3 = train3_agg1.calls\n",
    "x_train3 = train3_agg1.drop(['calls'], axis= 1)\n",
    "\n",
    "y_test3= test3_agg1.calls\n",
    "x_test3 = test3_agg1.drop(['calls'], axis= 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d3626270",
   "metadata": {},
   "source": [
    "<h2> Experiement 3: Model training & Prediction </h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "id": "453f051c",
   "metadata": {},
   "outputs": [],
   "source": [
    "lgbm3 = lightgbm.LGBMRegressor(n_estimators = 300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 287,
   "id": "b72b1cc5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LGBMRegressor(n_estimators=300)"
      ]
     },
     "execution_count": 287,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lgbm3.fit(x_train3, y_train3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 388,
   "id": "c4ee0253",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34mMean Absolute Error on test data is :0.9657449328845706\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "pred3 = lgbm3.predict(x_test3)\n",
    "print(colored('Mean Absolute Error on test data is :' +  str(mean_absolute_error(y_test3, pred3)), 'blue'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 387,
   "id": "5b81f13a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34mMean Absolute Error on train data is :0.8835902047987519\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "print(colored('Mean Absolute Error on train data is :' +  str(mean_absolute_error(y_train3,lgbm3.predict(x_train3) )), 'blue'))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8c484c33",
   "metadata": {},
   "source": [
    "<h2> Experiment 3: Lightgbm Error Analysis </h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 291,
   "id": "7c19733e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>actual</th>\n",
       "      <th>predicted</th>\n",
       "      <th>change_percent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   actual  predicted  change_percent\n",
       "0       1        1.0             0.0\n",
       "1       1        2.0             1.0\n",
       "2       1        1.0             0.0"
      ]
     },
     "execution_count": 291,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_p3 = pd.DataFrame()\n",
    "df_p3['actual'] = y_test3\n",
    "df_p3['predicted']= list(pred3)\n",
    "df_p3['predicted']=round(df_p3['predicted'])\n",
    "df_p3['change_percent'] = abs(df_p3['actual']- df_p3['predicted'])\n",
    "df_p3.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 292,
   "id": "33870d9a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.25     0.0\n",
       "0.50     0.0\n",
       "0.75     1.0\n",
       "0.85     1.0\n",
       "0.95     3.0\n",
       "0.99    10.0\n",
       "Name: change_percent, dtype: float64"
      ]
     },
     "execution_count": 292,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(colored('Understanding distribution of errors in test data', 'blue'))\n",
    "print(' ')\n",
    "df_p3.change_percent.quantile([0.25, 0.5, 0.75, 0.85, 0.95, 0.99])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 295,
   "id": "1f2e152a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1     7154\n",
       "2     1372\n",
       "3      499\n",
       "4      255\n",
       "5      150\n",
       "6       98\n",
       "7       65\n",
       "8       48\n",
       "9       46\n",
       "10      43\n",
       "11      30\n",
       "14      22\n",
       "12      21\n",
       "13      17\n",
       "16      13\n",
       "17      12\n",
       "18      11\n",
       "15       8\n",
       "20       7\n",
       "21       7\n",
       "19       5\n",
       "22       4\n",
       "23       4\n",
       "24       4\n",
       "27       3\n",
       "31       3\n",
       "26       2\n",
       "29       2\n",
       "39       2\n",
       "25       2\n",
       "40       1\n",
       "46       1\n",
       "36       1\n",
       "34       1\n",
       "35       1\n",
       "37       1\n",
       "30       1\n",
       "33       1\n",
       "28       1\n",
       "32       1\n",
       "Name: actual, dtype: int64"
      ]
     },
     "execution_count": 295,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(colored('Understanding distribution of actual in test data', 'blue'))\n",
    "print(' ')\n",
    "df_p3.actual.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 296,
   "id": "a3b0b4e7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0     5774\n",
       "2.0     3194\n",
       "3.0      473\n",
       "4.0      158\n",
       "5.0      104\n",
       "6.0       75\n",
       "8.0       40\n",
       "7.0       33\n",
       "9.0       29\n",
       "10.0       8\n",
       "15.0       7\n",
       "12.0       6\n",
       "19.0       4\n",
       "11.0       4\n",
       "17.0       2\n",
       "14.0       2\n",
       "20.0       2\n",
       "31.0       2\n",
       "43.0       1\n",
       "45.0       1\n",
       "Name: predicted, dtype: int64"
      ]
     },
     "execution_count": 296,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(colored('Understanding distribution of predictions in test data', 'blue'))\n",
    "print(' ')\n",
    "df_p3.predicted.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "17c7b1e0",
   "metadata": {},
   "source": [
    "<h1> Save models  </h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "id": "b65544ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('lgbm1_daily', 'wb') as f:\n",
    "    dill.dump(lgbm1, f)\n",
    "    \n",
    "with open('lgbm2_monthly', 'wb') as f:\n",
    "    dill.dump(lgbm2, f)\n",
    "    \n",
    "with open('lgbm3_monthly', 'wb') as f:\n",
    "    dill.dump(lgbm3, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c8c410f",
   "metadata": {},
   "source": [
    "<h1> Observations </h1>\n",
    "<li> Experiment 1: Daily predictions with location , MAE: 0.163, 95% of predictions were correct </li>\n",
    "<li> Experiment 1.1:  Daily predictions with location , MAE: 31.9, 25% of predictions were correct </li>\n",
    "<li> Experiment 2:  Monthly predictions with location , MAE: 0.93, 95% of predictions were correct </li>\n",
    "<li> Experiment 3:  Monthly predictions with location and all data except last 3 months , MAE: 0.96, 50% of predictions were correct </li>\n",
    "<li> Accuracy measures ca be checked by collecting demographic data, weather data, population data </li>\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1b47f7d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}