dia.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## License \n",
    "\n",
    "Copyright 2019 Patrick Hall and the H2O.ai team\n",
    "\n",
    "Licensed under the Apache License, Version 2.0 (the \"License\");\n",
    "you may not use this file except in compliance with the License.\n",
    "You may obtain a copy of the License at\n",
    "\n",
    "    http://www.apache.org/licenses/LICENSE-2.0\n",
    "\n",
    "Unless required by applicable law or agreed to in writing, software\n",
    "distributed under the License is distributed on an \"AS IS\" BASIS,\n",
    "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
    "See the License for the specific language governing permissions and\n",
    "limitations under the License."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**DISCLAIMER:** This notebook is not legal compliance advice."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Python imports\n",
    "In general, NumPy and Pandas will be used for data manipulation purposes and h2o will be used for modeling tasks. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/patrickh/anaconda3/lib/python3.6/site-packages/sklearn/ensemble/weight_boosting.py:29: DeprecationWarning: numpy.core.umath_tests is an internal NumPy module and should not be imported. It will be removed in a future NumPy release.\n",
      "  from numpy.core.umath_tests import inner1d\n"
     ]
    }
   ],
   "source": [
    "# imports\n",
    "\n",
    "# h2o Python API with specific classes\n",
    "import h2o                                        \n",
    "from h2o.estimators.gbm import H2OGradientBoostingEstimator       # for GBM\n",
    "\n",
    "import numpy as np   # array, vector, matrix calculations\n",
    "import pandas as pd  # DataFrame handling\n",
    "\n",
    "import shap # Python Shapley value package\n",
    "\n",
    "# display plots in-notebook\n",
    "%matplotlib inline   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Start h2o"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking whether there is an H2O instance running at http://localhost:54321 ..... not found.\n",
      "Attempting to start a local H2O server...\n",
      "  Java Version: java version \"1.8.0_201\"; Java(TM) SE Runtime Environment (build 1.8.0_201-b09); Java HotSpot(TM) 64-Bit Server VM (build 25.201-b09, mixed mode)\n",
      "  Starting server from /home/patrickh/anaconda3/lib/python3.6/site-packages/h2o/backend/bin/h2o.jar\n",
      "  Ice root: /tmp/tmphmbx_vxu\n",
      "  JVM stdout: /tmp/tmphmbx_vxu/h2o_patrickh_started_from_python.out\n",
      "  JVM stderr: /tmp/tmphmbx_vxu/h2o_patrickh_started_from_python.err\n",
      "  Server is running at http://127.0.0.1:54321\n",
      "Connecting to H2O server at http://127.0.0.1:54321 ... successful.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td>H2O cluster uptime:</td>\n",
       "<td>01 secs</td></tr>\n",
       "<tr><td>H2O cluster timezone:</td>\n",
       "<td>America/New_York</td></tr>\n",
       "<tr><td>H2O data parsing timezone:</td>\n",
       "<td>UTC</td></tr>\n",
       "<tr><td>H2O cluster version:</td>\n",
       "<td>3.24.0.2</td></tr>\n",
       "<tr><td>H2O cluster version age:</td>\n",
       "<td>19 days </td></tr>\n",
       "<tr><td>H2O cluster name:</td>\n",
       "<td>H2O_from_python_patrickh_llt3t7</td></tr>\n",
       "<tr><td>H2O cluster total nodes:</td>\n",
       "<td>1</td></tr>\n",
       "<tr><td>H2O cluster free memory:</td>\n",
       "<td>1.778 Gb</td></tr>\n",
       "<tr><td>H2O cluster total cores:</td>\n",
       "<td>8</td></tr>\n",
       "<tr><td>H2O cluster allowed cores:</td>\n",
       "<td>8</td></tr>\n",
       "<tr><td>H2O cluster status:</td>\n",
       "<td>accepting new members, healthy</td></tr>\n",
       "<tr><td>H2O connection url:</td>\n",
       "<td>http://127.0.0.1:54321</td></tr>\n",
       "<tr><td>H2O connection proxy:</td>\n",
       "<td>None</td></tr>\n",
       "<tr><td>H2O internal security:</td>\n",
       "<td>False</td></tr>\n",
       "<tr><td>H2O API Extensions:</td>\n",
       "<td>Amazon S3, XGBoost, Algos, AutoML, Core V3, Core V4</td></tr>\n",
       "<tr><td>Python version:</td>\n",
       "<td>3.6.4 final</td></tr></table></div>"
      ],
      "text/plain": [
       "--------------------------  ---------------------------------------------------\n",
       "H2O cluster uptime:         01 secs\n",
       "H2O cluster timezone:       America/New_York\n",
       "H2O data parsing timezone:  UTC\n",
       "H2O cluster version:        3.24.0.2\n",
       "H2O cluster version age:    19 days\n",
       "H2O cluster name:           H2O_from_python_patrickh_llt3t7\n",
       "H2O cluster total nodes:    1\n",
       "H2O cluster free memory:    1.778 Gb\n",
       "H2O cluster total cores:    8\n",
       "H2O cluster allowed cores:  8\n",
       "H2O cluster status:         accepting new members, healthy\n",
       "H2O connection url:         http://127.0.0.1:54321\n",
       "H2O connection proxy:\n",
       "H2O internal security:      False\n",
       "H2O API Extensions:         Amazon S3, XGBoost, Algos, AutoML, Core V3, Core V4\n",
       "Python version:             3.6.4 final\n",
       "--------------------------  ---------------------------------------------------"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "h2o.init(max_mem_size='2G')       # start h2o\n",
    "h2o.remove_all()                  # remove any existing data structures from h2o memory"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Download, explore, and prepare UCI credit card default data\n",
    "\n",
    "UCI credit card default data: https://archive.ics.uci.edu/ml/datasets/default+of+credit+card+clients\n",
    "\n",
    "The UCI credit card default data contains demographic and payment information about credit card customers in Taiwan in the year 2005. The data set contains 23 input variables: \n",
    "\n",
    "* **`LIMIT_BAL`**: Amount of given credit (NT dollar)\n",
    "* **`SEX`**: 1 = male; 2 = female\n",
    "* **`EDUCATION`**: 1 = graduate school; 2 = university; 3 = high school; 4 = others \n",
    "* **`MARRIAGE`**: 1 = married; 2 = single; 3 = others\n",
    "* **`AGE`**: Age in years \n",
    "* **`PAY_0`, `PAY_2` - `PAY_6`**: History of past payment; `PAY_0` = the repayment status in September, 2005; `PAY_2` = the repayment status in August, 2005; ...; `PAY_6` = the repayment status in April, 2005. The measurement scale for the repayment status is: -1 = pay duly; 1 = payment delay for one month; 2 = payment delay for two months; ...; 8 = payment delay for eight months; 9 = payment delay for nine months and above. \n",
    "* **`BILL_AMT1` - `BILL_AMT6`**: Amount of bill statement (NT dollar). `BILL_AMNT1` = amount of bill statement in September, 2005; `BILL_AMT2` = amount of bill statement in August, 2005; ...; `BILL_AMT6` = amount of bill statement in April, 2005. \n",
    "* **`PAY_AMT1` - `PAY_AMT6`**: Amount of previous payment (NT dollar). `PAY_AMT1` = amount paid in September, 2005; `PAY_AMT2` = amount paid in August, 2005; ...; `PAY_AMT6` = amount paid in April, 2005. \n",
    "\n",
    "These 23 input variables are used to predict the target variable, whether or not a customer defaulted on their credit card bill in late 2005."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import data and clean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parse progress: |█████████████████████████████████████████████████████████| 100%\n"
     ]
    }
   ],
   "source": [
    "# import XLS file\n",
    "path = 'default_of_credit_card_clients.xls'\n",
    "data = pd.read_excel(path,\n",
    "                     skiprows=1)\n",
    "\n",
    "# remove spaces from target column name \n",
    "data = data.rename(columns={'default payment next month': 'DEFAULT_NEXT_MONTH'}) \n",
    "data = h2o.H2OFrame(data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Assign modeling roles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y = DEFAULT_NEXT_MONTH\n",
      "X = ['PAY_0', 'PAY_2', 'PAY_3', 'PAY_4', 'PAY_5', 'PAY_6', 'BILL_AMT1', 'BILL_AMT2', 'BILL_AMT3', 'BILL_AMT4', 'BILL_AMT5', 'BILL_AMT6', 'PAY_AMT1', 'PAY_AMT2', 'PAY_AMT3', 'PAY_AMT4', 'PAY_AMT5', 'PAY_AMT6']\n"
     ]
    }
   ],
   "source": [
    "# assign target and inputs for GBM\n",
    "y = 'DEFAULT_NEXT_MONTH'\n",
    "X = [name for name in data.columns if name not in [y, 'ID', 'AGE', 'EDUCATION', 'SEX', 'MARRIAGE', 'LIMIT_BAL']]\n",
    "print('y =', y)\n",
    "print('X =', X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ensure target is handled as a categorical variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[y] = data[y].asfactor() "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Display descriptive statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rows:30000\n",
      "Cols:19\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th>       </th><th>PAY_0                </th><th>PAY_2              </th><th>PAY_3               </th><th>PAY_4              </th><th>PAY_5               </th><th>PAY_6              </th><th>BILL_AMT1        </th><th>BILL_AMT2        </th><th>BILL_AMT3         </th><th>BILL_AMT4         </th><th>BILL_AMT5        </th><th>BILL_AMT6        </th><th>PAY_AMT1         </th><th>PAY_AMT2         </th><th>PAY_AMT3         </th><th>PAY_AMT4          </th><th>PAY_AMT5          </th><th>PAY_AMT6         </th><th>DEFAULT_NEXT_MONTH  </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>type   </td><td>int                  </td><td>int                </td><td>int                 </td><td>int                </td><td>int                 </td><td>int                </td><td>int              </td><td>int              </td><td>int               </td><td>int               </td><td>int              </td><td>int              </td><td>int              </td><td>int              </td><td>int              </td><td>int               </td><td>int               </td><td>int              </td><td>enum                </td></tr>\n",
       "<tr><td>mins   </td><td>-2.0                 </td><td>-2.0               </td><td>-2.0                </td><td>-2.0               </td><td>-2.0                </td><td>-2.0               </td><td>-165580.0        </td><td>-69777.0         </td><td>-157264.0         </td><td>-170000.0         </td><td>-81334.0         </td><td>-339603.0        </td><td>0.0              </td><td>0.0              </td><td>0.0              </td><td>0.0               </td><td>0.0               </td><td>0.0              </td><td>                    </td></tr>\n",
       "<tr><td>mean   </td><td>-0.016700000000000118</td><td>-0.1337666666666667</td><td>-0.16619999999999988</td><td>-0.2206666666666661</td><td>-0.26620000000000105</td><td>-0.2910999999999988</td><td>51223.33090000043</td><td>49179.07516666629</td><td>47013.154799999655</td><td>43262.94896666659 </td><td>40311.40096666682</td><td>38871.76039999976</td><td>5663.580500000035</td><td>5921.163499999979</td><td>5225.681499999981</td><td>4826.076866666652 </td><td>4799.387633333336 </td><td>5215.50256666665 </td><td>                    </td></tr>\n",
       "<tr><td>maxs   </td><td>8.0                  </td><td>8.0                </td><td>8.0                 </td><td>8.0                </td><td>8.0                 </td><td>8.0                </td><td>964511.0         </td><td>983931.0         </td><td>1664089.0         </td><td>891586.0          </td><td>927171.0         </td><td>961664.0         </td><td>873552.0         </td><td>1684259.0        </td><td>896040.0         </td><td>621000.0          </td><td>426529.0          </td><td>528666.0         </td><td>                    </td></tr>\n",
       "<tr><td>sigma  </td><td>1.1238015279973332   </td><td>1.1971859730345495 </td><td>1.1968675684465686  </td><td>1.1691386224023357 </td><td>1.1331874060027525  </td><td>1.149987625607897  </td><td>73635.86057552966</td><td>71173.76878252832</td><td>69349.38742703677 </td><td>64332.856133916444</td><td>60797.15577026471</td><td>59554.1075367459 </td><td>16563.28035402577</td><td>23040.87040205719</td><td>17606.96146980311</td><td>15666.159744032062</td><td>15278.305679144742</td><td>17777.46577543531</td><td>                    </td></tr>\n",
       "<tr><td>zeros  </td><td>14737                </td><td>15730              </td><td>15764               </td><td>16455              </td><td>16947               </td><td>16286              </td><td>2008             </td><td>2506             </td><td>2870              </td><td>3195              </td><td>3506             </td><td>4020             </td><td>5249             </td><td>5396             </td><td>5968             </td><td>6408              </td><td>6703              </td><td>7173             </td><td>                    </td></tr>\n",
       "<tr><td>missing</td><td>0                    </td><td>0                  </td><td>0                   </td><td>0                  </td><td>0                   </td><td>0                  </td><td>0                </td><td>0                </td><td>0                 </td><td>0                 </td><td>0                </td><td>0                </td><td>0                </td><td>0                </td><td>0                </td><td>0                 </td><td>0                 </td><td>0                </td><td>0                   </td></tr>\n",
       "<tr><td>0      </td><td>2.0                  </td><td>2.0                </td><td>-1.0                </td><td>-1.0               </td><td>-2.0                </td><td>-2.0               </td><td>3913.0           </td><td>3102.0           </td><td>689.0             </td><td>0.0               </td><td>0.0              </td><td>0.0              </td><td>0.0              </td><td>689.0            </td><td>0.0              </td><td>0.0               </td><td>0.0               </td><td>0.0              </td><td>1                   </td></tr>\n",
       "<tr><td>1      </td><td>-1.0                 </td><td>2.0                </td><td>0.0                 </td><td>0.0                </td><td>0.0                 </td><td>2.0                </td><td>2682.0           </td><td>1725.0           </td><td>2682.0            </td><td>3272.0            </td><td>3455.0           </td><td>3261.0           </td><td>0.0              </td><td>1000.0           </td><td>1000.0           </td><td>1000.0            </td><td>0.0               </td><td>2000.0           </td><td>1                   </td></tr>\n",
       "<tr><td>2      </td><td>0.0                  </td><td>0.0                </td><td>0.0                 </td><td>0.0                </td><td>0.0                 </td><td>0.0                </td><td>29239.0          </td><td>14027.0          </td><td>13559.0           </td><td>14331.0           </td><td>14948.0          </td><td>15549.0          </td><td>1518.0           </td><td>1500.0           </td><td>1000.0           </td><td>1000.0            </td><td>1000.0            </td><td>5000.0           </td><td>0                   </td></tr>\n",
       "<tr><td>3      </td><td>0.0                  </td><td>0.0                </td><td>0.0                 </td><td>0.0                </td><td>0.0                 </td><td>0.0                </td><td>46990.0          </td><td>48233.0          </td><td>49291.0           </td><td>28314.0           </td><td>28959.0          </td><td>29547.0          </td><td>2000.0           </td><td>2019.0           </td><td>1200.0           </td><td>1100.0            </td><td>1069.0            </td><td>1000.0           </td><td>0                   </td></tr>\n",
       "<tr><td>4      </td><td>-1.0                 </td><td>0.0                </td><td>-1.0                </td><td>0.0                </td><td>0.0                 </td><td>0.0                </td><td>8617.0           </td><td>5670.0           </td><td>35835.0           </td><td>20940.0           </td><td>19146.0          </td><td>19131.0          </td><td>2000.0           </td><td>36681.0          </td><td>10000.0          </td><td>9000.0            </td><td>689.0             </td><td>679.0            </td><td>0                   </td></tr>\n",
       "<tr><td>5      </td><td>0.0                  </td><td>0.0                </td><td>0.0                 </td><td>0.0                </td><td>0.0                 </td><td>0.0                </td><td>64400.0          </td><td>57069.0          </td><td>57608.0           </td><td>19394.0           </td><td>19619.0          </td><td>20024.0          </td><td>2500.0           </td><td>1815.0           </td><td>657.0            </td><td>1000.0            </td><td>1000.0            </td><td>800.0            </td><td>0                   </td></tr>\n",
       "<tr><td>6      </td><td>0.0                  </td><td>0.0                </td><td>0.0                 </td><td>0.0                </td><td>0.0                 </td><td>0.0                </td><td>367965.0         </td><td>412023.0         </td><td>445007.0          </td><td>542653.0          </td><td>483003.0         </td><td>473944.0         </td><td>55000.0          </td><td>40000.0          </td><td>38000.0          </td><td>20239.0           </td><td>13750.0           </td><td>13770.0          </td><td>0                   </td></tr>\n",
       "<tr><td>7      </td><td>0.0                  </td><td>-1.0               </td><td>-1.0                </td><td>0.0                </td><td>0.0                 </td><td>-1.0               </td><td>11876.0          </td><td>380.0            </td><td>601.0             </td><td>221.0             </td><td>-159.0           </td><td>567.0            </td><td>380.0            </td><td>601.0            </td><td>0.0              </td><td>581.0             </td><td>1687.0            </td><td>1542.0           </td><td>0                   </td></tr>\n",
       "<tr><td>8      </td><td>0.0                  </td><td>0.0                </td><td>2.0                 </td><td>0.0                </td><td>0.0                 </td><td>0.0                </td><td>11285.0          </td><td>14096.0          </td><td>12108.0           </td><td>12211.0           </td><td>11793.0          </td><td>3719.0           </td><td>3329.0           </td><td>0.0              </td><td>432.0            </td><td>1000.0            </td><td>1000.0            </td><td>1000.0           </td><td>0                   </td></tr>\n",
       "<tr><td>9      </td><td>-2.0                 </td><td>-2.0               </td><td>-2.0                </td><td>-2.0               </td><td>-1.0                </td><td>-1.0               </td><td>0.0              </td><td>0.0              </td><td>0.0               </td><td>0.0               </td><td>13007.0          </td><td>13912.0          </td><td>0.0              </td><td>0.0              </td><td>0.0              </td><td>13007.0           </td><td>1122.0            </td><td>0.0              </td><td>0                   </td></tr>\n",
       "</tbody>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data[X + [y]].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Train an H2O Montonic GBM classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Split data into training and test sets for early stopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train data rows = 21060, columns = 25\n",
      "Test data rows = 8940, columns = 25\n"
     ]
    }
   ],
   "source": [
    "# split into training and validation\n",
    "train, test = data.split_frame([0.7], seed=12345)\n",
    "\n",
    "# summarize split\n",
    "print('Train data rows = %d, columns = %d' % (train.shape[0], train.shape[1]))\n",
    "print('Test data rows = %d, columns = %d' % (test.shape[0], test.shape[1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Establish monotonicity constraints based on Pearson correlation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DEFAULT_NEXT_MONTH</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>PAY_0</th>\n",
       "      <td>0.318419</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PAY_2</th>\n",
       "      <td>0.250650</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PAY_3</th>\n",
       "      <td>0.220672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PAY_4</th>\n",
       "      <td>0.201708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PAY_5</th>\n",
       "      <td>0.193368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PAY_6</th>\n",
       "      <td>0.183716</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BILL_AMT1</th>\n",
       "      <td>-0.023252</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BILL_AMT2</th>\n",
       "      <td>-0.019563</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BILL_AMT3</th>\n",
       "      <td>-0.021105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BILL_AMT4</th>\n",
       "      <td>-0.018092</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BILL_AMT5</th>\n",
       "      <td>-0.012072</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BILL_AMT6</th>\n",
       "      <td>-0.011350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PAY_AMT1</th>\n",
       "      <td>-0.085398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PAY_AMT2</th>\n",
       "      <td>-0.062419</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PAY_AMT3</th>\n",
       "      <td>-0.069603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PAY_AMT4</th>\n",
       "      <td>-0.055825</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PAY_AMT5</th>\n",
       "      <td>-0.051942</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PAY_AMT6</th>\n",
       "      <td>-0.047694</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           DEFAULT_NEXT_MONTH\n",
       "PAY_0                0.318419\n",
       "PAY_2                0.250650\n",
       "PAY_3                0.220672\n",
       "PAY_4                0.201708\n",
       "PAY_5                0.193368\n",
       "PAY_6                0.183716\n",
       "BILL_AMT1           -0.023252\n",
       "BILL_AMT2           -0.019563\n",
       "BILL_AMT3           -0.021105\n",
       "BILL_AMT4           -0.018092\n",
       "BILL_AMT5           -0.012072\n",
       "BILL_AMT6           -0.011350\n",
       "PAY_AMT1            -0.085398\n",
       "PAY_AMT2            -0.062419\n",
       "PAY_AMT3            -0.069603\n",
       "PAY_AMT4            -0.055825\n",
       "PAY_AMT5            -0.051942\n",
       "PAY_AMT6            -0.047694"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(test.as_data_frame()[X + [y]].corr()[y]).iloc[:-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'BILL_AMT1': -1,\n",
       " 'BILL_AMT2': -1,\n",
       " 'BILL_AMT3': -1,\n",
       " 'BILL_AMT4': -1,\n",
       " 'BILL_AMT5': -1,\n",
       " 'BILL_AMT6': -1,\n",
       " 'PAY_0': 1,\n",
       " 'PAY_2': 1,\n",
       " 'PAY_3': 1,\n",
       " 'PAY_4': 1,\n",
       " 'PAY_5': 1,\n",
       " 'PAY_6': 1,\n",
       " 'PAY_AMT1': -1,\n",
       " 'PAY_AMT2': -1,\n",
       " 'PAY_AMT3': -1,\n",
       " 'PAY_AMT4': -1,\n",
       " 'PAY_AMT5': -1,\n",
       " 'PAY_AMT6': -1}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "names = list(test.as_data_frame()[X + [y]].corr()[y].index)[:-1]\n",
    "signs = list([int(i) for i in np.sign(test.as_data_frame()[X + [y]].corr()[y].values[:-1])])\n",
    "\n",
    "mc = {}\n",
    "for i, name in enumerate(names):\n",
    "    mc[name] = signs[i]\n",
    "\n",
    "mc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train h2o monotonic GBM classifier "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gbm Model Build progress: |███████████████████████████████████████████████| 100%\n",
      "GBM Test AUC = 0.77\n"
     ]
    }
   ],
   "source": [
    "# initialize GBM model\n",
    "model = H2OGradientBoostingEstimator(ntrees=150,                # maximum 150 trees in GBM\n",
    "                                     max_depth=4,               # trees can have maximum depth of 4\n",
    "                                     sample_rate=0.9,           # use 90% of rows in each iteration (tree)\n",
    "                                     col_sample_rate=0.9,       # use 90% of variables in each iteration (tree)\n",
    "                                     #balance_classes=True,     # sample to balance 0/1 distribution of target - can help LOCO\n",
    "                                     stopping_rounds=5,         # stop if validation error does not decrease for 5 iterations (trees)\n",
    "                                     score_tree_interval=1,     # for reproducibility, set higher for bigger data\n",
    "                                     monotone_constraints=mc,   # enforces monotonicity wrt each input\n",
    "                                     model_id='dia_gbm',        # for locating the model in Flow UI \n",
    "                                     seed=12345)                # for reproducibility\n",
    "\n",
    "# train a GBM model\n",
    "model.train(y=y, x=X, training_frame=train, validation_frame=test)\n",
    "\n",
    "# print AUC\n",
    "print('GBM Test AUC = %.2f' % model.auc(valid=True))\n",
    "\n",
    "# uncomment to see model details\n",
    "# print(model) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Display variable importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/patrickh/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:2: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n",
      "  \n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh0AAAIMCAYAAABR6h7ZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3X+UnVWd5/t3iB3oGCAYGmVofijehFGhmV5fcAJDg4wBaa2FY1TomzQgkCqFFdZwtS8ZQxY2+SFzwbadBbG1QkBJgpgF0zEXbVrRgOkm4pcfAS8MLoxo0rFwIqSNBDEhdf94nqIPx1PJOadOnlSl3q+1aiVn7/3svZ/6pz5r732eZ0x/fz+SJEl72wH7egKSJGl0MHRIkqRKGDokSVIlDB2SJKkShg5JklQJQ4ckSaqEoUOSJFXC0CFJkiph6JAkSZV4w76ewP5m9erV/V1dXft6GpIkVWVMsw1d6ZAkSZUwdEiSpEoYOiRJUiUMHZIkqRKGDkmSVAlDhyRJqoShQ5IkVcLQIUmSKmHokCRJlTB0SJKkShg6JElSJQwdkiSpEoYOSZJUCUOHJEmqhKFDkiRVYkx/f/++nsN+5ewrfu4vVJI0LH138TF7o9sxzTZ0pUOSJFXC0CFJkiph6JAkSZUwdEiSpEoYOiRJUiUMHZIkqRJv2NcT2J2IWANMBXYArwIbgAWZeXdNmyXAZcCZmflgWTYeeARYkZnza9oeBTwJ9GTmyj2MPQe4CjgU+CegOzOf69jNSZI0yoyElY75mTkBmATcCdwVEZMBIuJg4ELgBaBn4ILM3A7MAOZExCll2zHAbcCqJgLHxcDVwJ8DRwDPAqsiYiT8viRJGpZGzB/RzNwJLAbGAieWxTOBV4DZwPSImFTT/lHgemBZufIxGzieYvViT7qBxZn5eGa+BMwBplCsukiSpDaMmNAREeOAKym2WtaXxd3AcmAlsA24uO6yG4HNwApgATAjM7c1MdyfUGzPAJCZvwZ+UpZLkqQ2jITQMTcitgKbgPOB6Zn5bEScCpwMLM3MHcAdFCHkNZm5C7gImAbcnJnr9jRYuQ3zRuBf66q2AocM9WYkSRqtRkLoWJiZEzPziMw8LTNXl+U9wGOZ+Xj5+VZgSkScVXtxZm4EXgSeaGawzOwHXqI4QFprIvDrNu9BkqRRb1h/e2UwEXEIcAFwQET01VT1U6x2rBniEOuBPwX+35rxjufftnUkSVKLRmTooDhAugs4CdheU/4B4JaIODwztwyh/y8DN0TE31N8c2UR8Azw0BD6lCRpVBupoaMb6M3MDbWFEXE7MA+4BLip3c4z8ysR8e+A+/i353R8sDwjIkmS2jCmv79/X89hv3L2FT/3FypJGpa+u/iYvdHtmGYbjtSVjmHr6vPW09XVta+nIUnSsDMqQ0dEzAOuGaR6WmZ6dkOSpA4blaGjfB/L/D02lCRJHTMSntMhSZL2A4YOSZJUCUOHJEmqhKFDkiRVwtAhSZIq4cPBOsyHg+0f9tIDdCRpf9T0w8Fc6ZAkSZUwdEiSpEoYOiRJUiUMHZIkqRKGDkmSVIlh/e6ViFgDTAV2AK8CG4AFmXl3TZslwGXAmZn5YFk2HngEWFG+Z2Wg7VHAk0BPZq7czbj/N/AXwNuAl4E1wF9l5sZO3p8kSaPJSFjpmJ+ZE4BJwJ3AXRExGSAiDgYuBF4AegYuyMztwAxgTkScUrYdA9wGrNpd4Cj9AXAl8GZgMvA74O87eVOSJI02IyF0AJCZO4HFwFjgxLJ4JvAKMBuYHhGTato/ClwPLCtXPmYDxwNXNTHWwsz858z8bWb+Gvgc8KcRcUgn70mSpNFkxISOiBhHsfqwA1hfFncDy4GVwDbg4rrLbgQ2AyuABcCMzNzWxvD/GXiuDCCSJKkNIyF0zI2IrcAm4HxgemY+GxGnAicDSzNzB3AHRQh5TWbuAi4CpgE3Z+a6VgePiDOAvwY+MbTbkCRpdBvWB0lLCzNzQYPyHuCxzHy8/HwrcHVEnJWZawYaZebGiHgReKLVgSPiLOAe4LLM/IeWZy5Jkl4zEkLH7ynPVlwAHBARfTVV/RSrHWs6MMafU2zLXJyZq4banyRJo91I2F5pZCawC3gXxRbLwE83xYHSw4fSeUR8lOKbMn9h4JAkqTNG5EoHRbjozcwNtYURcTswD7gEuGkI/X8OeCOwMiJqyydn5uYh9CtJ0qjlq+07zFfb7x98tb0kNc1X20uSpOFlpG6vDElEzAOuGaR6WmY+1G7fV5+3nq6urnYvlyRpvzUqQ0f5Ppb5e2woSZI6xu0VSZJUCUOHJEmqhKFDkiRVwtAhSZIqYeiQJEmVMHRIkqRK+ETSDvOJpHuHTwiVpGHLJ5JKkqThxdAhSZIqYeiQJEmVMHRIkqRKGDokSVIlhvUL3yJiDTAV2AG8CmwAFmTm3TVtlgCXAWdm5oNl2XjgEWBF+XK3gbZHAU8CPZm5cjfj/lfgSuAIYCfwMPBXmfmjjt6gJEmjyEhY6ZifmROAScCdwF0RMRkgIg4GLgReAHoGLsjM7cAMYE5EnFK2HQPcBqzaXeAorQamZuahwJHA94B7O3pXkiSNMiMhdACQmTuBxcBY4MSyeCbwCjAbmB4Rk2raPwpcDywrVz5mA8cDVzUx1k8yc0v5cQywCzi67EeSJLVhWG+v1IqIcRRbHjuA9WVxN7AcWAl8AbgY+Juay24E3gesAM4GzsnMbU2OdyawCjgU6AduKFdQJElSG0bCSsfciNgKbALOB6Zn5rMRcSpwMrA0M3cAd1CEkNdk5i7gImAacHNmrmt20Mx8IDMnAm8CPklxrkOSJLVpJISOhZk5MTOPyMzTMnN1Wd4DPJaZj5efbwWmRMRZtRdn5kbgReCJdgbPzBeB/wF8deAsiSRJat2I2V6pFRGHABcAB0REX01VP8Vqx5oOD3kAMA54G/DjDvctSdKoMBJWOhqZSXG4810UWywDP90UB0oPH0rnEfGJ8uu1lH0tBn4D/HAo/UqSNJqNyJUOinDRm5kbagsj4nZgHnAJcNMQ+n83MC8iDgW2AT8A3puZvxpCn5IkjWq+2r7DfLX93uGr7SVp2PLV9pIkaXgZqdsrQxIR84BrBqmelpkPtdv31eetp6urq93LJUnab43K0FG+j2X+HhtKkqSOcXtFkiRVwtAhSZIqYeiQJEmVMHRIkqRKGDokSVIlDB2SJKkSPpG0w3wiafN8yqgk7Rd8IqkkSRpeDB2SJKkShg5JklQJQ4ckSaqEoUOSJFWipRe+RcQaYCqwA3gV2AAsyMy7a9osAS4DzszMB8uy8cAjwIryZWsDbY8CngR6MnNlE+NfAtwGzM3MRXV1a4HTgemZeU9N+enAWuAnmfn2iHgGOKqsHkcRvH5b09Vk4ApgTl35FzJz7p7mKEmSGmtnpWN+Zk4AJgF3AndFxGSAiDgYuBB4AegZuCAztwMzgDkRcUrZdgxFgFjVTOAodZd9z4qIRnN/GphVVzarLB+Yy5TMnFDewyJgzcDn8mdz2fQ7deUGDkmShqDt7ZXM3AksBsYCJ5bFM4FXgNnA9IiYVNP+UeB6YFm58jEbOB64qpnxIuJEilWWi4CjgXMaNFsJnBoRx5bXHAp8EPhKq/cnSZI6q+3QERHjgCsptlrWl8XdwHKKP/7bgIvrLrsR2AysABYAMzJzW5ND9gCPZea9wH3lWPVeBr4GXFp+ngF8B3i+yTFqnR4RWyJiQ0R8KSIOb6MPSZJUaid0zI2IrcAm4HyKMxTPRsSpwMnA0szcAdxBXTDIzF0UKxXTgJszc10zA5YrIzOBpWXRrUBXRBzZoHkvcGlEjKXYWult9QYpgss7gD8C3gscB/zPNvqRJEmldkLHwsycmJlHZOZpmbm6LB9YiXi8/HwrMCUizqq9ODM3Ai8CT7Qw5gXAQRSrKACrKc52XFbfsBy/D7gOOAz4dgvjDPTxo8zcmJn9mbmBIjz9p4FtG0mS1LqOfGU2Ig6hCAYnRERfRPQB9wP9NN4GaVUPxdmRp8u+NwITgcsHOVD6ZeBaYEm5ujJUA300/Xx5SZL0ei19ZXY3ZlL8YT4J2F5T/gHglog4PDO3tNNxRJwEvBt4P/BoTdWRwA+Bc4Fv1V22DPgpkG2OOR14IDO3RMQfA18EHs7M59rpT5IkdS50dAO95VbEayLidmAecAlwU5t991D8wf9mXXlfRNxT1r8udGTmyxQHSNv1EeDvIuKNwK8oDq5ePoT+JEka9Xy1fYf5avvm+Wp7Sdov+Gp7SZI0vHRqe2VIImIecM0g1dMy86Eq5zMUV5+3nq6urn09DUmShp1hETrK97HM32NDSZI0Yrm9IkmSKmHokCRJlTB0SJKkShg6JElSJQwdkiSpEoYOSZJUCZ9I2mGj8YmkPllUkkY1n0gqSZKGF0OHJEmqhKFDkiRVwtAhSZIqYeiQJEmVaOmFbxGxBpgK7ABeBTYACzLz7po2S4DLgDMz88GybDzwCLCifLnbQNujgCeBnsxc2cT4lwC3AXMzc1Fd3VrgdGB6Zt5TU346sBb4SWa+PSKeAY4qq8dRBK/f1nQ1GXgXMAc4GTgMODIz+/Y0P0mSNLh2VjrmZ+YEYBJwJ3BXREwGiIiDgQuBF4CegQsyczswA5gTEaeUbcdQBIhVzQSOUnfZ96yIaDT3p4FZdWWzyvKBuUzJzAnlPSwC1gx8Ln82A78BbgcuaXJekiRpD9reXsnMncBiYCxwYlk8E3gFmA1Mj4hJNe0fBa4HlpUrH7OB44GrmhkvIk6kWGW5CDgaOKdBs5XAqRFxbHnNocAHga+0eG//nJlfBZ5q5TpJkjS4tkNHRIwDrqTYallfFncDyyn++G8DLq677EZgM7ACWADMyMxtTQ7ZAzyWmfcC95Vj1XsZ+Bpwafl5BvAd4Pkmx5AkSXtJO6FjbkRsBTYB51OcoXg2Ik6lOAOxNDN3AHdQFwwycxfFSsU04ObMXNfMgOXKyExgaVl0K9AVEUc2aN4LXBoRYym2VnpbvUFJktR57YSOhZk5MTOPyMzTMnN1WT6wEvF4+flWYEpEnFV7cWZuBF4EnmhhzAuAgyhWUQBWU5ztuKy+YTl+H3AdxSHQb7cwjiRJ2ks68pXZiDiEIhicEBF9EdEH3A/003gbpFU9FGdHni773ghMBC4f5EDpl4FrgSXl6ookSdrHWvrK7G7MBHYBJwHba8o/ANwSEYdn5pZ2Oo6Ik4B3A+8HHq2pOhL4IXAu8K26y5YBPwWyzTEPoPg67biy6MCIOAh4JTNH3QvdJEnqhE6Fjm6gNzM31BZGxO3APIqvnt7UZt89wMOZ+c268r6IuKesf13oyMyXKQ6QtutsXr8t81z57xkUz/yQJEkt8tX2Hear7SVJo4yvtpckScNLp7ZXhiQi5gHXDFI9LTMfqnI+Q3H1eevp6ura19OQJGnYGRaho3wfy/w9NpQkSSOW2yuSJKkShg5JklQJQ4ckSaqEoUOSJFXC0CFJkiph6JAkSZXwiaQdNhqeSOoTSCVJNXwiqSRJGl4MHZIkqRKGDkmSVAlDhyRJqoShQ5IkVWJYvPBtMBGxBpgK7ABeBTYACzLz7po2S4DLgDMz88GybDzwCLCifJncQNujgCeBnsxc2eQc7gY+BEzNzHWduC9JkkajkbDSMT8zJwCTgDuBuyJiMkBEHAxcCLwA9AxckJnbgRnAnIg4pWw7BrgNWNVC4LgAOLSD9yJJ0qg1EkIHAJm5E1gMjAVOLItnAq8As4HpETGppv2jwPXAsnLlYzZwPHBVM+NFxB8BN1ATZiRJUvtGTOiIiHHAlRRbLevL4m5gObAS2AZcXHfZjcBmYAWwAJiRmduaHHIx8DfAz4Y2c0mSBCMjdMyNiK3AJuB8YHpmPhsRpwInA0szcwdwB0UIeU1m7gIuAqYBNzd7JiMiPgwcBdzSuduQJGl0GwmhY2FmTszMIzLztMxcXZb3AI9l5uPl51uBKRFxVu3FmbkReBF4opnByi2azwOXl6FFkiR1wLD+9spgIuIQ4ALggIjoq6nqp1jtWDOE7v8D8BbgwYioLf+HiLg5M68dQt+SJI1aIzJ0UBwg3QWcBGyvKf8AcEtEHJ6ZW9rs+/vAW2s+jwWeA/4SeLDNPiVJGvVGaujoBnozc0NtYUTcDswDLgFuaqfjzHyF4vzIQJ8Dv6P/nZn/2k6fkiTJV9t3nK+2lySNMr7aXpIkDS8jdXtlSCJiHnDNINXTMvOhdvu++rz1dHV1tXu5JEn7rVEZOsr3sczfY0NJktQxbq9IkqRKGDokSVIlDB2SJKkShg5JklQJQ4ckSaqEDwfrsP354WA+FEyS1IAPB5MkScOLoUOSJFXC0CFJkiph6JAkSZUwdEiSpEoYOiRJUiWG9QvfImINMBXYAbwKbAAWZObdNW2WAJcBZ2bmg2XZeOARYEX5creBtkcBTwI9mblyN+MuAOYAv60p/kJmzu3QrUmSNOqMhJWO+Zk5AZgE3AncFRGTASLiYOBC4AWgZ+CCzNwOzADmRMQpZdsxwG3Aqt0FjhrfycwJNT8GDkmShmAkhA4AMnMnsBgYC5xYFs8EXgFmA9MjYlJN+0eB64Fl5crHbOB44Koq5y1JkgrDenulVkSMA66k2GpZXxZ3A8uBlcAXgIuBv6m57EbgfcAK4GzgnMzc1uSQp0fEFuDXwLeBuZm5Zaj3IUnSaDUSVjrmRsRWYBNwPjA9M5+NiFOBk4GlmbkDuIMihLwmM3cBFwHTgJszc12TY34NeAfwR8B7geOA/9mBe5EkadQaCSsdCzNzQYPyHuCxzHy8/HwrcHVEnJWZawYaZebGiHgReKLZATPzRzUfN0REN/BcRBybmT9r/RYkSdJICB2/JyIOAS4ADoiIvpqqforVjjUdHnJX+W/TL7WRJEmvNxK2VxqZSREE3kWxxTLw001xoPTwoXQeEa/1ERF/DHwReDgznxtKv5IkjWYjcqWDIlz0ZuaG2sKIuB2YB1wC3DSE/j8C/F1EvBH4FXAfcPkQ+pMkadQb09/fv6/nsF85+4qf77e/0O8uPmZfT0GSNPw0ffRgpG6vSJKkEWakbq8MSUTMA64ZpHpaZj7Ubt9Xn7eerq6udi+XJGm/NSpDR/k+lvl7bChJkjrG7RVJklQJQ4ckSaqEoUOSJFXC0CFJkiph6JAkSZXw4WAdtr88HMwHgUmSmuTDwSRJ0vBi6JAkSZUwdEiSpEoYOiRJUiUMHZIkqRKGDkmSVIlh/cK3iFgDTAV2AK8CG4AFmXl3TZslwGXAmZn5YFk2HngEWFG+3G2g7VHAk0BPZq7cw9gB/Hfg3cDvgDWZ+aHO3Z0kSaPLSFjpmJ+ZE4BJwJ3AXRExGSAiDgYuBF4AegYuyMztwAxgTkScUrYdA9wGrGoicLwTuB/4GnAE8GZgUYfvS5KkUWUkhA4AMnMnsBgYC5xYFs8EXgFmA9MjYlJN+0eB64Fl5crHbOB44KomhvsMsDozezNze2buyMzs2M1IkjQKjZjQERHjgCsptlrWl8XdwHJgJbANuLjushuBzcAKYAEwIzO3NTHce4BfRsSDEfGriPhBRLy3A7chSdKoNRJCx9yI2ApsAs4HpmfmsxFxKnAysDQzdwB3UISQ12TmLuAiYBpwc2au29Ng5TbMJOBy4NPAW4C/A74REcd17K4kSRplRkLoWJiZEzPziMw8LTNXl+U9wGOZ+Xj5+VZgSkScVXtxZm4EXgSeaGawzOwHfgPck5lry62V2ygOsZ7TgfuRJGlUGtbfXhlMRBwCXAAcEBF9NVX9FKsda4Y4xONlX/X2i5e5SZK0L4zI0EFxgHQXcBKwvab8A8AtEXF4Zm4ZQv+LgS9FxJeAhym+CXMc8I9D6FOSpFFtpIaObqA3MzfUFkbE7cA84BLgpnY7z8w7I+II4OvAROBp4P2Z+bN2+5QkabQb09/vjkEnnX3Fz/eLX+h3Fx+zr6cgSRoZxjTbcCQcJJUkSfuBkbq9MiQRMQ+4ZpDqaZn5ULt9X33eerq6utq9XJKk/daoDB3l+1jm77GhJEnqGLdXJElSJQwdkiSpEoYOSZJUCUOHJEmqhKFDkiRVwoeDddhIeziYDwGTJA2RDweTJEnDi6FDkiRVwtAhSZIqYeiQJEmVMHRIkqRKGDokSVIl9vjCt4hYA0wFdpRFfcDNmfm3Zf1zwLWZuSwijgN+ChydmZsa9NUPnJGZa1udaEQcU/b9QGaeXVf3GeA64IuZeUVN+UHAZuAw4K3AHGBmWX0A8IfASzVd9QBrgMXAycAxwF9m5rJW5ytJkl6v2ZWO+Zk5ITMnUPzRXhgR5+zFeTVyObAVeE9ETG5Q/2PgwogYX1P2YYqQBEBmfrzmPs4pyybU/CwHdgH/CPyfwO8FJ0mS1J6Wt1cycx3wFPCuzk+nsYgYC1wKfBb4EdDdoNlGYB3w0ZqyWUBvK2Nl5i8y85bM/Cfg1fZmLEmS6rUUOiJiTEScDpwAPLR3ptRQF/Bm4A5gKXBxRBzYoF0vRdCgXA05AVhV1SQlSdLgmg0dcyNiK8X5h7XAcuDhvTar39cN3JuZz1MEj0OADzVotxp4W0S8s7zmq8DvKpulJEkaVLOhY2FmTszM8cDRwDsoVhz2uog4Fjh3YLzM3AJ8g+LQ5+tk5k7gduBK4CJgSRVzlCRJe9bOmY5NwNdpvNKwN8yimOeSiOiLiD6KEHJmRExp0L6XIpA8lZnPVDRHSZK0B3v8ymy9iHgL8BFg/W6aHVh+XXXAq5k58JXbcXV1uzKz4RZIRLwB+BhwA/CFuurvUWyhfLK2MDM3RMSfAb/Y480MomZ+Y4A/KD/vLFdSJElSG5oNHfMiYk75/5eAB4BP7ab9s3Wf7wPeV/7//rq6ZygOfDbSBbwJ+Hxm/rK2IiI+DyyKiE/XX1R+82QoXq75/9Ly56+BzwyxX0mSRq0x/f39+3oO+5Wzr/j5iPqFfnfxMft6CpKkkW1Msw19DLokSapEy2c6Oi0izgC+NUj1osxcVOV8hurq89bT1dW1r6chSdKws89DR2Z+H5iwr+chSZL2LrdXJElSJQwdkiSpEoYOSZJUCUOHJEmqhKFDkiRVwoeDddhIeTiYDwWTJHWIDweTJEnDi6FDkiRVwtAhSZIqYeiQJEmVMHRIkqRKGDokSVIlWnrhW0SsAaYCO4BXgQ3Agsy8u6bNEuAy4MzMfLAsGw88AqzIzPk1bY8CngR6MnNlE+NfAtwGzK1/+2xErAVOB6Zn5j015acDa4GfZObbI+IZ4KiyehxF8PptTVeTgZnAXwBvA14G1gB/lZkb9zRHSZLUWDsrHfMzcwIwCbgTuCsiJgNExMHAhcALQM/ABZm5HZgBzImIU8q2YygCxKpmAkepu+x7VkQ0mvvTwKy6slll+cBcpmTmhPIeFgFrBj6XP5uBPwCuBN5MEUJ+B/x9k3OUJEkNtL29kpk7gcXAWODEsngm8AowG5geEZNq2j8KXA8sK1c+ZgPHA1c1M15EnEixynIRcDRwToNmK4FTI+LY8ppDgQ8CX2nx3hZm5j9n5m8z89fA54A/jYhDWulHkiT9m7ZDR0SMo1gN2AGsL4u7geUUf/y3ARfXXXYjsBlYASwAZmTmtiaH7AEey8x7gfvKseq9DHwNuLT8PAP4DvB8k2MM5j8Dz5UBRJIktaGd0DE3IrYCm4DzKc5QPBsRpwInA0szcwdwB3XBIDN3UaxUTANuzsx1zQxYrozMBJaWRbcCXRFxZIPmvcClETGWYmult9UbrBv7DOCvgU8MpR9Jkka7dkLHwsycmJlHZOZpmbm6LB9YiXi8/HwrMCUizqq9uDyM+SLwRAtjXgAcRLGKArCa4mzHZfUNy/H7gOuAw4BvtzDO65RzXwVclpn/0G4/kiSpQ1+ZLc86XACcEBF9EdEH3A/003gbpFU9FGdHni773ghMBC4f5EDpl4FrgSXl6krLIuLPKQ6Pfiwzv97etCVJ0oCWvjK7GzOBXcBJwPaa8g8At0TE4Zm5pZ2OI+Ik4N3A+4FHa6qOBH4InAt8q+6yZcBPgWxzzI9SbMtcmJn1fUuSpDZ0KnR0A72ZuaG2MCJuB+YBlwA3tdl3D/BwZn6zrrwvIu4p618XDDLzZYoDpO36HPBGYGVE1JZPLr9SK0mSWjSmv79/X89hv3L2FT8fEb/Q7y4+Zl9PQZK0fxjTbMNOrXSodPV56+nq6trX05AkadgZFqEjIuYB1wxSPS0zH6pyPpIkqfOGRego38cyf48NJUnSiOVbZiVJUiUMHZIkqRKGDkmSVAlDhyRJqoShQ5IkVcKHg3XYSHg4mA8GkyR1UNMPB3OlQ5IkVcLQIUmSKmHokCRJlTB0SJKkShg6JElSJVp690pErAGmAjuAV4ENwILMvLumzRLgMuDMzHywLBsPPAKsKN+zMtD2KOBJoCczVzYx/iXAbcDczFxUV7cWOB2Ynpn31JSfDqwFfpKZb4+IZ4CjyupxFMHrtzVdTc7MzRERwH8H3g38DliTmR/a0xwlSVJj7ax0zM/MCcAk4E7groiYDBARBwMXAi8APQMXZOZ2YAYwJyJOKduOoQgQq5oJHKXusu9ZEdFo7k8Ds+rKZpXlA3OZkpkTyntYRBEmJtT8bI6IdwL3A18DjgDeXLaVJEltant7JTN3AouBscCJZfFM4BVgNjA9IibVtH8UuB5YVq58zAaOB65qZryIOJFileUi4GjgnAbNVgKnRsSx5TWHAh8EvtLi7X0GWJ2ZvZm5PTN3ZGa22IckSarRduiIiHHAlRRbLevL4m5gOcUf/23AxXWX3QhsBlYAC4AZmbmtySF7gMcy817gvnKsei9TrE5cWn6eAXwHeL7JMQa8B/hlRDwYEb+KiB9ExHtb7EOSJNVoJ3TMjYitwCbgfIozFM9GxKnAycDSzNwB3EFdMMjMXRQrFdOAmzNzXTMDlisjM4GlZdGtQFdEHNmgeS9waUSMpdha6W3l5sptn0nA5cCngbcAfwd8IyKOa6UvSZL0b9oJHQszc2JmHpGZp2Xm6rJ8YCXi8fLzrcCUiDir9uLM3Ai8CDzRwpgXAAdRrKIArKY423FZfcNy/D7gOuAw4NstjEOJqCWEAAAgAElEQVRm9gO/Ae7JzLXl1sptFIdmG23pSJKkJnTkK7MRcQhFMDghIvoioo/iIGY/jbdBWtVDcXbk6bLvjcBE4PJBDpR+GbgWWFKurrTqcYq51xv271WRJGm4aukrs7sxE9gFnARsryn/AHBLRByemVva6TgiTqL42ur7gUdrqo4EfgicC3yr7rJlwE+Bdg9/Lga+FBFfAh6mOBtyHPCPbfYnSdKo16nQ0Q30ZuaG2sKIuB2YB1wC3NRm3z3Aw5n5zbryvoi4p6x/XejIzJcpDpC2JTPvjIgjgK9TrKg8Dbw/M3/Wbp+SJI12vtq+w3y1vSRplPHV9pIkaXjp1PbKkETEPOCaQaqnZeZDVc5nKK4+bz1dXV37ehqSJA07wyJ0lO9jmb/HhpIkacRye0WSJFXC0CFJkiph6JAkSZUwdEiSpEoYOiRJUiUMHZIkqRI+kbTDhusTSX0KqSRpL/GJpJIkaXgxdEiSpEoYOiRJUiUMHZIkqRKGDkmSVImWXvgWEWuAqcAO4FVgA7AgM++uabMEuAw4MzMfLMvGA48AK8qXuw20PQp4EujJzJVNjH8JcBswNzMX1dWtBU4HpmfmPTXlpwNrgZ9k5tsj4hngqLJ6HEXw+m1NV5Mzc3PN9XcDHwKmZua6Pc1RkiQ11s5Kx/zMnABMAu4E7oqIyQARcTBwIfAC0DNwQWZuB2YAcyLilLLtGIoAsaqZwFHqLvueFRGN5v40MKuubFZZPjCXKZk5obyHRcCagc/lT23guAA4tMm5SZKk3Wh7eyUzdwKLgbHAiWXxTOAVYDYwPSIm1bR/FLgeWFaufMwGjgeuama8iDiRYpXlIuBo4JwGzVYCp0bEseU1hwIfBL7S6v1FxB8BN1ATniRJUvvaDh0RMQ64kmKrZX1Z3A0sp/jjvw24uO6yG4HNwApgATAjM7c1OWQP8Fhm3gvcV45V72Xga8Cl5ecZwHeA55sco9Zi4G+An7VxrSRJqtNO6JgbEVuBTcD5FGcono2IU4GTgaWZuQO4g7pgkJm7KFYqpgE3N3tGolwZmQksLYtuBboi4sgGzXuBSyNiLMXWSm+rNxgRH6Y493FLq9dKkqTG2gkdCzNzYmYekZmnZebqsnxgJeLx8vOtwJSIOKv24szcCLwIPNHCmBcAB1GsogCspjjbcVl9w3L8PuA64DDg2y2MQ7kl9Hng8jIkSZKkDujIV2Yj4hCKYHBCRPRFRB9wP9BP422QVvVQnB15uux7IzARuHyQA6VfBq4FlrQRHP4D8BbgwYjYQhFgAP4hIha0NXtJktTaV2Z3YyawCzgJ2F5T/gHglog4PDO3tNNxRJwEvBt4P/BoTdWRwA+Bc4Fv1V22DPgpkG0M+X3grTWfxwLPAX8JPNhGf5Ikic6Fjm6gNzM31BZGxO3APOAS4KY2++4BHs7Mb9aV90XEPWX960JHZr5McYC0ZZn5CsV5FQAiYuB39L8z81/b6VOSJPlq+47z1faSpFHGV9tLkqThpVPbK0MSEfOAawapnpaZD1U5n6G4+rz1dHV17etpSJI07AyL0FG+j2X+HhtKkqQRy+0VSZJUCUOHJEmqhKFDkiRVwtAhSZIqYeiQJEmVMHRIkqRK+ETSDhsuTyT1CaSSpIr4RFJJkjS8GDokSVIlDB2SJKkShg5JklQJQ4ckSapESy98i4g1wFRgB/AqsAFYkJl317RZAlwGnJmZD5Zl44FHgBXly90G2h4FPAn0ZObKJsa/BLgNmJuZi+rq1gKnA9Mz856a8tOBtcBPMvPtEfEMcFRZPY4ieP22pqvJwEeBK4EjgJ3Aw8BfZeaP9jRHSZLUWDsrHfMzcwIwCbgTuCsiJgNExMHAhcALQM/ABZm5HZgBzImIU8q2YygCxKpmAkepu+x7VkQ0mvvTwKy6slll+cBcpmTmhPIeFgFrBj6XP5uB1cDUzDwUOBL4HnBvk3OUJEkNtL29kpk7gcXAWODEsngm8AowG5geEZNq2j8KXA8sK1c+ZgPHA1c1M15EnEixynIRcDRwToNmK4FTI+LY8ppDgQ8CX2nx3n6SmVvKj2OAXcDR5bwlSVIb2g4dETGOYgtiB7C+LO4GllP88d8GXFx32Y3AZmAFsACYkZnbmhyyB3gsM+8F7ivHqvcy8DXg0vLzDOA7wPNNjvGaiDgzIrZSbL38P8AN5YqNJElqQzuhY275x3gTcD7FGYpnI+JU4GRgaWbuAO6gLhhk5i6KlYppwM2Zua6ZAcsVhpnA0rLoVqArIo5s0LwXuDQixlJsrfS2eoPlXB/IzInAm4BPUpzrkCRJbWondCzMzImZeURmnpaZq8vygZWIx8vPtwJTIuKs2oszcyPwIvBEC2NeABxEsYoCxZmLFygOrL5OOX4fcB1wGPDtFsb5PZn5IvA/gK8OnF2RJEmt68hXZiPiEIpgcEJE9EVEH3A/0E/jbZBW9VCcHXm67HsjMBG4fJADpV8GrgWWlKsrQ3UAxTdd3taBviRJGpU69ZyOmRSHLd9FscUy8NNNcaD08HY7joiTgHdTbOXU9v0fgT8Gzm1w2TKKg6Y3tznmJ8qv81LOfTHwG+CH7fQnSZJafE7HbnQDvZm5obYwIm4H5gGXADe12XcP8HBmfrOuvC8i7inrv1VbkZkvUxwgbde7gXnlt1+2AT8A3puZvxpCn5IkjWq+2r7DfLW9JGmU8dX2kiRpeOnU9sqQRMQ84JpBqqdl5kNVzmcorj5vPV1dXft6GpIkDTvDInSU72OZv8eGkiRpxHJ7RZIkVcLQIUmSKmHokCRJlTB0SJKkShg6JElSJQwdkiSpEj6RtMP21hNJfcKoJGmY8omkkiRpeDF0SJKkShg6JElSJQwdkiSpEoYOSZJUiaZe+BYRa4CpwI6yqA+4OTP/tqx/Drg2M5dFxHHAT4GjM3NTg776gTMyc22rk42IY8q+H8jMs+vqPgNcB3wxM6+oKT8I2AwcBrwVmAPMLKsPAP4QeKmmqx7gX4Dv1ZU/kZmntTpnSZJUaOUts/MzcwFARPxH4P6IeCoz/3HvTK2hy4GtwHsiYnJm/riu/sfAhRHxqczcXpZ9mCIkHQaQmR8HPg4QEf8J+H5mTqjtJCLOAl6tL5ckSe1ra3slM9cBTwHv6ux0BhcRY4FLgc8CPwK6GzTbCKwDPlpTNgvo3esTlCRJu9Vy6IiIMRFxOnAC8FDnpzSoLuDNwB3AUuDiiDiwQbteiqBBREymmOeqNsYbGxEbI6IvIu6NiD9pc96SJInWQsfciNhKcc5hLbAceHivzKqxbuDezHyeIngcAnyoQbvVwNsi4p3lNV8FftfiWP8LOJniDMgJwBPAdyPi37U5d0mSRr1WQsfCzJyYmeOBo4F3UKw47HURcSxw7sB4mbkF+AbFoc/XycydwO3AlcBFwJJWx8vMvsxcn5k7M3NrZv434AXgvLZvQpKkUa7dMx2bgK/TeKVhb5hFMdcl5XZHH0UIOTMipjRo30sRSJ7KzGc6NIddtPB8eUmS9HqtfHvlNRHxFuAjwPrdNDuw/LrqgFczc+Art+Pq6nZlZsMtkIh4A/Ax4AbgC3XV36PYQvlkbWFmboiIPwN+scebaTzm2cDPgQ3AeOBTFOdJ7munP0mS1FromBcRc8r/vwQ8QPHHeDDP1n2+D3hf+f/76+qeoTg70UgX8Cbg85n5y9qKiPg8sCgiPl1/UWb+027mtid/AtwGHE5xr48C0zJz4xD6lCRpVPPV9h3mq+0lSaOMr7aXJEnDS1tnOjotIs4AvjVI9aLMXFTlfIbi6vPW09XVta+nIUnSsDMsQkdmfh/wkeOSJO3H3F6RJEmVMHRIkqRKGDokSVIlDB2SJKkShg5JklQJQ4ckSaqETyTtsHaeSOrTRiVJI5hPJJUkScOLoUOSJFXC0CFJkiph6JAkSZUwdEiSpEo09cK3iFgDTAV2lEV9wM2Z+bdl/XPAtZm5LCKOA34KHJ2Zmxr01Q+ckZlrW51sRBxT9v1AZp5dV/cZ4Drgi5l5RU35QcBm4DDgrcAcYGZZfQDwh8BLNV31ZObymus/ASwG5mXmglbnLEmSCq2sdMzPzAmZOYHij/bCiDhnL81rMJcDW4H3RMTkBvU/Bi6MiPE1ZR+mCEkAZObHa+7jnLJsQs1PbeA4Fvgk8OReuBdJkkaVtrZXMnMd8BTwrs5OZ3ARMRa4FPgs8COgu0GzjcA64KM1ZbOA3jaHvRWYC7zQ5vWSJKnUcuiIiDERcTpwAvBQ56c0qC7gzcAdwFLg4og4sEG7XoqgQbkacgKwqtXBIqIH2J6Zd7U9Y0mS9JpWQsfciNhKcf5hLbAceHivzKqxbuDezHyeIngcAnyoQbvVwNsi4p3lNV8FftfKQOXZkWuBTwxpxpIk6TWthI6FmTkxM8cDRwPvoFhx2OvKsxXnDoyXmVuAbwA99W0zcydwO3AlcBGwpI0hlwALMvNf2pyyJEmq0+6Zjk3A12m80rA3zKKY65KI6IuIPooQcmZETGnQvpcikDyVmc+0Md40YFFEbImILcDpwH+LiO+3OX9Jkka9pr4yWy8i3gJ8BFi/m2YHll9XHfBqZg585XZcXd2uzGy4BRIRbwA+BtwAfKGu+nsUWyifrC3MzA0R8WfAL/Z4M40dXfd5JfB94HNt9idJ0qjXSuiYFxFzyv+/BDwAfGo37Z+t+3wf8L7y//fX1T1DceCzkS7gTcDnM/OXtRUR8XmKFYlP11+Umf+0m7ntVv3zRSLiFeDX5XkSSZLUBl9t32G+2l6SNMr4antJkjS8tHWmo9Mi4gzgW4NUL8rMRVXOZyiuPm89XV1d+3oakiQNO8MidGTm94EJ+3oekiRp73F7RZIkVcLQIUmSKmHokCRJlTB0SJKkShg6JElSJXw4WIe1+nAwHwwmSRrhfDiYJEkaXgwdkiSpEoYOSZJUCUOHJEmqhKFDkiRVwtAhSZIq0dQL3yJiDTAV2FEW9QE3Z+bflvXPAddm5rKIOA74KXB0Zm5q0Fc/cEZmrm11shFxTNn3A5l5dl3dZ4DrgC9m5hU15QcBm4HDgLcCc4CZZfUBwB8CL9V01QP8HPgCcBwwFvgJsCAz72l1zpIkqdDKSsf8zJyQmRMo/mgvjIhz9tK8BnM5sBV4T0RMblD/Y+DCiBhfU/ZhipAEQGZ+vOY+zinLJtT8LAeeAf4LMAmYCPxXYFlE/Pu9cleSJI0CbW2vZOY64CngXZ2dzuAiYixwKfBZ4EdAd4NmG4F1wEdrymYBva2MlZm/zMyfZWY/xUNPdlH8rt7extQlSRJthI6IGBMRpwMnAA91fkqD6gLeDNwBLAUujogDG7TrpQgalKshJwCr2hkwIrYCrwDfB34A/GM7/UiSpNZCx9zyj/BLwFpgOfDwXplVY93AvZn5PEXwOAT4UIN2q4G3RcQ7y2u+CvyunQEzcyIwgWKr5ZvAznb6kSRJrYWOhZk5MTPHA0cD76BYcdjrIuJY4NyB8TJzC/ANikOfr5OZO4HbgSuBi4AlQxk7M1/JzL8HzqQ4UyJJktrQ7pmOTcDXabzSsDfMopjrkojoi4g+ihByZkRMadC+lyKQPJWZz3RoDm8A/o8O9SVJ0qjT1Fdm60XEW4CPAOt30+zA8uuqA17NzIGv3I6rq9uVmQ23QCLiDcDHgBsovsZa63sUWyifrC3MzA0R8WfAL/Z4M43HnE7xTZinKX5HfwmcDdzYTn+SJKm10DEvIuaU/38JeAD41G7aP1v3+T7gfeX/76+re4biwGcjXcCbgM9n5i9rKyLi88CiiPh0/UWZ+U+7mdueHEkRco6kOA/yDPAXmfntIfQpSdKoNqa/v39fz2G/cvYVP2/pF/rdxcfsralIklSFMc029DHokiSpEm2d6ei0iDgD+NYg1Ysyc1GV8xmKq89bT1dX176ehiRJw86wCB2Z+X2K52FIkqT9lNsrkiSpEoYOSZJUCUOHJEmqhKFDkiRVwtAhSZIq4cPBOmywh4P5EDBJ0n7Kh4NJkqThxdAhSZIqYeiQJEmVMHRIkqRKGDokSVIlDB2SJKkSTb3wLSLWAFOBHWVRH3BzZv5tWf8ccG1mLouI44CfAkdn5qYGffUDZ2Tm2lYnGxHHlH0/kJln19V9BrgO+GJmXlFTfhCwGTgMeCswB5hZVh8A/CHwUk1XPZm5PCKOAG4EPgD8AbAB+PPM3NzqvCVJUmsrHfMzc0JmTqD4o70wIs7ZS/MazOXAVuA9ETG5Qf2PgQsjYnxN2YcpQhIAmfnxmvs4pyybUPOzvAwq9wO/A6YAE4EZwG/2yl1JkjQKtLW9kpnrgKeAd3V2OoOLiLHApcBngR8B3Q2abQTWAR+tKZsF9LY43MUUQeOKzNySmbsy8//LzF+3PnNJkgRthI6IGBMRpwMnAA91fkqD6gLeDNwBLAUujogDG7TrpQgalKshJwCrWhzrPRSh6ksR8auI+F8R8X+1PXNJktRS6JgbEVspzj+sBZYDD++VWTXWDdybmc9TBI9DgA81aLcaeFtEvLO85qsU2yStOJxi62U9cCTFdtKnI2JGm3OXJGnUayV0LMzMiZk5HjgaeAfFisNeFxHHAucOjJeZW4BvAD31bTNzJ3A7cCVwEbCkjSG3Af+SmV/IzN9lZgLLgPPbugFJktT2mY5NwNdpvNKwN8yimOuSiOiLiD6KEHJmRExp0L6XIpA8lZnPtDHe40CjF7f5djxJktrU1Fdm60XEW4CPUGw/DObA8lsgA17NzIGv3I6rq9uVmQ23QCLiDcDHgBuAL9RVf49iC+WTtYWZuSEi/gz+//buPFyOol7j+PdHFiDEsJhEZcmCBFDZhMIrsgVQFiGKCrIICFFIBB4FhQtGwFzBgIg3irIGSNgXvRcUEIKsAs8VbikXQYSYQBIIxCSQIAEUAnX/qBpSp5mZMzOZ0+eQ836e5zxnuqu7urqrqru6uqaHFzrdmeqmASc5544BLiQOmP0KcGyL8YmIiPR6zTQ6TnXOnZw+vwrcB5xQZ/mZhenpwJ7p812FsKeIAz6rGQOsA0z23i/IA5xzk4FJzrkJxZW89w/WSVtd3vs5zrnPApOBs4nv+Zjovb++1ThFRER6OwtBTwzaadej51Y9oHefP6zspIiIiJTBGl1Qr0EXERGRUrQ0pqPdnHM7ArfVCJ7kvZ9UZnpWxPF7PcqYMWO6OxkiIiI9To9odHjv7wcGdnc6REREpOvo8YqIiIiUQo0OERERKYUaHSIiIlIKNTpERESkFGp0iIiISCn0crA2q7wcTC8DExGRXkIvBxMREZGeRY0OERERKYUaHSIiIlIKNTpERESkFGp0iIiISCnU6BAREZFSNPSDb865e4HtgDfTrPnAL7z3P03hs4FTvPdXOedGAM8AG3jvn6sSVwB29N4/0GxinXPDUtz3ee93LYRNBL4PXOC9PzqbvxrwPLA2MBI4GTgkBa8CrA68mkU1DlgMnABsAfQBHgcmpB+mExERkRY009Nxuvd+oPd+IPGi/UPn3O5dlK5avg4sAXZxzm1cJXwGcKBzbkA2bz9iIwkA7/34bD92T/MGZn9XExsoPwc2AoYA1wC3Oec26JK9EhER6QVaerzivf8D8ASwWXuTU5tzrg8wFjiT2PNwVJXFngX+AHw5m3ckMKWZbXnvr/be3+i9X+K9X+a9vwB4HXAtJV5ERESab3Q458w5tz2wKfA/7U9STWOADwBXApcBX3XOrVpluSnEhgapN2RT4NcrsmHn3BbA+4mNHREREWlBM42O7znnlhDHPzwAXA083CWpqu4o4Fbv/d+JDY9BwBerLHczsKFz7mNpnSuAN1rdqHNuKPAr4Gzv/d9ajUdERKS3a6bR8UPv/Vre+wHABsBHiT0OXc45NxzYo7I97/0i4DfEQZ8deO+XAdOAY4DDgEtWYLvrAvcAdwDfbTUeERERaX1Mx3PADVTvaegKRxLTeolzbr5zbj6xEbKzc26TKstPITZInvDeP9XKBtO3cO4HbvPeH+u91y/jiYiIrICGvjJb5Jz7ILA/8GidxVZNX1eteMt7X/nKbf9C2Nve+6qPQJxzfYEjgLOAnxWC7yE+QvlOPtN7/7RzbifghU53pvo2NwXuBKZ5709pJQ4RERHpqJlGx6nOuZPT51eB+4jvsqhlZmF6OrBn+nxXIewp4oDPasYA6wCTvfcL8gDn3GRgknNuQnEl7/2DddLWmZOA9YDjnHPHZfPHpa/UioiISJMsBD01aKddj54bAO4+f1h3J0VERKQM1uiCeg26iIiIlKKlMR3t5pzbEbitRvAk7/2kMtOzIo7f61HGjBnT3ckQERHpcXpEoyP9psnA7k6HiIiIdB09XhEREZFSqNEhIiIipVCjQ0REREqhRoeIiIiUQo0OERERKYUaHSIiIlIKNTpERESkFGp0iIiISCnU6BAREZFSqNEhIiIipVCjQ0REREqhRoeIiIiUQo0OERERKYUaHSIiIlIKCyF0dxpWKquuuurjb7zxxj+7Ox29Xd++fQcvW7ZsUXeno7dTPvQcyoueYSXNh0UhhD0bWbBvV6ekt9l8883/6b133Z2O3s4555UP3U/50HMoL3qG3p4PerwiIiIipVCjQ0REREqhRkf7XdzdCRBA+dBTKB96DuVFz9Cr80EDSUVERKQU6ukQERGRUqjRISIiIqXQV2Zb4JzbGLgceD/wInCY9/5vhWX6AOcCewIBOMt7f0nZaV2ZNZgPuwOTgM2Bn3vvTyg9oSu5BvPhVOBAYFn6m+C9n152Wld2DebFEcDxwNtAH2CK9/7cstO6MmskH7JlNwEeAc7vDecn9XS05kLgPO/9xsB5wEVVlvkKsBEwCtgOmOicG1FaCnuHRvLhaeBI4MdlJqyXaSQfHga29d5vCYwFrnfOrV5iGnuLRvLiv4AtvfdbAZ8CvuOc26LENPYGjeRD5eb0IuCmEtPWrdToaJJzbiiwNXBtmnUtsLVzbkhh0QOIdxBve+8XEgvV/uWldOXWaD5472d67x8h3l1LmzWRD9O996+lyT8DRrwLlDZpIi/+4b2vfINgANCP2BsrbdDENQLgZOAWYEZJyet2anQ0bwNgnvf+LYD0//k0PzcMmJNNz62yjLSu0XyQrtVKPhwGzPLeP1dC+nqThvPCOfc559xfiOeoH3vvHys1pSu3hvIh9S7tAUwuPYXdSI0OESmNc25n4HTgoO5OS2/mvf+N9/5jwMbAoWlcgZTEOdcPmAKMrzROegs1Opr3LLBeehZXeSa3bpqfmwsMz6aHVVlGWtdoPkjXajgfnHPbAVcB+3rvnyo1lb1D03XCez+XON5mn1JS2Ds0kg8fAj4M/NY5Nxs4DjjSObfSvzhMjY4mee8XAP/H8ju1g4BH0riN3C+JhWiV9CxvX+IALmmDJvJBulCj+eCc2xa4HtjPe/+nclPZOzSRF5tmnwcDuwB6vNImjeSD936u936w936E934E8FPiGMCjSk9wyfSV2daMBy53zp0GLCY+o8Y591vgNO+9B64E/g2ofE3qB977p7sjsSuxTvPBObcDcB0wCDDn3IHA1/R1zbZqpD6cD6wOXOTcOz+weajGErRdI3kxLn2V/E3igN5feO/v6K4Er6QayYdeSa9BFxERkVLo8YqIiIiUQo0OERERKYUaHSIiIlIKNTpERESkFGp0iIiISCnU6BDMbA8zuz+bHm1ms7sxSaUxs2lm1rZf/zWzEWYWsukhZjbHzAY3sO54M7uyXWl5LzCzHc1sSXenozcys0OaqeftritSX1fVjRby/Udmdnq7tq9GRy9nZkZ89//3O1nuG2b2uJn9w8wWm5k3swOy8NlmdkiV9d4136IZKa6BhbDRZhbMbGn6e97MpprZOiu2p90jhLAQuIbOj+8awA+AiSUkq8cIIdwfQliru9NRi5lNNLM7uzsdvUFXHWszu9fMTml3vF2tWDe6sSyeBRxjZuu1IzI1OmR3oD9wT60FzOwg4kXza8CaxFf6Hk986U0rdgE2BN6m+m9wvBVCGBhCGAjsAGxHfGPfe9VlwBFmNqjOMocAj4UQZpWUpg7MrI+Z6XwgIh2EEBYDtwHj2hGfTjIlSnf9p5jZPeku/jEz28LMDjKzmWb2spldYmZ9s3WGmdmvzOyF9Hexmb0vC59kZk+n+GaZ2XFZ2IjUa3ComT1hZq+Y2R1m9qEsWfsCd4b6b4n7FPD7EMJDIXo9tcJbfYvhOOB24ltb6xbkEMLTxJ9+/ngxzMz6pmPy+cL8y83ssvR5NzN7KPXOLDSz68xsaK3tpeO1QzY92syWFbY5IfXULDGzB81sm0724W/AIuDTdRbbF/hdIS3fMrMnU77NNbMzzaxPCjvHzG4sLL9LWnaNNL2ZmU03s0XZ+v1SWKVsfM3MngBeA4aa2YFm9mjqhXrBzC6qxJfW+6CZ3ZzK6oy0fjCzEdkyR6ZesZfN7BEz273WTlc5vtPM7Eozuywd33mpfmxlZv+b9u8eM1s3W2e2mZ1mZg+keuDNbNssvG4ZMLN+KU+fSvHPMrMvWezJmwCMtuU9bxvW2I+d0zZeTnk2LgsbbWbLzOyAFPfLZnZDXo+rxNfKuWILM7s77efTaf0+Wfgn0rFZamYPEBv++TYHpHL1jJm9ZGa3m9lGtdJYJc3vN7MrUrmZb7EerpOFd+j1zMrg+rWOtZkdnvb3pBTvAjP7SZVyvH4W7+FmNjN9/gWwI3BqirPqb/5Y7EW4y+KjhIVm9qKZfdvMhqdj+oqZ/dHMPpKts0J1JSvrU7Ky/q5ykz7XPT6FfenwGKxN+f474jlqxYUQ9FfSHzCb+Fr0jwD9iD9+NQu4GFiD+KNwC4CD0/KrATOJ3e6rA2sDvwUuy+I8hNjzYMCuwOvAHilsBBCIF+3BxFeBPwhMydZ/CPhmIZ2jgdnZ9P7AP4EzgN2AtWrs2yGdzQeGAP8Cvghslag1uc4AAAoCSURBVNK3TWHby7LpjYCn8n0uxH82cFM2PRBYCuyYpncAtiW+8v+DwO+Ba7PlpwGXZNMB2KFOeialY7Yh0IfY+7MIWDs/5lXSeTNwRp2y8Xfgc4V5XwJGprz9eFpmXAr7KPAGMCRb/nLg0vR5KPAisVHXH1gP8MBphbJxVzou/dP+7AV8jHhDshHwBHBmto27iL8hNCht494Uz4gUfhSxzG6Z4vhsyo+Naux38fhOI5bhvdP649P6vwHWBwYAdwMXF8rY88A2aT9OBhYCgxosAz9K+7lFOtbrA1uksInERnm9ej0ypfmItI1PAi8B+2f7GIBLieXzA8TzwPfaeK5YM5WPU4FV03pPAydm4S+mY9M/HY/5dKzn1xDPFR9Iy/wH8CTQr1pdqZLm24nlfO30dytwa51zwYh0XNavdayBw4mvaz+PeA78MDAD+G61OLJ1ZmbT9wKndJKHE9N2vs7yevAWcGchD+7I1lnRujKNWG4+l+L4YkrD8Bp1o9bxmVmY904+tSPf0zLbEHum+9c7jo38lXrR7e1/qdKdmE1/NhXC/MJxAzA5fd4PmFWIYxviRbtPjW38Cjg7fa5UyG2z8GOAR7LpGcDhhThG54UyzdsH+G/iie0t4uOYzQr79iqwpPD3Nh1PNP9OPFlWTmR/Ai4qbDukdRcDzwAXUqWhk5b/CPHiOzRNjwVm1MmDfYAF2fQ7FTRN12x0EC9IrwA7FeJ8rLKP1G50XA2cXyddbwCjOyk/5wA3ZNMPAcenz+8jXpy3T9MnAHcX1v8S6QSVlY2dOtnmscDD6fP6aZ0Ns/Dd6HgifRw4rBDHzdQ46VO90ZFfqAak+PfP5h1NxzI8Gzg9mzbirzwf3FkZSMsuBfausexEOm90TAAeLMw7E5heKNN5Pf8xcGOdOGfT3LniYOKvmFoWPg54Kn3+SjomefgPSfWceFMSgGFZ+CrAy6T6QJ1GB/HGJwCjsnmbpHkfyvaplUbHv4AB2byvk+p4MY5snVYaHX8pzFtQJQ8Wt7GuTCMr62neQuDzNepGreNTr9Gxwvme5o1Kyw2tdxwb+dMPvpXvhezza8TxCwsL8yrdriOBYfbuEcyBeMc2z8y+CRxJLORGvBu4ps42X83ih3hhrzfWIG4whFuIrWHMbFPiD3jdYmYjQyqVxLvwq/L1LBslbWaW0npVCOHNNPtS4Cwz+04IYWma91ZocHBhCOGvZvYnYo/PfxLvNqdm29yG2DuxJfECZsS7zVYMTuvebNk3VIh3QetXX+Udg4gNqFrelQ8Wx9J8m9ir0pd4F/KHbJGpxAvwZODLwLwQwoMpbCSwfaHsGPEuLje7sM3PAKcBmxLvmPsQT74Qe0sgnsQq5hTiGwmcZ2bnZvP6As/RuHfKawjhtVhs3lVvio8mZmfrBDObS8qTTsrAEGLPwYwm0le0AbFXITcLyB/7Fet5sR5W08y5YgPihSQvl7PSfIjHYk4hPC+PI9P/P6fjXdEvi6OeyjJ5nLOysBdo3YIQwmvZ9Gw6r2+tKKbxNeqUuzbUlWrbbKRcNKNd+T6I5TeDK0RjOnq2OcQW/VqFv9VCCPPMbHti1/A4YHC6UN9MPKk26hFiV33DQghPEi90w4ndqI3ajdgNOTY9851P7MobSLxTa9VU4PD0HPKTwBVZ2HXE3pSNQwiDqD5wNfcq8SJUsW72eVEK/3QhP9YIIZzVSbybEY91LR3ywcw2IHbnnkG8U1yT2MWc5+11wCgz25p4xzM1C5tDvCvK07lmiINzc29n2+wP3JTiHZaO10nZNuel/8Oy9fPPle2OLWx3YAjhG3X2vR1GVD6kxu0wljd06pWBhcQ8HVUj3rdrzM89y/KTd8WGaX5ZngWGW8crR56GeVXC8zRXLoijCnk3IIRwbYPbhywfWD52oBK2lNp1C2of66FmNiCbHsHyvK3cqLQSb8vaVFeaVW0/iscUOu5/u/J9M2JP0ButJr5CjY6e7RagMsjtfRatZ2ZfSOGDiI86FgLBzPYmPmdsxk3ExkBNZjbWzPa39K6JNGhrPPBECOGlJrZ1FPF5+qbE8RxbEQvzVFZsZPR1xMbMucDvQgjzsrBBxK7CV8xsGPHZZj0e+KqZ9U8Dvr5dCUh3Cz8DzjGzUQBmNtDie06KJ7p3pMbQEOLz4VpuouNA04HE+rkQeNPMPgkcmq8QQlgC3EhsmBQbW1cALuXdama2Shp4tmedNPQnjiNaHEJ43cw+SuwyrmzvOWJX9VmpPA4Fil9FnAxMtDjw08xsdTPbIfWOdaWxZra1xQGGJxJ7NG5NYTXLQMrTC4CzLQ68rdSxzdMi84m9jf3rbPtaYBszO8ziQONPEMvzpW3dw/puJebdhFR2NyFeBCtpuIVYpk60OHB2a+KjSABCCAuIPaTnW/pqpJmtZWZfsMLX2qsJITwP3AH8JK23NvAT4LYQQuVu3gMHpTozhDj+JFfrWK9CLHOrWxzIewJx/BIhhEWkhq7Fb2BtTuxNLcbb8IDYBrWjrjSr2vF5hNgo2yfV8S8AO2Xh7cr3zxDPUStMjY4eLHUp7ka8A36SeOK8i3ixBphO/AbIw8S78P2IF6FmTAeWmdnoOsssJnbj/9XMXiWOJVhCfDbekFTp9gXOCSHMz/+IvTUfNzPXZNoBCCG8TNzvvYhfT80dRXwG/ApxTMovO4nuWOIJ6iXiM/NphfDvA78Gfm1m/yAO9htP/bo0FpiW0lnLlcCW6aRKCOGv2baWEC+U1e44pxL3e3o68ZPWn0/8avK+xO7oxcRjVPXbF2mdpcA3iBfgpcSeleKjuoOJF/TngAdYfjz/leKYQhzcOzVtcy7x4tKvzr63w8XERudi4ADiGI3K8e6sDHyPmNc3pWXuY3nPxy+Jd+rzLX7DoNijQQjhGeLz/mOJg/auJA7YvaFte9eJtK+7ExuufyfW6yuIjxwrDdS9icdmMfFYXVCI5kjioO17zewV4lil/Ynd6o04hHj8nkx/S4DDsvBTiDdJLxAvyNcV1q91rOcQ79ifIZ57bieWsYqvEs9FL6f9LTb2JhMb4EvM7C8N7ktd7agrLXjX8QnxK/bfIpb/l4A9iYNXK+lc4Xw3s7WI5fvCFtPdgXV81CO9Ubr7nRBC2ClNjyZeJEd0Z7rei1LvyDMhBEvTg4E/Aq7wPL7auuOJA0EPrbdcT2JmexAbRquHbjqZWBw3dEpxPJG895nZ4cS8bXdPRel6Ql1phZmdSRxP1JYXrGkgqRBCuJ149yBtlrp/hze47IW06W6iq5jZlsQ7oMeIz4bPAK5/L51ERcqwstSVEMJ32xmfHq9INbN5b78BtDstIQ6OXVmtQ3xEsZTYZfxnYveuiHSkulKFHq+IiIhIKdTTISIiIqVQo0NERERKoUaHiIiIlEKNDhERESmFGh0iIiJSiv8HSRCg5IU2UiUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x626.4 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "contributions = model.predict_contributions(test)\n",
    "contributions_matrix = contributions.as_data_frame().as_matrix()\n",
    "shap_values = contributions_matrix[:,:-1]\n",
    "shap.summary_plot(shap_values, X, plot_type='bar', color='royalblue')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Bind model predictions to test data for further calculations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gbm prediction progress: |████████████████████████████████████████████████| 100%\n"
     ]
    }
   ],
   "source": [
    "# cbind predictions to training frame\n",
    "# give them a nice name\n",
    "yhat = 'p_DEFAULT_NEXT_MONTH'\n",
    "preds1 = test['ID'].cbind(model.predict(test).drop(['predict', 'p0']))\n",
    "preds1.columns = ['ID', yhat]\n",
    "test_yhat = test.cbind(preds1[yhat]).as_data_frame()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Select a Probability Cutoff by Maximizing F1 Statistic"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Function to calculate AUC-PR Curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_prroc(frame, y, yhat, pos=1, neg=0, res=0.01):\n",
    "    \n",
    "    \"\"\" Calculates precision, recall, and f1 for a pandas dataframe of y and yhat values.\n",
    "    \n",
    "    Args:\n",
    "        frame: Pandas dataframe of actual (y) and predicted (yhat) values.\n",
    "        y: Name of actual value column.\n",
    "        yhat: Name of predicted value column.\n",
    "        pos: Primary target value, default 1.\n",
    "        neg: Secondary target value, default 0.\n",
    "        res: Resolution by which to loop through cutoffs, default 0.01.\n",
    "    \n",
    "    Returns:\n",
    "        Pandas dataframe of precision, recall, and f1 values. \n",
    "    \"\"\"\n",
    "    \n",
    "    frame_ = frame.copy(deep=True) # don't destroy original data\n",
    "    dname = 'd_' + str(y) # column for predicted decisions\n",
    "    eps = 1e-20 # for safe numerical operations\n",
    "    \n",
    "    # init p-r roc frame\n",
    "    prroc_frame = pd.DataFrame(columns=['cutoff', 'recall', 'precision', 'f1'])\n",
    "    \n",
    "    # loop through cutoffs to create p-r roc frame\n",
    "    for cutoff in np.arange(0, 1 + res, res):\n",
    "\n",
    "        # binarize decision to create confusion matrix values\n",
    "        frame_[dname] = np.where(frame_[yhat] > cutoff , 1, 0)\n",
    "        \n",
    "        # calculate confusion matrix values\n",
    "        tp = frame_[(frame_[dname] == pos) & (frame_[y] == pos)].shape[0]\n",
    "        fp = frame_[(frame_[dname] == pos) & (frame_[y] == neg)].shape[0]\n",
    "        tn = frame_[(frame_[dname] == neg) & (frame_[y] == neg)].shape[0]\n",
    "        fn = frame_[(frame_[dname] == neg) & (frame_[y] == pos)].shape[0]\n",
    "        \n",
    "        # calculate precision, recall, and f1\n",
    "        recall = (tp + eps)/((tp + fn) + eps)\n",
    "        precision = (tp + eps)/((tp + fp) + eps)\n",
    "        f1 = 2/((1/(recall + eps)) + (1/(precision + eps)))\n",
    "        \n",
    "        # add new values to frame\n",
    "        prroc_frame = prroc_frame.append({'cutoff': cutoff,\n",
    "                                          'recall': recall,\n",
    "                                          'precision': precision,\n",
    "                                          'f1': f1}, \n",
    "                                          ignore_index=True)\n",
    "    \n",
    "    # housekeeping\n",
    "    del frame_\n",
    "    \n",
    "    return prroc_frame\n",
    "        \n",
    "prroc_frame = get_prroc(test_yhat, 'DEFAULT_NEXT_MONTH', 'p_DEFAULT_NEXT_MONTH')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Select best cutoff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.26\n"
     ]
    }
   ],
   "source": [
    "best_cut = prroc_frame.loc[prroc_frame['f1'].idxmax(), 'cutoff'] # Find cutoff w/ max F1\n",
    "print('%.2f' % best_cut)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Plot AUC-PR Curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XuYXFWZ7/HvrzqdDrlANEG5JBCYBDTkBOT0EBQvGcGZyIHEeUDlJuqgjDowHhUBRZTL6FGYGRUFHRQUEQSEQcLFYThABJFkaARiEmCmJwjpBLlEiOTW13f+2LuSSnX1rupOV3VX9+/zPP107V1r71q106m31nrXWlsRgZmZWV9yQ10BMzMb3hwozMwskwOFmZllcqAwM7NMDhRmZpbJgcLMzDI5UJiZWSYHCitJ0u8lbZG0UdILkn4kaWIfZT8iqTst+ydJT0g6psz5d5X0LUnPpce1pttTq/OOBq7oWrwi6U5J0wfpvEdlPD9fUk/6uvmf29Pn5ki6W9LLkqo2GUrSIZIelbQ5/X1IH+WaJF0l6VlJr0l6TNJ7i8qMl3RFWucNkh6oVr1tcDlQWJZjI2IicCjw58CXMso+nJadDFwB3CBpcqmCksYC9wIHAQuAXYG3AeuBw/pbSUlj+nvMAOSvxZ7AC8B3avCaAOsiYmLBz7Hp/k7gJuC0ar1w+u90G/BT4HXANcBt6f5iY4A1wLuA3YDzgZskzSgocyXweuDN6e/PVKvuNrgcKKysiFgL/BKYU0HZHuBaYAIwq49ipwL7AH8dEasioiciXoyIiyPiLgBJIWlm/gBJP5b0D+nj+ZLaJJ0j6Q/AjyQ9WdiKkTQm/eZ6aLp9uKTfSHo1bfHMH+C12ArcDMwueK0mSf+Yto5ekPR9Sbukz02VdEf6un+U9KCknKRr02twe9pSOLuf9Xg6Iq4CVg7kfVRoPkkA+FZEtEfEZYCAd5eoz6aIuCAifp/+e94BPAP8bwBJBwILgdMj4qWI6I6IR6tYdxtEDhRWVtrNcjTwWAVlG4CPknzjfbaPYkcB/xYRG3eiWnuQfCvdFzgd+BlwYsHzfwW8HBG/lbQ3cCfwD+kxZwG3SNo9rfO5ku6o5EUljQc+CCwt2P0N4ADgEGAmsDfw5fS5zwFtwO7AG4EvAhERHwKeI22pRMQl/Xv7/SNpeRqsSv1c0cdhBwHLY8d1fpan+8u93htJrkk+kM0j+Xu4MA3gv5N03E68JauhWjTZrX79QlIXsIHkg/ZrGWUPl/QqSUuiCzglIl7so+wUYGe/TfYAX4mIdgBJ1wOPSRofEZuBk4Dr07KnAHflWyvAPZJaSILfNRHx9QpeL38tJgIvkgQiJAn4ODA3Iv6Y7vta+tpfIAmYewL7RkQr8GA/3+de6XXNOz0iburnOYiIuf09huS9bijatwGYlHWQpEbgOpJr+1S6expJi/QWYC/grcCdklZFxJMDqJvVkFsUluV9ETE5IvaNiE9FxBZJ7yhIrBZ2eyyNiMkkfdmLgXdknHc9yYfnzngp7QYCIP0QfhI4Nv3Wv5DtgWJf4P2F36KBt/ezDu9L318TcAbwK0l7kLQUxgOPFpz739L9AJcCrcC/S1ot6dx+vs916b9B/qffQWInbCTJHxXaFXitrwMk5Ui6HjtIrlPeFpKg+Q8R0RERvwLuB/5yUGtsVeFAYf0SEQ8WJFZ7dUGk3UmfAj4k6S19nOb/A38laULGS20m+QDO26P4pUock+9+WgSsSoMHJEnWa4s+cCdU2JLY8UWTvvV/BbpJgs3LJB+CBxWce7c08U1EvBYRn4uI/YFjgc9KOjLjPVSFpJVFo6cKf77fx2ErgblpqylvLn3kRdJyV5F0sR0XEZ0FTy8fjPdhQ8OBwgZdRKwHfsj2fvpi15J8eN8i6U1pcneKpC9KOjot8zhwkqQGSQtIRtOUcwPJN9RPsr01AcmonWMl/VV6vnFpQnxaf9+bEotIWk5Ppsn7HwDflPSGtMzekvJdU8dImpl+iP6JJMB0p6d7Adi/v3UoqMc4YGy6PU5SU1/lI+KgotFThT+f6OOwJWld/z5N2OdbCPf1Uf57JCOajo2ILUXPPUCSk/lCOtDgCJJk+d0VvF0bYg4UVi3fAo6W1KtvPM0rHAU8BdxD8gH6H8BUYFla7NMk38BfBU4GflHuBSPieeBhkqG2NxbsX0PSyvgi8BJJkPo86d9/GqB+Web0t0vamNb1q8CHIyL/zfocku6lpZL+RNJiOjB9bla6vTGt2xURsSR97v8BX0q7rM4q9/6K7EvSksnXYQvwdD/PkSkiOoD3kYxSexX4G5IuuA7Y8bpJ2hf4W5KE/h8KWisnp+fqJPk3OJokz/ED4NSCHIYNY/KNi8zMLItbFGZmlsmBwszMMjlQmJlZJgcKMzPLVHczs6dOnRozZswY6mqMPE+nA2YOPDC7nJnVpUcfffTliNi9fMne6i5QzJgxg5aWlqGuxsgzf37ye8mSoayFmVWJpL7WXivLXU9mZpbJgcLMzDI5UJiZWSYHCjMzy+RAYWZmmRwozMwskwOFmZllqlqgkHS1pBclrejjeUm6TFJrej/fQys571N/+BNfvb3kKc3MrAqq2aL4MbAg4/n3kqzVPws4neSmJ2V1dgc/eOhZ9j/3zp2uoJmZlVe1QBERDwB/zCiyCPhJJJYCkyVVfA/jHnDLwsysBoYyR7E3yZ3G8trSfb1IOl1Si6SW7s0btu2/Y8UfqltDMzMb0kChEvtK3m4vIq6MiOaIaG4Yv9u2/cfM2aNadTMzs9RQBoo2YHrB9jRgXaUH54Dzjp0z2HUyM7MiQxkoFgOnpqOfDgc2RMTz5Q5qbBAfP2JfVn/9/1S/hmZmVr1lxiX9DJgPTJXUBnwFaASIiO8DdwFHA63AZuCjlZz3TXvs6paEmVkNVS1QRMSJZZ4P4O+q9fpmZjY4PDPbzMwyOVCYmVkmBwozM8vkQGFmZpkcKMzMLJMDhZmZZXKgMDOzTA4UZmaWyYHCzMwyOVCYmVkmBwozM8vkQGFmZpkcKMzMLJMDhZmZZXKgMDOzTA4UZmaWyYHCzMwyOVCYmVkmBwozM8vkQGFmZpkcKMzMLJMDhZmZZXKgMDOzTA4UZmaWyYHCzMwyOVCYmVkmBwozM8vkQGFmZpkcKMzMLJMDhZmZZapqoJC0QNLTklolnVvi+X0k3S/pMUnLJR1dzfqYmVn/VS1QSGoALgfeC8wGTpQ0u6jYl4CbIuItwAnAFdWqj5mZDUw1WxSHAa0RsToiOoAbgEVFZQLYNX28G7CuivUxM7MBqGag2BtYU7Ddlu4rdAFwiqQ24C7gzFInknS6pBZJLS+99FI16mpmZn2oZqBQiX1RtH0i8OOImAYcDVwrqVedIuLKiGiOiObdd9+9ClU1M7O+jKniuduA6QXb0+jdtXQasAAgIh6WNA6YCrxYxXpV3fqN7bS9soVpr9uFKRObtm1PGNvApo7ubfvNzOpBNQPFI8AsSfsBa0mS1ScVlXkOOBL4saQ3A+OAfvctFX8wD4X1G9tZuW4Dv/nv9fzooWcY29BAZ08PH2iexk0tbQBs7eyhqUEoJy45bi4LDynuiTMzG36qFigiokvSGcDdQANwdUSslHQR0BIRi4HPAT+Q9BmSbqmPRERx91Sm2x5fyzm3LGdMTnR0B185djYnz9t3UN5DqQBUqnXw69aXOevnT9DZvb3q7V1dAPzk4ed2OGd7d0B3cNbNy5m9567MfOOkYRHozMz6on5+Lg+55ubmaGlpAZIP7SO+cR9bO3t2KPPJd+3Px96x/0596OYDUGMuR2dPD5ccN5cAzrllObC9dYBEV3cP3QO4jGMbxAmHTeemljYaczk6unv4myNm8NY/m8pBe+1a26Axf37ye8mS2r2mmdWMpEcjonlAx9ZzoHhizauc/MOlbGzv7lVubIM4892zOGnePv3+wC0VgJrG5ICgvas212tMDi5cNIc5e+1Wm5aGA4XZiLYzgaIul/BofeE1bm5ZQ2dXN+1dPSXLdHQH/3TPf/K2r9/L4sfX9uv8ba9soTG346VpyImG3gOyMjWNEae+dR/GNeYY19i/Y7t64LxbV3DSD5ZyxDfu6/d7MDMbLNVMZlfFule3cNQ3H6i4fHtXcPYtyzli5tSKv5VPe90udPbsGIC6e4Leo3sTjQ0iIsjHrAbB/z3qgG2tmU8feQBtr2yhs6ubk364jI5+9FNt6khaS2ffkuQ0PGrKzGqt7gLF+k0d7NnPYxpzOdpe2VLxh+uUiU1cctxczi7KUUDygQ29RzAdMXMqK9dtANQrvzBlYtO27X98/8E7nPcDzdO48ZG2PltGhY6+7EGaxiSjqc4/ZnbtuqXMbFSruxxF056zYs8Pf6tfx4xrzPHQOe8eUK6iklFPO3ve9RvbuX7Zc3z73v+kgnixzYSxDXRHbBtqu1Ojp5yjMBvRdiZHUXctinLGj83x14dM4+bfrtk2l+GS4+YO6Ft3YUsga9/OnnfKxCbOPDJJvOdbJWv+uJmL71xFYy5He3cPikiG1hYo7JZ6bWvXtvL595yfp1EYQICSwa+pvYumMTka6Xteiofxmo1Oddei2GvmQTH2+Ev6fD7feoDeH4j1prD1csx3f91rGHDehKYGOrt6dsh95K/Dr1tf3jbMd2tXNxHBLo1jeg35vfbac4iAOy67bttw3cKAUzxc2F1fZvVlVLUo9pq8Czd85p08vuZVDpk+mWXP/JELb19JY0NuWzdM/oOr3j/AClse+ZxJQ05sKhoO3NkdNDbk6Ojevr8xl2Plug2cc8tytnb2sJXtQea19mQy4OdvXk5+yG+SrN8+QTBfPp9ELz7Pebeu6NX1ZWYjU90FCoCZb5zEzDdO2vZ4wZw96r71UM7CQ/bmiJlTaXtlCyvWbtihm+n8Y2Zz8R2rdiifjNpS0pKgdEukIScIAb3noeQ15nI8vubVkucp7Po6YuZUoP5bcWbWW10GiuK+8sHIG9SD/Ps8ePrkXsFxUtOYXqO0Dtpr117DfAtlDfnN6+zp4ZDpkzPP05jLcd2y57hiSeu2GeZn/MXMAU12NLPhp+5yFDNnHxxjjvt6yaTtaFcq2bz48bXbAkipHAUkLYJyOYr8eUp1ffU1a71pjPjysQc5l2E2DIyqJTzG7TUr9jh1+/DYgQ59HU0qGvX0niOTUU8PPlB21FNx19ffzZ/JlQ+s3pb7KDZhbI7OHrYt2FiYpF+3YQul5p6Y2eAaVclsFd0Pqb+T6UajUsNxi5+naUyf5Yv3F3d9AVy+pLXP19/UsT0B/sSaV1j8xPMAO4ziahBc9L45g7byr5kNnrpb6ymK+tQ7e3q2fVhZ7eQDRj54XHLc3LQLKttNLWuT0VNFQ327Iwkk3/z3p3lizaus39heraqbWT/VXYti2uTxjGnM0ZATnd3B+cfMdmtiGMiPyrp+2XN89/7/YkxDrlcuoxLfvq+V7y1pRbnc0C25bmY7qLscRXNzc3zmu7f0mjvhhPZOGsQlPPI5iGXPrOdrdz210+drEJyz4E3M23+Kk+JmAzSqchRdPcHFd66iozu2TTDr7+qwVl2FuYwJY8fsENSLbw1bie6Ar/3yKcbmINeQ8xcDsxqru0CxtaObnBPadePkw/ftNecjv+z6hLEN/HTZs/z4N89WdK6OHqCnh8/f7C8GZrVUd4Hi9+s3sUdn0RIWTmgPa6VGXeW3L1g4h1lvmMSFt69EoqI7CLZ39fCde/+Lvz50mruizGqg7gJF8cdI0xgNeHVYGx4KWx0TxjbwyxV/KLvk+o8ffpbrlz0Ludy2+RlmVh11FygKjR/bwPdPOZR3HvCGoa6K7aTCVsaZb5zESfP24YcPruaHD64GRGdP75ZGvivqvFtXsKm9i9Pf+We1rbTZKFF38ygK9URw0F67DXU1rAqmTGzinPe+maVfPIqrPtJMuSkaX7vrKa5bWlmuw8z6p+4ChQSTmsYwrjHnLqdRYMrEJt55wBu4cNGcsmUvuH0l6ze2s35juyftmQ2iuut6etMeu3LNx+Y5iTnKnDxvXwi48PaVADvcpCmvszs4/7bfcd9TL3nRSLNBVHeBYkxOHDx98lBXw4ZAYdL7vqde4Nv39l5f6q7fvQBsv/HSWTcnN17K37/EzPqv7gKFjW6Fk/k6unr43q9WZ5bv6Orh6Mse5CsLvdy52UA5UFjd+tg79ufqh54pO/eiozu23bq1qyd8UyWzfqq7ZLZZ3pSJTVx6/MGMa8xtG+Bw6lv3YWwfQ6Q2dXTT3tXDP93zn7zt6/dy3bJnnfQ2q4BbFFbXCu8lnu9WOvXwGRx92YMlE9557V3bWxleWNIsm1sUVvcK740BMPONk/jH9yctjQlNDZnHburoZmtnD2fdvJzWF16rRXXN6k5VA4WkBZKeltQq6dw+ynxA0ipJKyVdX8362Oix8JC9eeicd3P9xw7nq++bU/amSvmk9+LH19aohmb1o2pdT5IagMuB9wBtwCOSFkfEqoIys4AvAEdExCuSvBaHDZriW7eWu6lSR3d4yXqzEqqZozgMaI2I1QCSbgAWAasKynwcuDwiXgGIiBerWB8bxaZMbOLMI2dx0rx9aHtlCyvWbuDCO1bRUbTyYE5i5boNXj/MrEBFgUJSE3AcMKPwmIi4KOOwvYE1BdttwLyiMgek538IaAAuiIh/K/H6pwOnA+yzzz6VVNmspMJWxrz9Xt8r6b25o5uP/6SFS48/2Mlts1SlOYrbSFoDXcCmgp8sKrGveBjKGGAWMB84EfihpF7TriPiyohojojm3XffvcIqm2XLJ72L8xftXUkXlIfNmiUq7XqaFhEL+nnuNmB64TmAdSXKLI2ITuAZSU+TBI5H+vlaZgOy8JC9mTx+LJ+49lE2F9wQy3dNNNuu0hbFbyT9r36e+xFglqT9JI0FTgAWF5X5BfAXAJKmknRFZa/JYDbIDtprV3qKGru+a6LZdpUGircDj6ZDXZdL+p2k5VkHREQXcAZwN/AkcFNErJR0kaSFabG7gfWSVgH3A5+PiPUDeytmAzNlYhOXHDd3hxneA1nC3sub20iliPL3KJZU8j6TEVHzO8U0NzdHS0tLrV925Js/P/m9ZMlQ1mJIrd/YvsMM7/647fG1nHPLci9vbsOWpEcjonkgx1aUo4iIZyUdDLwj3fVgRDwxkBc0G64Kb8faH+s3tnPOLcvZ2tmzbXlzz8ewkaSiridJnwauA96Q/vxU0pnVrJhZvWh7ZQuNuR3/K+WT4e6OspGg0lFPpwHzImITgKRvAA8D36lWxczqxbTX7UJnz44T9zp7elixdgMfvPLhXt1RO9PFZTYUKg0UAgrXPOim9DwJs1Ennww/uyBHcf4xs7n4jlW9uqNe29rFxXeuci7D6kqlgeJHwDJJt6bb7wOuqk6VzOpP8XLn+e6ofJAAaMiJC29fSUd3bNv/+ZuXM3n8WA7aa1e3LmzYqjSZ/c+SlpAMkxXw0Yh4rJoVM6s3xcnwXt1R3UFjQ46O7u2N8/auHv722ha6A75y7GxOnldygKHZkMpMZkvaNf39euD3wE+Ba4Fn031mVkKpuRlfOXY23SWGo2/p7KGjq4fzbl3BdUtrPuLcrKxyLYrrgWOAR9lxnSal2/tXqV5mda/U3fcmNY3h7FuWk5PY3NF7qfMLbl/J9NePd1eUDSuZgSIijkl/71eb6piNLMXdUfngsXLdBk67poXOotu1dnYHn7j2UXrw7Vlt+Kh0HsURkiakj0+R9M+SvN632QBMmdjEOw94AxcsPKjk85s7fXtWG14qXevpe8DmdHb22cCzJLkKMxugk+fty1ffN4exDWKXxt7/FfO3Z71u2bOetGdDqtLhsV0REZIWAd+OiKskfbiaFTMbDU4+fF8WzNmDles28PGftNDetWNXVEd3cN6tK5gwtoHuCM4/ZjZz9trNk/WspioNFK9J+gJwCvDO9H7YjdWrltnoke+KuvT4gznr5uW9bs8KsClNfBcGDecwrFYq7Xr6INAOnBYRfyC5zemlVauV2Si08JC9uevMtzO2IXvRg00dSQ7Dd+GzWqkoUETEHyLinyPiwXT7uYj4SXWrZjb65G/POq4xx4Smhsyy+YUHzaots+tJ0q8j4u2SXqPEPIqI2LWqtTMbhQrnX6xYu4GL71xFQ05sat9x3kXxXfi82KBVS7l5FG9Pf0+qTXXMDLbPvzh4+mQWzNljh6BRuKBgPiD4xklWTRUlsyUdDqyMiNfS7YnAQRGxrJqVM7PSQaOw1eAbJ1m19WcexcaC7c3pPjOroXzAKAwAWTdOMhsMlQYKRcHNtSOih8qH1ppZFfV146TC/IXZzqg0UKyW9PeSGtOfTwOrq1kxM6tMqZVqC/MXZjur0lbBJ4DLgC+RjH66Fzi9WpUys/4ptVJtf3nUlPWl0hsXvQicUOW6mNlOKF6ptj88asqyVLp67AGS7pW0It2eK+lL1a2amdVC4aip19q7POvbeqk0R/ED4AtAJ0BELMctDLMRoT+jptZvbPdKtqNQpTmK8RHxH9IOa9B0VaE+ZlZjlY6acvfU6FVpi+JlSX9GuoyHpOOB56tWKzOrmUpGTbl7anSrtEXxd8CVwJskrQWeAU6uWq3MrKbKjZrKd0/lZ37D9u4pj5Aa+coGCkk5oDkijkpvh5rLL+VhZiNH1qgpT+ob3cp2PaWzsM9IH2/qT5CQtEDS05JaJZ2bUe54SSGpudw5t3R0u7lrVmP9ndTnpPfIUmnX0z2SzgJuBDbld0bEH/s6IL0L3uXAe4A24BFJiyNiVVG5ScDfAxUtMLj65U0c8Y37nEgzq7FKJ/U56T3yVJrM/hvgU8CvgJaCnyyHAa0RsToiOoAbgEUlyl0MXAJsraQiPRFOpJkNkVKLEhZy0ntkqjRQzCZpHTwBPA58BziozDF7A2sKttvSfdtIegswPSLuyDqRpNMltUhq6d68AfDqmGbDkVeyHZkqDRTXAG8mWe/pO+nja8ocU+rGv9tWoE2T5N8EPlfuxSPiyohojojmhvG7AU6kmQ1H/Ul6O49RPyrNURwYEQcXbN8v6Ykyx7QB0wu2pwHrCrYnAXOAJelEvj2AxZIWRkSf3Vo5yatjmg1T+aT32UU5iuL/q85j1JdKA8Vjkg6PiKUAkuYBD5U55hFglqT9gLUkS36clH8yIjYAU/PbkpYAZ2UFCYD9p07ggXPe7SBhNkyVS3r7jnz1p9JAMQ84VdJz6fY+wJOSfgdERMwtPiAiuiSdAdwNNABXR8RKSRcBLRGxeCAV3mVsg/+YzIa5rDkZnrxXfyoNFAsGcvKIuAu4q2jfl/soO38gr2Fm9aW/k/d8n4yhV+n9KJ6tdkXMbHSoNI8BzmUMF77vtZnVXCWT95zLGD4cKMxsSJS7I59zGcNHpfMozMxqygsRDh8OFGY2LPV3IcJintA3eNz1ZGbDVqULERZzEnxwOVCY2bBWLpdRzEnwweeuJzMbUbww4eBzoDCzEWUgE/qcy8jmriczG1E8oW/wOVCY2YjjCX2Dy4HCzEakwZ7QN5rXnHKgMLNRqT+5jNHeReVktpmNSpVO6PN9wN2iMLNRrJJchteccqAws1GuXC5joGtOjaSchgOFmVmG/gy3zRtpOQ0HCjOzMvqz5tRIHHbrQGFmVoFK15waiTkNj3oyMxtE/clp1MvyIW5RmJkNokpzGvWUx3CgMDMbZOVyGv3NYwz1CCoHCjOzKsjKafQnjzEcWh7OUZiZ1VileYyBzAqvRt7DLQozsxqrNI/R3xFUxa2P84+ZzZy9dis7ObAcBwozsyFQydyM/o6gKs57nHfrCsY35ujoDsZM3nO/gdbVXU9mZkNkysQmDp4+uc9cRqULF0LpW8ACbO7soasnyI2b8PqB1tMtCjOzYazSWeGlWh+DxS0KM7NhrlzLI18m3/qY0NQwqK/vFoWZ2QhR2Pq478kX+PZ9rYNy3qq2KCQtkPS0pFZJ55Z4/rOSVklaLuleSftWsz5mZiNdvvXxmb88kHcfuPugnLNqgUJSA3A58F5gNnCipNlFxR4DmiNiLnAzcEm16mNmNtpc/dHDOHSf3Xb6PNVsURwGtEbE6ojoAG4AFhUWiIj7I2JzurkUmFbF+piZjTr/+qm3c/PfHk73plefH+g5qhko9gbWFGy3pfv6chrwy1JPSDpdUouklpdeemkQq2hmNvI17zeF7tdeXjfQ46sZKFRiX5QsKJ0CNAOXlno+Iq6MiOaIaN5998HpczMzs8pUc9RTGzC9YHsa0CuiSToKOA94V0QM70XZzcxGoWq2KB4BZknaT9JY4ARgcWEBSW8B/gVYGBEvVrEuZmY2QFULFBHRBZwB3A08CdwUESslXSRpYVrsUmAi8HNJj0ta3MfpzMxsiFR1wl1E3AXcVbTvywWPj6rm65uZ2c7zEh5mZpbJgcLMzDI5UJiZWSYHCjMzy+RAYWZmmRwozMwskwOFmZllcqAwM7NMDhRmZpbJgcLMzDI5UJiZWSYHCjMzy+RAYWZmmRwozMwskwOFmZllcqAwM7NMDhRmZpbJgcLMzDI5UJiZWSYHCjMzy+RAYWZmmRwozMwskwOFmZllcqAwM7NMDhRmZpbJgcLMzDI5UJiZWSYHCjMzy+RAYWZmmRwozMwsU1UDhaQFkp6W1Crp3BLPN0m6MX1+maQZ5c65paOb9Rvbq1FdMzMroWqBQlIDcDnwXmA2cKKk2UXFTgNeiYiZwDeBb5Q77+qXN3HEN+5j8eNrB7vKZmZWQjVbFIcBrRGxOiI6gBuARUVlFgHXpI9vBo6UpKyT9kSwtbOHs29Z7paFmVkNVDNQ7A2sKdhuS/eVLBMRXcAGYErxiSSdLqlFUkv35g0ANOZytL2ypRr1NjOzAtUMFKVaBjGAMkTElRHRHBHNDeN3A6Czp4dpr9tl52tpZmaZqhko2oDpBdvTgHV9lZE0BtgN+GPWSXMS4xpzXHLcXKZMbBrE6pqZWSljqnjuR4BZkvYD1gInACcVlVkMfBh4GDgeuC8ierUoCu0/dQIPnPNuBwkzsxqpWqCIiC5JZwB3Aw3A1RGxUtJFQEtELAauAq6V1ErSkjih3Hl3GdvgIGFmVkPqnW3EAAAFm0lEQVTVbFEQEXcBdxXt+3LB463A+6tZBzMz2zmemW1mZpkcKMzMLJMDhZmZZXKgMDOzTA4UZmaWyYHCzMwyqcz8tmFH0mvA00Ndj2FiKvDyUFdimPC12M7XYjtfi+0OjIhJAzmwqvMoquTpiGge6koMB5JafC0Svhbb+Vps52uxnaSWgR7rriczM8vkQGFmZpnqMVBcOdQVGEZ8LbbztdjO12I7X4vtBnwt6i6ZbWZmtVWPLQozM6shBwozM8s0bAOFpAWSnpbUKuncEs83SboxfX6ZpBm1r2VtVHAtPitplaTlku6VtO9Q1LMWyl2LgnLHSwpJI3ZoZCXXQtIH0r+NlZKur3Uda6WC/yP7SLpf0mPp/5Ojh6Ke1SbpakkvSlrRx/OSdFl6nZZLOrSiE0fEsPshudHRfwP7A2OBJ4DZRWU+BXw/fXwCcONQ13sIr8VfAOPTx58czdciLTcJeABYCjQPdb2H8O9iFvAY8Lp0+w1DXe8hvBZXAp9MH88Gfj/U9a7StXgncCiwoo/njwZ+CQg4HFhWyXmHa4viMKA1IlZHRAdwA7CoqMwi4Jr08c3AkZJUwzrWStlrERH3R8TmdHMpyf3JR6JK/i4ALgYuAbbWsnI1Vsm1+DhweUS8AhARL9a4jrVSybUIYNf08W7AuhrWr2Yi4gGSu4X2ZRHwk0gsBSZL2rPceYdroNgbWFOw3ZbuK1kmIrqADcCUmtSutiq5FoVOI/nGMBKVvRaS3gJMj4g7almxIVDJ38UBwAGSHpK0VNKCmtWutiq5FhcAp0hqI7nr5pm1qdqw09/PE2D4LuFRqmVQPI63kjIjQcXvU9IpQDPwrqrWaOhkXgtJOeCbwEdqVaEhVMnfxRiS7qf5JK3MByXNiYhXq1y3WqvkWpwI/Dgi/knSW4Fr02vRU/3qDSsD+twcri2KNmB6wfY0ejcVt5WRNIakOZnV5KpXlVwLJB0FnAcsjIj2GtWt1spdi0nAHGCJpN+T9MEuHqEJ7Ur/j9wWEZ0R8QzJYpqzalS/WqrkWpwG3AQQEQ8D40gWDBxtKvo8KTZcA8UjwCxJ+0kaS5KsXlxUZjHw4fTx8cB9kWZrRpiy1yLtbvkXkiAxUvuhocy1iIgNETE1ImZExAySfM3CiBjwYmjDWCX/R35BMtABSVNJuqJW17SWtVHJtXgOOBJA0ptJAsVLNa3l8LAYODUd/XQ4sCEini930LDseoqILklnAHeTjGi4OiJWSroIaImIxcBVJM3HVpKWxAlDV+PqqfBaXApMBH6e5vOfi4iFQ1bpKqnwWowKFV6Lu4G/lLQK6AY+HxHrh67W1VHhtfgc8ANJnyHpavnISPxiKelnJF2NU9N8zFeARoCI+D5JfuZooBXYDHy0ovOOwGtlZmaDaLh2PZmZ2TDhQGFmZpkcKMzMLJMDhZmZZXKgMDOzTA4UZjUkaUZ+ZU9J8yWN9KVGbARwoDCrQDpByf9fbFTyH75ZH9Jv/09KugL4LfAhSQ9L+q2kn0uamJb7c0m/kfSEpP+QNCk99sG07G8lvW1o343ZwDlQmGU7EPgJ8B6S9YKOiohDgRbgs+mSETcCn46Ig4GjgC3Ai8B70rIfBC4bisqbDYZhuYSH2TDybEQslXQMyQ1vHkqXSRkLPEwSSJ6PiEcAIuJPAJImAN+VdAjJ8hkHDEXlzQaDA4VZtk3pbwH3RMSJhU9KmkvpZZo/A7wAHEzSch/JN1GyEc5dT2aVWQocIWkmgKTxkg4AngL2kvTn6f5JBcveP5/e7+BDJIvVmdUlBwqzCkTESyQ3RPqZpOUkgeNN6a03Pwh8R9ITwD0kS1hfAXxY0lKSbqdNJU9sVge8eqyZmWVyi8LMzDI5UJiZWSYHCjMzy+RAYWZmmRwozMwskwOFmZllcqAwM7NM/wMx2EroiGCAqwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot P-R ROC w/ best cutoff\n",
    "title_ = 'P-R Curve: Best F1 = ' + str(best_cut)\n",
    "ax = prroc_frame.plot(x='recall', y='precision', kind='scatter', title=title_, xlim=[0,1])\n",
    "_ = ax.axvline(best_cut, color='r')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Report Raw Confusion Matrices"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Function to print confusion matrices by an input variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>actual: 1</th>\n",
       "      <th>actual: 0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>predicted: 1</th>\n",
       "      <td>1083</td>\n",
       "      <td>920</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>predicted: 0</th>\n",
       "      <td>949</td>\n",
       "      <td>5988</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             actual: 1 actual: 0\n",
       "predicted: 1      1083       920\n",
       "predicted: 0       949      5988"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_confusion_matrix(frame, y, yhat, by=None, level=None, cutoff=0.5):\n",
    "\n",
    "    \"\"\" Creates confusion matrix from pandas dataframe of y and yhat values, can be sliced \n",
    "        by a variable and level.\n",
    "    \n",
    "    Args:\n",
    "        frame: Pandas dataframe of actual (y) and predicted (yhat) values.\n",
    "        y: Name of actual value column.\n",
    "        yhat: Name of predicted value column.\n",
    "        by: By variable to slice frame before creating confusion matrix, default None.\n",
    "        level: Value of by variable to slice frame before creating confusion matrix, default None.\n",
    "        cutoff: Cutoff threshold for confusion matrix, default 0.5. \n",
    "\n",
    "    Returns:\n",
    "        Confusion matrix as pandas dataframe. \n",
    "    \"\"\"\n",
    "    \n",
    "    # determine levels of target (y) variable\n",
    "    # sort for consistency\n",
    "    level_list = list(frame[y].unique())\n",
    "    level_list.sort(reverse=True)\n",
    "\n",
    "    # init confusion matrix\n",
    "    cm_frame = pd.DataFrame(columns=['actual: ' +  str(i) for i in level_list], \n",
    "                            index=['predicted: ' + str(i) for i in level_list])\n",
    "    \n",
    "    # don't destroy original data\n",
    "    frame_ = frame.copy(deep=True)\n",
    "    \n",
    "    # convert numeric predictions to binary decisions using cutoff\n",
    "    dname = 'd_' + str(y)\n",
    "    frame_[dname] = np.where(frame_[yhat] > cutoff , 1, 0)\n",
    "    \n",
    "    # slice frame\n",
    "    if (by is not None) & (level is not None):\n",
    "        frame_ = frame_[frame[by] == level]\n",
    "    \n",
    "    # calculate size of each confusion matrix value\n",
    "    for i, lev_i in enumerate(level_list):\n",
    "        for j, lev_j in enumerate(level_list):\n",
    "            cm_frame.iat[i, j] = frame_[(frame_[y] == lev_i) & (frame_[dname] == lev_j)].shape[0]\n",
    "    \n",
    "    # output results\n",
    "    if by is None:\n",
    "        print('Confusion matrix:')\n",
    "    else:\n",
    "        print('Confusion matrix by ' + by + '=' + str(level))\n",
    "    \n",
    "    return cm_frame\n",
    "    \n",
    "    \n",
    "get_confusion_matrix(test_yhat, 'DEFAULT_NEXT_MONTH', 'p_DEFAULT_NEXT_MONTH', cutoff=best_cut)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Report confusion matrices by `MARRIAGE` level "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "marriage_dict = {0:'other', 1:'married', 2:'single', 3:'divorced'}\n",
    "test_yhat['MARRIAGE'] = test_yhat['MARRIAGE'].apply(lambda i: marriage_dict[i]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "['married', 'single', 'divorced', 'other']"
      ],
      "text/plain": [
       "['married', 'single', 'divorced', 'other']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "levels = list(test_yhat['MARRIAGE'].unique())\n",
    "levels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Confusion matrix for `Marriage = married`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix by MARRIAGE=married\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>actual: 1</th>\n",
       "      <th>actual: 0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>predicted: 1</th>\n",
       "      <td>527</td>\n",
       "      <td>450</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>predicted: 0</th>\n",
       "      <td>453</td>\n",
       "      <td>2660</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             actual: 1 actual: 0\n",
       "predicted: 1       527       450\n",
       "predicted: 0       453      2660"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "married_cm = get_confusion_matrix(test_yhat, 'DEFAULT_NEXT_MONTH', 'p_DEFAULT_NEXT_MONTH', by='MARRIAGE', level='married', cutoff=best_cut)\n",
    "married_cm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Confusion matrix for `Marriage = single`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix by MARRIAGE=single\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>actual: 1</th>\n",
       "      <th>actual: 0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>predicted: 1</th>\n",
       "      <td>545</td>\n",
       "      <td>457</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>predicted: 0</th>\n",
       "      <td>481</td>\n",
       "      <td>3256</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             actual: 1 actual: 0\n",
       "predicted: 1       545       457\n",
       "predicted: 0       481      3256"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "single_cm = get_confusion_matrix(test_yhat, 'DEFAULT_NEXT_MONTH', 'p_DEFAULT_NEXT_MONTH', by='MARRIAGE', level='single', cutoff=best_cut)\n",
    "single_cm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Confusion matrix for `Marriage = divorced`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix by MARRIAGE=divorced\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>actual: 1</th>\n",
       "      <th>actual: 0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>predicted: 1</th>\n",
       "      <td>10</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>predicted: 0</th>\n",
       "      <td>13</td>\n",
       "      <td>59</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             actual: 1 actual: 0\n",
       "predicted: 1        10        13\n",
       "predicted: 0        13        59"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "divorced_cm = get_confusion_matrix(test_yhat, 'DEFAULT_NEXT_MONTH', 'p_DEFAULT_NEXT_MONTH', by='MARRIAGE', level='divorced', cutoff=best_cut)\n",
    "divorced_cm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Confusion matrix for `MARRIAGE = other`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix by MARRIAGE=other\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>actual: 1</th>\n",
       "      <th>actual: 0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>predicted: 1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>predicted: 0</th>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             actual: 1 actual: 0\n",
       "predicted: 1         1         0\n",
       "predicted: 0         2        13"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "other_cm = get_confusion_matrix(test_yhat, 'DEFAULT_NEXT_MONTH', 'p_DEFAULT_NEXT_MONTH', by='MARRIAGE', level='other', cutoff=best_cut)\n",
    "other_cm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5. Disparate Impact Analysis (DIA)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To perform the following basic DIA many different values from the confusion matrices reflecting different prediction behavior are calculated. These metrics essentially help us understand the GBM's overall performance and how it behaves when predicting:\n",
    "\n",
    "* Default correctly\n",
    "* Non-default correctly\n",
    "* Default incorrectly (type I errors)\n",
    "* Non-default incorrectly (type II errors)\n",
    "\n",
    "In a real-life lending scenario, type I errors essentially amount to false accusations of financial impropriety and type II errors result in awarding loans to undeserving customers. Both types of errors can be costly to the lender too. Type I errors likely result in lost interest and fees. Type II errors often result in write-offs."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Dictionary of metrics used to assess group parity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# represent metrics as dictionary for use later\n",
    "metric_dict = {\n",
    "\n",
    "#### overall performance\n",
    "'Adverse Impact': '(tp + fp) / (tp + tn + fp + fn)', # how often the model predicted default for each group   \n",
    "#'Prevalence': '(tp + fn) / (tp + tn +fp + fn)', # how much default actually happens for this group\n",
    "'Accuracy':   '(tp + tn) / (tp + tn +fp + fn)', # how often the model predicts default and non-default correctly for this group\n",
    "\n",
    "#### predicting default will happen\n",
    "# (correctly)\n",
    "'True Positive Rate': 'tp / (tp + fn)',  # out of the people in the group *that did* default, how many the model predicted *correctly* would default              \n",
    "'Precision':          'tp / (tp + fp)',  # out of the people in the group the model *predicted* would default, how many the model predicted *correctly* would default\n",
    "\n",
    "#### predicting default won't happen\n",
    "# (correctly)\n",
    "'Specificity':              'tn / (tn + fp)', # out of the people in the group *that did not* default, how many the model predicted *correctly* would not default\n",
    "'Negative Predicted Value': 'tn / (tn + fn)', # out of the people in the group the model *predicted* would not default, how many the model predicted *correctly* would not default  \n",
    "\n",
    "#### analyzing errors - type I\n",
    "# false accusations \n",
    "'False Positive Rate':  'fp / (tn + fp)', # out of the people in the group *that did not* default, how many the model predicted *incorrectly* would default\n",
    "'False Discovery Rate': 'fp / (tp + fp)', # out of the people in the group the model *predicted* would default, how many the model predicted *incorrectly* would default\n",
    "\n",
    "#### analyzing errors - type II\n",
    "# costly ommisions\n",
    "'False Negative Rate': 'fn / (tp + fn)', # out of the people in the group *that did* default, how many the model predicted *incorrectly* would not default\n",
    "'False Omissions Rate':'fn / (tn + fn)'  # out of the people in the group the model *predicted* would not default, how many the model predicted *incorrectly* would not default\n",
    "}    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Utility function to translate metrics into Pandas statements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# small utility function\n",
    "# translates abbreviated metric expressions into executable Python statements\n",
    "\n",
    "def cm_exp_parser(expression):\n",
    "    \n",
    "    # tp | fp       cm_dict[level].iat[0, 0] | cm_dict[level].iat[0, 1]\n",
    "    # -------  ==>  --------------------------------------------\n",
    "    # fn | tn       cm_dict[level].iat[1, 0] | cm_dict[level].iat[1, 1]\n",
    "\n",
    "    expression = expression.replace('tp', 'cm_dict[level].iat[0, 0]')\\\n",
    "                           .replace('fp', 'cm_dict[level].iat[0, 1]')\\\n",
    "                           .replace('fn', 'cm_dict[level].iat[1, 0]')\\\n",
    "                           .replace('tn', 'cm_dict[level].iat[1, 1]')\n",
    "\n",
    "    return expression\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Calculate and report group metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Adverse Impact</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>True Positive Rate</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Negative Predicted Value</th>\n",
       "      <th>False Positive Rate</th>\n",
       "      <th>False Discovery Rate</th>\n",
       "      <th>False Negative Rate</th>\n",
       "      <th>False Omissions Rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>married</th>\n",
       "      <td>0.238875</td>\n",
       "      <td>0.779218</td>\n",
       "      <td>0.537755</td>\n",
       "      <td>0.539406</td>\n",
       "      <td>0.855305</td>\n",
       "      <td>0.854481</td>\n",
       "      <td>0.144695</td>\n",
       "      <td>0.460594</td>\n",
       "      <td>0.462245</td>\n",
       "      <td>0.145519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>single</th>\n",
       "      <td>0.211437</td>\n",
       "      <td>0.802068</td>\n",
       "      <td>0.531189</td>\n",
       "      <td>0.543912</td>\n",
       "      <td>0.876919</td>\n",
       "      <td>0.871287</td>\n",
       "      <td>0.123081</td>\n",
       "      <td>0.456088</td>\n",
       "      <td>0.468811</td>\n",
       "      <td>0.128713</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>divorced</th>\n",
       "      <td>0.242105</td>\n",
       "      <td>0.726316</td>\n",
       "      <td>0.434783</td>\n",
       "      <td>0.434783</td>\n",
       "      <td>0.819444</td>\n",
       "      <td>0.819444</td>\n",
       "      <td>0.180556</td>\n",
       "      <td>0.565217</td>\n",
       "      <td>0.565217</td>\n",
       "      <td>0.180556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>0.062500</td>\n",
       "      <td>0.875000</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.866667</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.133333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Adverse Impact  Accuracy  True Positive Rate  Precision  \\\n",
       "married         0.238875  0.779218            0.537755   0.539406   \n",
       "single          0.211437  0.802068            0.531189   0.543912   \n",
       "divorced        0.242105  0.726316            0.434783   0.434783   \n",
       "other           0.062500  0.875000            0.333333   1.000000   \n",
       "\n",
       "          Specificity  Negative Predicted Value  False Positive Rate  \\\n",
       "married      0.855305                  0.854481             0.144695   \n",
       "single       0.876919                  0.871287             0.123081   \n",
       "divorced     0.819444                  0.819444             0.180556   \n",
       "other        1.000000                  0.866667             0.000000   \n",
       "\n",
       "          False Discovery Rate  False Negative Rate  False Omissions Rate  \n",
       "married               0.460594             0.462245              0.145519  \n",
       "single                0.456088             0.468811              0.128713  \n",
       "divorced              0.565217             0.565217              0.180556  \n",
       "other                 0.000000             0.666667              0.133333  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# initialize dict of confusion matrices and corresponding rows of dataframe\n",
    "cm_dict = {'other': other_cm, \n",
    "           'married': married_cm, \n",
    "           'single': single_cm, \n",
    "           'divorced': divorced_cm} \n",
    "\n",
    "metrics_frame = pd.DataFrame(index=levels) # frame for metrics\n",
    "\n",
    "# nested loop through:\n",
    "# - marriage statuses\n",
    "# - metrics \n",
    "for level in levels:\n",
    "    for metric in metric_dict.keys():\n",
    "              \n",
    "        # parse metric expressions into executable pandas statements\n",
    "        expression = cm_exp_parser(metric_dict[metric])\n",
    "\n",
    "        # dynamically evaluate metrics to avoid code duplication\n",
    "        metrics_frame.loc[level, metric] = eval(expression)  \n",
    "\n",
    "# display results                \n",
    "metrics_frame"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Calculate and report disparity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row0_col0 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row0_col1 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row0_col2 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row0_col3 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row0_col4 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row0_col5 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row0_col6 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row0_col7 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row0_col8 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row0_col9 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row1_col0 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row1_col1 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row1_col2 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row1_col3 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row1_col4 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row1_col5 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row1_col6 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row1_col7 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row1_col8 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row1_col9 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row2_col0 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row2_col1 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row2_col2 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row2_col3 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row2_col4 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row2_col5 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row2_col6 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row2_col7 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row2_col8 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row2_col9 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row3_col0 {\n",
       "            color:  red;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row3_col1 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row3_col2 {\n",
       "            color:  red;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row3_col3 {\n",
       "            color:  red;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row3_col4 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row3_col5 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row3_col6 {\n",
       "            color:  red;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row3_col7 {\n",
       "            color:  red;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row3_col8 {\n",
       "            color:  red;\n",
       "        }    #T_dd5594ce_7018_11e9_bb57_0242b5940aa5row3_col9 {\n",
       "            color:  blue;\n",
       "        }</style>  \n",
       "<table id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5\" > \n",
       "<thead>    <tr> \n",
       "        <th class=\"blank level0\" ></th> \n",
       "        <th class=\"col_heading level0 col0\" >Adverse Impact Disparity</th> \n",
       "        <th class=\"col_heading level0 col1\" >Accuracy Disparity</th> \n",
       "        <th class=\"col_heading level0 col2\" >True Positive Rate Disparity</th> \n",
       "        <th class=\"col_heading level0 col3\" >Precision Disparity</th> \n",
       "        <th class=\"col_heading level0 col4\" >Specificity Disparity</th> \n",
       "        <th class=\"col_heading level0 col5\" >Negative Predicted Value Disparity</th> \n",
       "        <th class=\"col_heading level0 col6\" >False Positive Rate Disparity</th> \n",
       "        <th class=\"col_heading level0 col7\" >False Discovery Rate Disparity</th> \n",
       "        <th class=\"col_heading level0 col8\" >False Negative Rate Disparity</th> \n",
       "        <th class=\"col_heading level0 col9\" >False Omissions Rate Disparity</th> \n",
       "    </tr></thead> \n",
       "<tbody>    <tr> \n",
       "        <th id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5level0_row0\" class=\"row_heading level0 row0\" >married</th> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row0_col0\" class=\"data row0 col0\" >1</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row0_col1\" class=\"data row0 col1\" >1</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row0_col2\" class=\"data row0 col2\" >1</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row0_col3\" class=\"data row0 col3\" >1</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row0_col4\" class=\"data row0 col4\" >1</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row0_col5\" class=\"data row0 col5\" >1</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row0_col6\" class=\"data row0 col6\" >1</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row0_col7\" class=\"data row0 col7\" >1</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row0_col8\" class=\"data row0 col8\" >1</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row0_col9\" class=\"data row0 col9\" >1</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5level0_row1\" class=\"row_heading level0 row1\" >single</th> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row1_col0\" class=\"data row1 col0\" >0.885135</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row1_col1\" class=\"data row1 col1\" >1.02932</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row1_col2\" class=\"data row1 col2\" >0.98779</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row1_col3\" class=\"data row1 col3\" >1.00835</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row1_col4\" class=\"data row1 col4\" >1.02527</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row1_col5\" class=\"data row1 col5\" >1.01967</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row1_col6\" class=\"data row1 col6\" >0.850627</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row1_col7\" class=\"data row1 col7\" >0.990217</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row1_col8\" class=\"data row1 col8\" >1.0142</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row1_col9\" class=\"data row1 col9\" >0.88451</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5level0_row2\" class=\"row_heading level0 row2\" >divorced</th> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row2_col0\" class=\"data row2 col0\" >1.01352</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row2_col1\" class=\"data row2 col1\" >0.932109</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row2_col2\" class=\"data row2 col2\" >0.808514</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row2_col3\" class=\"data row2 col3\" >0.806039</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row2_col4\" class=\"data row2 col4\" >0.958072</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row2_col5\" class=\"data row2 col5\" >0.958996</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row2_col6\" class=\"data row2 col6\" >1.24784</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row2_col7\" class=\"data row2 col7\" >1.22715</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row2_col8\" class=\"data row2 col8\" >1.22277</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row2_col9\" class=\"data row2 col9\" >1.24077</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5level0_row3\" class=\"row_heading level0 row3\" >other</th> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row3_col0\" class=\"data row3 col0\" >0.261643</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row3_col1\" class=\"data row3 col1\" >1.12292</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row3_col2\" class=\"data row3 col2\" >0.619861</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row3_col3\" class=\"data row3 col3\" >1.85389</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row3_col4\" class=\"data row3 col4\" >1.16917</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row3_col5\" class=\"data row3 col5\" >1.01426</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row3_col6\" class=\"data row3 col6\" >0</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row3_col7\" class=\"data row3 col7\" >0</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row3_col8\" class=\"data row3 col8\" >1.44224</td> \n",
       "        <td id=\"T_dd5594ce_7018_11e9_bb57_0242b5940aa5row3_col9\" class=\"data row3 col9\" >0.916262</td> \n",
       "    </tr></tbody> \n",
       "</table> "
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f6df064c860>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ref_level = 'married' # user-defined reference level\n",
    "\n",
    "parity_threshold_low = 0.8    # user-defined low threshold value\n",
    "parity_threshold_hi = 1.25    # user-defined high threshold value\n",
    "\n",
    "# init frame to store disparity measures\n",
    "disp_frame = pd.DataFrame(index=levels)\n",
    "\n",
    "# compare all metrics to reference level\n",
    "disp_frame = metrics_frame/metrics_frame.loc[ref_level, :]\n",
    "\n",
    "# change column names\n",
    "disp_frame.columns=[col + ' Disparity' for col in metrics_frame.columns]\n",
    "\n",
    "# small utility function to format pandas table output\n",
    "def disparate_red(val):\n",
    "    \n",
    "    color = 'blue' if (parity_threshold_low < val < parity_threshold_hi) else 'red'\n",
    "    return 'color: %s' % color \n",
    "\n",
    "# display results\n",
    "disp_frame.style.applymap(disparate_red)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Assess and report group parity\n",
    "A binary indication of parity for metrics is reported by simply checking whether disparity values are within the user-defined thresholds. Further parity indicators are defined as combinations of other disparity values:\n",
    "\n",
    "* Type I Parity: Fairness in both FDR Parity and FPR Parity\n",
    "* Type II Parity: Fairness in both FOR Parity and FNR Parity\n",
    "* Equalized Odds: Fairness in both FPR Parity and TPR Parity\n",
    "* Supervised Fairness: Fairness in both Type I and Type II Parity\n",
    "* Overall Fairness: Fairness across all parities for all metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row0_col0 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row0_col1 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row0_col2 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row0_col3 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row0_col4 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row0_col5 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row0_col6 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row0_col7 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row0_col8 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row0_col9 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row0_col10 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row0_col11 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row0_col12 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row0_col13 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row0_col14 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row1_col0 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row1_col1 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row1_col2 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row1_col3 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row1_col4 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row1_col5 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row1_col6 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row1_col7 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row1_col8 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row1_col9 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row1_col10 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row1_col11 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row1_col12 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row1_col13 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row1_col14 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row2_col0 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row2_col1 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row2_col2 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row2_col3 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row2_col4 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row2_col5 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row2_col6 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row2_col7 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row2_col8 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row2_col9 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row2_col10 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row2_col11 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row2_col12 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row2_col13 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row2_col14 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row3_col0 {\n",
       "            color:  red;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row3_col1 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row3_col2 {\n",
       "            color:  red;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row3_col3 {\n",
       "            color:  red;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row3_col4 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row3_col5 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row3_col6 {\n",
       "            color:  red;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row3_col7 {\n",
       "            color:  red;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row3_col8 {\n",
       "            color:  red;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row3_col9 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row3_col10 {\n",
       "            color:  red;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row3_col11 {\n",
       "            color:  red;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row3_col12 {\n",
       "            color:  red;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row3_col13 {\n",
       "            color:  red;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row3_col14 {\n",
       "            color:  red;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row4_col0 {\n",
       "            color:  red;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row4_col1 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row4_col2 {\n",
       "            color:  red;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row4_col3 {\n",
       "            color:  red;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row4_col4 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row4_col5 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row4_col6 {\n",
       "            color:  red;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row4_col7 {\n",
       "            color:  red;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row4_col8 {\n",
       "            color:  red;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row4_col9 {\n",
       "            color:  blue;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row4_col10 {\n",
       "            color:  red;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row4_col11 {\n",
       "            color:  red;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row4_col12 {\n",
       "            color:  red;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row4_col13 {\n",
       "            color:  red;\n",
       "        }    #T_dd5594cf_7018_11e9_bb57_0242b5940aa5row4_col14 {\n",
       "            color:  red;\n",
       "        }</style>  \n",
       "<table id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5\" > \n",
       "<thead>    <tr> \n",
       "        <th class=\"blank level0\" ></th> \n",
       "        <th class=\"col_heading level0 col0\" >Adverse Impact Parity</th> \n",
       "        <th class=\"col_heading level0 col1\" >Accuracy Parity</th> \n",
       "        <th class=\"col_heading level0 col2\" >True Positive Rate Parity</th> \n",
       "        <th class=\"col_heading level0 col3\" >Precision Parity</th> \n",
       "        <th class=\"col_heading level0 col4\" >Specificity Parity</th> \n",
       "        <th class=\"col_heading level0 col5\" >Negative Predicted Value Parity</th> \n",
       "        <th class=\"col_heading level0 col6\" >False Positive Rate Parity</th> \n",
       "        <th class=\"col_heading level0 col7\" >False Discovery Rate Parity</th> \n",
       "        <th class=\"col_heading level0 col8\" >False Negative Rate Parity</th> \n",
       "        <th class=\"col_heading level0 col9\" >False Omissions Rate Parity</th> \n",
       "        <th class=\"col_heading level0 col10\" >Type I Parity</th> \n",
       "        <th class=\"col_heading level0 col11\" >Type II Parity</th> \n",
       "        <th class=\"col_heading level0 col12\" >Equalized Odds</th> \n",
       "        <th class=\"col_heading level0 col13\" >Supervised Fairness</th> \n",
       "        <th class=\"col_heading level0 col14\" >Overall Fairness</th> \n",
       "    </tr></thead> \n",
       "<tbody>    <tr> \n",
       "        <th id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5level0_row0\" class=\"row_heading level0 row0\" >married</th> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row0_col0\" class=\"data row0 col0\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row0_col1\" class=\"data row0 col1\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row0_col2\" class=\"data row0 col2\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row0_col3\" class=\"data row0 col3\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row0_col4\" class=\"data row0 col4\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row0_col5\" class=\"data row0 col5\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row0_col6\" class=\"data row0 col6\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row0_col7\" class=\"data row0 col7\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row0_col8\" class=\"data row0 col8\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row0_col9\" class=\"data row0 col9\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row0_col10\" class=\"data row0 col10\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row0_col11\" class=\"data row0 col11\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row0_col12\" class=\"data row0 col12\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row0_col13\" class=\"data row0 col13\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row0_col14\" class=\"data row0 col14\" >True</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5level0_row1\" class=\"row_heading level0 row1\" >single</th> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row1_col0\" class=\"data row1 col0\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row1_col1\" class=\"data row1 col1\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row1_col2\" class=\"data row1 col2\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row1_col3\" class=\"data row1 col3\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row1_col4\" class=\"data row1 col4\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row1_col5\" class=\"data row1 col5\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row1_col6\" class=\"data row1 col6\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row1_col7\" class=\"data row1 col7\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row1_col8\" class=\"data row1 col8\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row1_col9\" class=\"data row1 col9\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row1_col10\" class=\"data row1 col10\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row1_col11\" class=\"data row1 col11\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row1_col12\" class=\"data row1 col12\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row1_col13\" class=\"data row1 col13\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row1_col14\" class=\"data row1 col14\" >True</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5level0_row2\" class=\"row_heading level0 row2\" >divorced</th> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row2_col0\" class=\"data row2 col0\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row2_col1\" class=\"data row2 col1\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row2_col2\" class=\"data row2 col2\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row2_col3\" class=\"data row2 col3\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row2_col4\" class=\"data row2 col4\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row2_col5\" class=\"data row2 col5\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row2_col6\" class=\"data row2 col6\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row2_col7\" class=\"data row2 col7\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row2_col8\" class=\"data row2 col8\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row2_col9\" class=\"data row2 col9\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row2_col10\" class=\"data row2 col10\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row2_col11\" class=\"data row2 col11\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row2_col12\" class=\"data row2 col12\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row2_col13\" class=\"data row2 col13\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row2_col14\" class=\"data row2 col14\" >True</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5level0_row3\" class=\"row_heading level0 row3\" >other</th> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row3_col0\" class=\"data row3 col0\" >False</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row3_col1\" class=\"data row3 col1\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row3_col2\" class=\"data row3 col2\" >False</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row3_col3\" class=\"data row3 col3\" >False</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row3_col4\" class=\"data row3 col4\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row3_col5\" class=\"data row3 col5\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row3_col6\" class=\"data row3 col6\" >False</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row3_col7\" class=\"data row3 col7\" >False</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row3_col8\" class=\"data row3 col8\" >False</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row3_col9\" class=\"data row3 col9\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row3_col10\" class=\"data row3 col10\" >False</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row3_col11\" class=\"data row3 col11\" >False</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row3_col12\" class=\"data row3 col12\" >False</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row3_col13\" class=\"data row3 col13\" >False</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row3_col14\" class=\"data row3 col14\" >False</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5level0_row4\" class=\"row_heading level0 row4\" >all</th> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row4_col0\" class=\"data row4 col0\" >False</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row4_col1\" class=\"data row4 col1\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row4_col2\" class=\"data row4 col2\" >False</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row4_col3\" class=\"data row4 col3\" >False</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row4_col4\" class=\"data row4 col4\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row4_col5\" class=\"data row4 col5\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row4_col6\" class=\"data row4 col6\" >False</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row4_col7\" class=\"data row4 col7\" >False</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row4_col8\" class=\"data row4 col8\" >False</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row4_col9\" class=\"data row4 col9\" >True</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row4_col10\" class=\"data row4 col10\" >False</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row4_col11\" class=\"data row4 col11\" >False</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row4_col12\" class=\"data row4 col12\" >False</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row4_col13\" class=\"data row4 col13\" >False</td> \n",
       "        <td id=\"T_dd5594cf_7018_11e9_bb57_0242b5940aa5row4_col14\" class=\"data row4 col14\" >False</td> \n",
       "    </tr></tbody> \n",
       "</table> "
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f6df0685ef0>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# parity checks\n",
    "# low_threshold (0.8) < *_metric/white_metric < (high_threshold) 1.25 => parity, else disparity \n",
    "\n",
    "# init frame for parity\n",
    "par_frame = pd.DataFrame(index=levels, \n",
    "                         columns=[col + ' Parity' for col in metrics_frame.columns])\n",
    "# nested loop through: \n",
    "# - marriage statuses\n",
    "# - disparity metrics\n",
    "for i, _ in enumerate(levels):\n",
    "    for j, _ in enumerate(par_frame.columns):\n",
    "        par_frame.iat[i, j] = (parity_threshold_low < disp_frame.iat[i, j] < parity_threshold_hi)\n",
    "\n",
    "# add overall parity checks\n",
    "# Type I Parity: Fairness in both FDR Parity and FPR Parity\n",
    "# Type II Parity: Fairness in both FOR Parity and FNR Parity\n",
    "# Equalized Odds: Fairness in both FPR Parity and TPR Parity\n",
    "# Supervised Fairness: Fairness in both Type I and Type II Parity\n",
    "# Overall Fairness: Fairness across all parities for all metrics\n",
    "par_frame['Type I Parity'] = (par_frame['False Discovery Rate Parity']) & (par_frame['False Positive Rate Parity'])\n",
    "par_frame['Type II Parity'] = (par_frame['False Omissions Rate Parity']) & (par_frame['False Negative Rate Parity'])\n",
    "par_frame['Equalized Odds'] = (par_frame['False Positive Rate Parity']) & (par_frame['True Positive Rate Parity'])\n",
    "par_frame['Supervised Fairness'] = (par_frame['Type I Parity']) & (par_frame['Type II Parity'])\n",
    "par_frame['Overall Fairness'] = par_frame.all(axis='columns')\n",
    "par_frame.loc['all', :] = par_frame.all(axis='index')\n",
    "    \n",
    "# small utility function to format pandas table output    \n",
    "def color_false_red(val):\n",
    "\n",
    "    color = 'red' if not val else 'blue'\n",
    "    return 'color: %s' % color \n",
    "    \n",
    "par_frame.style.applymap(color_false_red)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}