model_fairness.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import modules\n",
    "from pathlib import Path\n",
    "import joblib\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import roc_auc_score, precision_score, recall_score, accuracy_score, f1_score\n",
    "from sklearn.metrics import make_scorer\n",
    "\n",
    "from fairlearn.metrics import selection_rate, MetricFrame\n",
    "from fairlearn.reductions import GridSearch, EqualizedOdds, FalsePositiveRateParity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set paths\n",
    "DATAPATH = \"./data\"\n",
    "DATAPROCESSEDPATH = \"./data_preprocessed\"\n",
    "MODELPATH = \"./models\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(DATAPROCESSEDPATH + '/data_preprocessed.pkl','rb') as f:\n",
    "    df = joblib.load(f)\n",
    "\n",
    "# load trained pipeline\n",
    "with open(MODELPATH + '/pipeline.pkl','rb') as f:\n",
    "    pipe = joblib.load(f)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>year</th>\n",
       "      <th>loan_limit</th>\n",
       "      <th>Gender</th>\n",
       "      <th>approv_in_adv</th>\n",
       "      <th>loan_type</th>\n",
       "      <th>loan_purpose</th>\n",
       "      <th>Credit_Worthiness</th>\n",
       "      <th>business_or_commercial</th>\n",
       "      <th>loan_amount</th>\n",
       "      <th>Neg_ammortization</th>\n",
       "      <th>interest_only</th>\n",
       "      <th>income</th>\n",
       "      <th>credit_type</th>\n",
       "      <th>Credit_Score</th>\n",
       "      <th>co-applicant_credit_type</th>\n",
       "      <th>age</th>\n",
       "      <th>submission_of_application</th>\n",
       "      <th>Region</th>\n",
       "      <th>Status</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>24890</td>\n",
       "      <td>2019</td>\n",
       "      <td>cf</td>\n",
       "      <td>Sex Not Available</td>\n",
       "      <td>nopre</td>\n",
       "      <td>type1</td>\n",
       "      <td>p1</td>\n",
       "      <td>l1</td>\n",
       "      <td>nob/c</td>\n",
       "      <td>116500</td>\n",
       "      <td>not_neg</td>\n",
       "      <td>not_int</td>\n",
       "      <td>1740.0</td>\n",
       "      <td>EXP</td>\n",
       "      <td>758</td>\n",
       "      <td>CIB</td>\n",
       "      <td>25-34</td>\n",
       "      <td>to_inst</td>\n",
       "      <td>south</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>24891</td>\n",
       "      <td>2019</td>\n",
       "      <td>cf</td>\n",
       "      <td>Male</td>\n",
       "      <td>nopre</td>\n",
       "      <td>type2</td>\n",
       "      <td>p1</td>\n",
       "      <td>l1</td>\n",
       "      <td>b/c</td>\n",
       "      <td>206500</td>\n",
       "      <td>not_neg</td>\n",
       "      <td>not_int</td>\n",
       "      <td>4980.0</td>\n",
       "      <td>EQUI</td>\n",
       "      <td>552</td>\n",
       "      <td>EXP</td>\n",
       "      <td>55-64</td>\n",
       "      <td>to_inst</td>\n",
       "      <td>North</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>24892</td>\n",
       "      <td>2019</td>\n",
       "      <td>cf</td>\n",
       "      <td>Male</td>\n",
       "      <td>pre</td>\n",
       "      <td>type1</td>\n",
       "      <td>p1</td>\n",
       "      <td>l1</td>\n",
       "      <td>nob/c</td>\n",
       "      <td>406500</td>\n",
       "      <td>neg_amm</td>\n",
       "      <td>not_int</td>\n",
       "      <td>9480.0</td>\n",
       "      <td>EXP</td>\n",
       "      <td>834</td>\n",
       "      <td>CIB</td>\n",
       "      <td>35-44</td>\n",
       "      <td>to_inst</td>\n",
       "      <td>south</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>24893</td>\n",
       "      <td>2019</td>\n",
       "      <td>cf</td>\n",
       "      <td>Male</td>\n",
       "      <td>nopre</td>\n",
       "      <td>type1</td>\n",
       "      <td>p4</td>\n",
       "      <td>l1</td>\n",
       "      <td>nob/c</td>\n",
       "      <td>456500</td>\n",
       "      <td>not_neg</td>\n",
       "      <td>not_int</td>\n",
       "      <td>11880.0</td>\n",
       "      <td>EXP</td>\n",
       "      <td>587</td>\n",
       "      <td>CIB</td>\n",
       "      <td>45-54</td>\n",
       "      <td>not_inst</td>\n",
       "      <td>North</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>24894</td>\n",
       "      <td>2019</td>\n",
       "      <td>cf</td>\n",
       "      <td>Joint</td>\n",
       "      <td>pre</td>\n",
       "      <td>type1</td>\n",
       "      <td>p1</td>\n",
       "      <td>l1</td>\n",
       "      <td>nob/c</td>\n",
       "      <td>296500</td>\n",
       "      <td>not_neg</td>\n",
       "      <td>not_int</td>\n",
       "      <td>10440.0</td>\n",
       "      <td>CRIF</td>\n",
       "      <td>602</td>\n",
       "      <td>EXP</td>\n",
       "      <td>25-34</td>\n",
       "      <td>not_inst</td>\n",
       "      <td>North</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ID  year loan_limit             Gender approv_in_adv loan_type  \\\n",
       "0  24890  2019         cf  Sex Not Available         nopre     type1   \n",
       "1  24891  2019         cf               Male         nopre     type2   \n",
       "2  24892  2019         cf               Male           pre     type1   \n",
       "3  24893  2019         cf               Male         nopre     type1   \n",
       "4  24894  2019         cf              Joint           pre     type1   \n",
       "\n",
       "  loan_purpose Credit_Worthiness business_or_commercial  loan_amount  \\\n",
       "0           p1                l1                  nob/c       116500   \n",
       "1           p1                l1                    b/c       206500   \n",
       "2           p1                l1                  nob/c       406500   \n",
       "3           p4                l1                  nob/c       456500   \n",
       "4           p1                l1                  nob/c       296500   \n",
       "\n",
       "  Neg_ammortization interest_only   income credit_type  Credit_Score  \\\n",
       "0           not_neg       not_int   1740.0         EXP           758   \n",
       "1           not_neg       not_int   4980.0        EQUI           552   \n",
       "2           neg_amm       not_int   9480.0         EXP           834   \n",
       "3           not_neg       not_int  11880.0         EXP           587   \n",
       "4           not_neg       not_int  10440.0        CRIF           602   \n",
       "\n",
       "  co-applicant_credit_type    age submission_of_application Region  Status  \n",
       "0                      CIB  25-34                   to_inst  south       1  \n",
       "1                      EXP  55-64                   to_inst  North       1  \n",
       "2                      CIB  35-44                   to_inst  south       0  \n",
       "3                      CIB  45-54                  not_inst  North       0  \n",
       "4                      EXP  25-34                  not_inst  North       0  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# show data\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check nas\n",
    "df.dropna(how='any', axis=0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------------\n",
      "numeric features: \n",
      " ['loan_amount', 'income', 'Credit_Score']\n",
      "-----------------------------------\n",
      "categorical features: \n",
      " ['loan_limit', 'Gender', 'approv_in_adv', 'loan_type', 'loan_purpose', 'Credit_Worthiness', 'business_or_commercial', 'Neg_ammortization', 'interest_only', 'credit_type', 'co-applicant_credit_type', 'age', 'submission_of_application', 'Region']\n"
     ]
    }
   ],
   "source": [
    "# seperate into numerical and categorical features\n",
    "num_features = list(df.dtypes[(df.dtypes == 'int64') | (df.dtypes == 'float64')].index[2:-1])\n",
    "cat_features = list(df.dtypes[df.dtypes == 'object'].index)\n",
    "label = ['Status']\n",
    "print('-----------------------------------')\n",
    "print('numeric features: \\n', num_features)\n",
    "print('-----------------------------------')\n",
    "print('categorical features: \\n', cat_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Separate into train and test sets\n",
    "X = df[num_features + cat_features]\n",
    "y = df[label].values.reshape(-1)\n",
    "X_train, X_test, y_train, y_test, df_train, df_test = train_test_split(X, y, df, test_size=0.3, stratify=y, random_state=123)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pipeline(steps=[('preprocessor',\n",
      "                 ColumnTransformer(transformers=[('num',\n",
      "                                                  Pipeline(steps=[('imputer',\n",
      "                                                                   SimpleImputer(strategy='median')),\n",
      "                                                                  ('scaler',\n",
      "                                                                   MinMaxScaler())]),\n",
      "                                                  [0, 1, 2]),\n",
      "                                                 ('cat',\n",
      "                                                  Pipeline(steps=[('imputer',\n",
      "                                                                   SimpleImputer(strategy='most_frequent')),\n",
      "                                                                  ('onehotencode',\n",
      "                                                                   OneHotEncoder(drop='first'))]),\n",
      "                                                  [3, 4, 5, 6, 7, 8, 9, 10, 11,\n",
      "                                                   12, 13, 14, 15, 16])])),\n",
      "                ('gbc', LGBMClassifier(random_state=123))])\n"
     ]
    }
   ],
   "source": [
    "# show trained pipeline\n",
    "print(pipe)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((94604, 17), (94604,), (40545, 17), (40545,))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape, y_train.shape, X_test.shape, y_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analysie Model Fairness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Gender</th>\n",
       "      <th>age</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>145003</th>\n",
       "      <td>Sex Not Available</td>\n",
       "      <td>55-64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10348</th>\n",
       "      <td>Joint</td>\n",
       "      <td>35-44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33469</th>\n",
       "      <td>Male</td>\n",
       "      <td>35-44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123975</th>\n",
       "      <td>Sex Not Available</td>\n",
       "      <td>&gt;74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94831</th>\n",
       "      <td>Joint</td>\n",
       "      <td>45-54</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Gender    age\n",
       "145003  Sex Not Available  55-64\n",
       "10348               Joint  35-44\n",
       "33469                Male  35-44\n",
       "123975  Sex Not Available    >74\n",
       "94831               Joint  45-54"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "S_train = X_train[['Gender','age']]\n",
    "S_test = X_test[['Gender','age']]\n",
    "#S_train['age'] = S_train['age'].astype('str')\n",
    "#S_test['age'] = S_test['age'].astype('str')\n",
    "S_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def define_age_group(age_range:str):\n",
    "    age_max = age_range[-2:]\n",
    "    try:\n",
    "        age_max = int(age_max)\n",
    "        if age_max > 50:\n",
    "            return 'older_50'\n",
    "        else:\n",
    "            return 'younger_50'\n",
    "    except:\n",
    "        return 'age_unknown'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Sebas\\AppData\\Local\\Temp\\ipykernel_21608\\398818651.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  S_train['age'] = S_train['age'].apply(lambda x: define_age_group(str(x)))\n",
      "C:\\Users\\Sebas\\AppData\\Local\\Temp\\ipykernel_21608\\398818651.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  S_test['age'] = S_test['age'].apply(lambda x: define_age_group(str(x)))\n"
     ]
    }
   ],
   "source": [
    "# too many age subgroups; define only two i.e. over or under 50\n",
    "S_train['age'] = S_train['age'].apply(lambda x: define_age_group(str(x)))\n",
    "S_test['age'] = S_test['age'].apply(lambda x: define_age_group(str(x))) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Gender</th>\n",
       "      <th>age</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>145003</th>\n",
       "      <td>Sex Not Available</td>\n",
       "      <td>older_50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10348</th>\n",
       "      <td>Joint</td>\n",
       "      <td>younger_50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33469</th>\n",
       "      <td>Male</td>\n",
       "      <td>younger_50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123975</th>\n",
       "      <td>Sex Not Available</td>\n",
       "      <td>older_50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94831</th>\n",
       "      <td>Joint</td>\n",
       "      <td>older_50</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Gender         age\n",
       "145003  Sex Not Available    older_50\n",
       "10348               Joint  younger_50\n",
       "33469                Male  younger_50\n",
       "123975  Sex Not Available    older_50\n",
       "94831               Joint    older_50"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "S_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Metrics by Group:\n",
      "                             selection_rate  accuracy    recall precision  \\\n",
      "Gender            age                                                       \n",
      "Female            older_50         0.133651  0.864061  0.495536  0.950071   \n",
      "                  younger_50       0.129843  0.889118  0.539931  0.987302   \n",
      "Joint             older_50         0.110263  0.890312  0.501325  0.988251   \n",
      "                  younger_50       0.096759  0.921354  0.552486  0.985222   \n",
      "Male              older_50         0.147776   0.84188  0.482359     0.957   \n",
      "                  younger_50       0.127713  0.880506   0.51715  0.970297   \n",
      "Sex Not Available older_50         0.149143  0.840877   0.48299  0.950047   \n",
      "                  younger_50       0.154267  0.850564  0.508233  0.966597   \n",
      "\n",
      "                                    f1  \n",
      "Gender            age                   \n",
      "Female            older_50    0.651345  \n",
      "                  younger_50  0.698092  \n",
      "Joint             older_50    0.665202  \n",
      "                  younger_50  0.707965  \n",
      "Male              older_50    0.641421  \n",
      "                  younger_50  0.674699  \n",
      "Sex Not Available older_50    0.640407  \n",
      "                  younger_50  0.666187  \n"
     ]
    }
   ],
   "source": [
    "# Get metrics by sensitive group from fairlearn\n",
    "print('\\nMetrics by Group:')\n",
    "metrics = {'selection_rate': selection_rate,\n",
    "           'accuracy': accuracy_score,\n",
    "           'recall': recall_score,\n",
    "           'precision': precision_score,\n",
    "           'f1': f1_score}\n",
    "\n",
    "group_metrics = MetricFrame(metrics=metrics,\n",
    "                             y_true=y_test,\n",
    "                             y_pred=pipe.predict(X_test),\n",
    "                             sensitive_features=S_test)\n",
    "\n",
    "print(group_metrics.by_group)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "b6c8fbdf5566a1c94416aebb511efd5a1778a81808a3f2835a0511346cb2e575"
  },
  "kernelspec": {
   "display_name": "Python 3.8.13 ('pycaretenv')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}