Election Finances Project Notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data 102 Final Project: Election Finances\n",
    "\n",
    "By Jessie Houng, Rithika Neti, Ritvik Iyer, and Sunny Shen\n",
    "\n",
    "*Remark*: The Data Cleaning and EDA sections of the notebook are shown in reverse order compared to where they appear in the report because we performed EDA in multiple stages. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1) Data Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Installing required libraries \n",
    "!pip install pandas\n",
    "!pip install numpy\n",
    "!pip install matplotlib\n",
    "!pip install seaborn \n",
    "!pip install fuzzy-match\n",
    "!pip install scipy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import Libraries \n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns \n",
    "from fuzzy_match import match\n",
    "import scipy.stats as stats\n",
    "import warnings\n",
    "warnings.simplefilter('ignore')\n",
    "\n",
    "# Settings\n",
    "pd.set_option('max_columns', None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a) Clean Endorsements Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in Primary Candidates 2018 data\n",
    "dem = pd.read_csv('data/dem_candidates.csv', error_bad_lines=False, encoding='latin-1')\n",
    "rep = pd.read_csv('data/rep_candidates.csv', error_bad_lines=False, encoding='latin-1')\n",
    "\n",
    "# Remove candidates not running in national elections \n",
    "rep = rep[rep['Office Type'] != 'Governor']\n",
    "dem = dem[dem['Office Type'] != 'Governor']\n",
    "\n",
    "# Set column for party affiliations \n",
    "dem['Party'] = 'D'\n",
    "rep['Party'] = 'R'\n",
    "\n",
    "# Fill values for partisan lean for Republican candidates\n",
    "partisan_lean_by_district = dict(zip(dem['District'], dem['Partisan Lean']))\n",
    "rep['Partisan Lean'] = rep['District'].map(partisan_lean_by_district)\n",
    "prop_filled_v1 = len(rep['Partisan Lean'].dropna())/len(rep['Partisan Lean'])\n",
    "\n",
    "# Impute partisan lean for Republican candidates based on Senate, then average of House districts within state\n",
    "senate_leans = dem[dem['District'].str.contains('Senate')][['State', 'Partisan Lean']].drop_duplicates()\n",
    "partisan_lean_by_state = dict(zip(senate_leans['State'], senate_leans['Partisan Lean']))\n",
    "house_leans = dem[['Candidate','State','District','Partisan Lean']][\n",
    "    dem['District'].str.contains('House')].groupby('State').mean().reset_index().dropna()\n",
    "for state, lean in zip(house_leans['State'], house_leans['Partisan Lean']):\n",
    "    if state not in partisan_lean_by_state.keys():\n",
    "        partisan_lean_by_state[state] = lean\n",
    "for index, row in rep.iterrows():\n",
    "    state = rep.loc[index, 'State']\n",
    "    if (pd.isna(rep.loc[index, 'Partisan Lean'])) and (state in partisan_lean_by_state):\n",
    "        rep.loc[index, 'Partisan Lean'] = partisan_lean_by_state[state]\n",
    "prop_filled_v2 = len(rep['Partisan Lean'].dropna())/len(rep['Partisan Lean'])\n",
    "\n",
    "# Identify the states with Republican candidates who still have missing partisan leans \n",
    "missing_state_rep = rep[rep['Partisan Lean'].isna()]['State'].value_counts().index.values\n",
    "\n",
    "# Select the Republican candidates who have missing partisan leans \n",
    "invalid_rep_cand = rep[rep['State'].isin(missing_state_rep)]\n",
    "\n",
    "# Select the Republican candidates who do not have missing partisan leans \n",
    "valid_rep_cand = rep[~(rep['State'].isin(missing_state_rep))]\n",
    "\n",
    "# Deal with missing values in endorsement data\n",
    "final_dem = dem\n",
    "final_rep = valid_rep_cand\n",
    "dem_endorse_support_cols = [x for x in final_dem.columns if 'Support' in x or 'Endorsed' in x]\n",
    "rep_endorse_support_cols = [x for x in final_rep.columns if 'Support' in x or 'Endorsed' in x]\n",
    "endorse_map = {'Yes': 1, 'No': -1, np.nan: 0}\n",
    "final_dem.loc[:, dem_endorse_support_cols] = final_dem.loc[:, dem_endorse_support_cols].replace(endorse_map)\n",
    "final_rep.loc[:, rep_endorse_support_cols] = final_rep.loc[:, rep_endorse_support_cols].replace(endorse_map)\n",
    "party_support = np.hstack([final_dem['Party Support?'].values, final_rep['Rep Party Support?'].values])\n",
    "full_candidates = pd.concat([final_dem, final_rep]).drop(columns=['Party Support?', 'Rep Party Support?'])\n",
    "full_candidates['Party Support?'] = party_support\n",
    "cand_endorse_support_cols = [x for x in full_candidates.columns if 'Support' in x or 'Endorsed' in x]\n",
    "full_candidates.loc[:, cand_endorse_support_cols] = full_candidates.loc[:, cand_endorse_support_cols].fillna(-1)\n",
    "full_candidates = full_candidates.drop(columns=['General Status', 'Race', 'Veteran?',\n",
    "                                                'LGBTQ?', 'Elected Official?', 'Self-Funder?',\n",
    "                                               'STEM?', 'Obama Alum?', 'Guns Sense Candidate?',\n",
    "                                               'Won Primary'])\n",
    "\n",
    "# Set district numbers \n",
    "full_candidates['District_Num'] = [\n",
    "    int(x[-1]) if len(x[-1])<=2 else 0 for x in full_candidates['District'].str.split()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b) Clean FEC Campaigner Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in FEC All Candidates data\n",
    "fin16 = pd.read_csv('data/weball16.txt', sep=\"|\", header=None)\n",
    "fin18 = pd.read_csv('data/weball18.txt',  sep=\"|\", header=None)\n",
    "\n",
    "# Set column names  \n",
    "fin_col = ['CAND_ID', 'CAND_NAME', 'CAND_ICI', 'PTY_CD', 'CAND_PTY_AFFILIATION',\n",
    "           'TTL_RECEIPTS', 'TRANS_FROM_AUTH', 'TTL_DISB', 'TRANS_TO_AUTH', 'COH_BOP', \n",
    "           'COH_COP', 'CAND_CONTRIB', 'CAND_LOANS', 'OTHER_LOANS', 'CAND_LOAN_REPAY', \n",
    "           'OTHER_LOAN_REPAY', 'DEBTS_OWED_BY', 'TTL_INDIV_CONTRIB', 'CAND_OFFICE_ST', \n",
    "           'CAND_OFFICE_DISTRICT', 'SPEC_ELECTION', 'PRIM_ELECTION', 'RUN_ELECTION', \n",
    "           'GEN_ELECTION', 'GEN_ELECTION_PRECENT', 'OTHER_POL_CMTE_CONTRIB',\n",
    "           'POL_PTY_CONTRIB', 'CVG_END_DT', 'INDIV_REFUNDS', 'CMTE_REFUNDS']\n",
    "fin16.columns = fin_col\n",
    "fin18.columns = fin_col\n",
    "\n",
    "# Set year\n",
    "fin18['YEAR'] = 18\n",
    "fin16['YEAR'] = 16\n",
    "\n",
    "# Combine datasets \n",
    "fin = pd.concat([fin16, fin18])\n",
    "\n",
    "# Reformat party names \n",
    "fin['CAND_PTY_AFFILIATION'] = fin['CAND_PTY_AFFILIATION'].map({'REP':'R',\n",
    "                                                               'DEM':'D'})\n",
    "\n",
    "# Reformat candidate names  \n",
    "fin['CAND_NAME'] = [\" \".join(n.lower().title().split(\", \")[::-1]) for n in fin['CAND_NAME']]\n",
    "\n",
    "# Remove missing district numbers and re-code data \n",
    "fin = fin[~fin['CAND_OFFICE_DISTRICT'].isna()]\n",
    "fin['CAND_OFFICE_DISTRICT']= fin['CAND_OFFICE_DISTRICT'].astype(int)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### c) Merge Endorsements Data with FEC Campaigner Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fuzzy match names \n",
    "fuzzy_name = [match.extract(x, fin['CAND_NAME'], limit=1, score_cutoff=0.55) for x in full_candidates['Candidate'].values]\n",
    "fuzzy_name_match = [x[0][0] if x!=[] else '' for x in fuzzy_name]\n",
    "fuzzy_name_match_score = [x[0][1] if x!=[] else '' for x in fuzzy_name]\n",
    "full_candidates['fuzzy_name_match'] = fuzzy_name_match\n",
    "full_candidates['fuzzy_name_match_score'] = fuzzy_name_match_score\n",
    "\n",
    "# Merge datasets based on fuzzy matched names, state, and party and drop duplicates running in special elections\n",
    "cand_fin = pd.merge(fin[['CAND_ID', 'CAND_NAME', 'CAND_OFFICE_DISTRICT', \n",
    "                         'CAND_OFFICE_ST', 'CAND_ICI', 'PTY_CD', \n",
    "                         'CAND_PTY_AFFILIATION', 'TTL_RECEIPTS', \n",
    "                        'TRANS_FROM_AUTH','CAND_CONTRIB',\n",
    "                        'TTL_INDIV_CONTRIB','OTHER_POL_CMTE_CONTRIB']], \n",
    "                    full_candidates, \n",
    "                    left_on=['CAND_NAME','CAND_OFFICE_ST', 'CAND_PTY_AFFILIATION'] , \n",
    "                    right_on=['fuzzy_name_match', 'State', 'Party'])\n",
    "cand_fin = cand_fin[['Candidate', 'CAND_ID', 'District', 'Party', 'State', 'Emily Endorsed?',\n",
    "                     'Partisan Lean', 'TTL_RECEIPTS', 'TRANS_FROM_AUTH', 'CAND_CONTRIB',\n",
    "                     'TTL_INDIV_CONTRIB', 'OTHER_POL_CMTE_CONTRIB', 'Primary %',\n",
    "                     'Biden Endorsed?', 'Warren Endorsed? ', 'Sanders Endorsed?',\n",
    "                     'Our Revolution Endorsed?', 'Justice Dems Endorsed?', \n",
    "                     'PCCC Endorsed?', 'Indivisible Endorsed?', 'WFP Endorsed?',\n",
    "                     'VoteVets Endorsed?', 'No Labels Support?', 'Party Support?', \n",
    "                     'Trump Endorsed?', 'Bannon Endorsed?', 'Great America Endorsed?',\n",
    "                     'NRA Endorsed?', 'Right to Life Endorsed?', 'Susan B. Anthony Endorsed?',\n",
    "                     'Club for Growth Endorsed?', 'Koch Support?', 'House Freedom Support?',\n",
    "                     'Tea Party Endorsed?', 'Main Street Endorsed?', 'Chamber Endorsed?', 'Race Type', 'CAND_ICI']]\n",
    "\n",
    "# Find duplicate candidates who are listed as running in multiple roles (i.e incumbent, open race, challenger)\n",
    "dupes = cand_fin.groupby('Candidate').count()[['CAND_ICI']].reset_index()\n",
    "dupe_names = dupes[dupes['CAND_ICI'] > 1]['Candidate'].values\n",
    "keep_dupe_idx = []\n",
    "for name in dupe_names:\n",
    "    filtered_dupe_cand = cand_fin[cand_fin['Candidate'] == name]\n",
    "    argmax_ind_cont = filtered_dupe_cand.TTL_INDIV_CONTRIB.argmax()\n",
    "    row_idx = filtered_dupe_cand.index[argmax_ind_cont]\n",
    "    keep_dupe_idx.append(row_idx)\n",
    "to_remove = cand_fin[cand_fin['Candidate'].isin(dupe_names)].drop(keep_dupe_idx).index.values\n",
    "cand_fin = cand_fin.drop(to_remove)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### d) Clean FEC Individual Contributions Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in Individual Donor data (warning: these are very large files, ~4 GB each)\n",
    "indiv16 = pd.read_csv('data/indiv16/itcont.txt', sep=\"|\", header=None, low_memory=False)\n",
    "indiv18 = pd.read_csv('data/indiv18/itcont.txt', sep=\"|\", header=None, low_memory=False)\n",
    "indiv_cols = pd.read_csv('data/indiv_header_file.csv').columns\n",
    "\n",
    "# Filter down to campaign years we are interested in \n",
    "P2018_indiv16 = indiv16[indiv16[3]=='P2018']\n",
    "P2018_indiv18 = indiv18[indiv18[3]=='P2018']\n",
    "\n",
    "# Set column names \n",
    "P2018_indiv16.columns = indiv_cols\n",
    "P2018_indiv18.columns = indiv_cols\n",
    "\n",
    "# Concatenate data for 2018 primary cycle together\n",
    "P2018 = pd.concat([P2018_indiv16, P2018_indiv18])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### e) Clean FEC Campaign-Committee Linkages Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in linkage data\n",
    "ccl16 = pd.read_csv('data/ccl16.txt', sep='|', header=None)\n",
    "ccl18 = pd.read_csv('data/ccl18.txt', sep='|', header=None)\n",
    "\n",
    "# Set column names \n",
    "ccl_cols = ['CAND_ID', 'CAND_ELECTION_YR', 'FEC_ELECTION_YR', 'CMTE_ID', 'CMTE_TP', 'CMTE_DSGN', 'LINKAGE_ID']\n",
    "ccl16.columns = ccl_cols\n",
    "ccl18.columns = ccl_cols\n",
    "\n",
    "# Filter down to campaign years we are interested in \n",
    "ccl16 = ccl16[ccl16['CAND_ELECTION_YR']==2018]\n",
    "ccl18 = ccl18[ccl18['CAND_ELECTION_YR']==2018]\n",
    "\n",
    "# Concatenate linkages for 2018 primary cycle together \n",
    "ccl = pd.concat([ccl16, ccl18]).drop_duplicates(subset=['CAND_ID', 'CMTE_ID'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### f) Merge Individual Contributions with Campaign-Committee Linkages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extract Number of Unique individual contributions from 2018 contribution data \n",
    "cmt_count = P2018[['CMTE_ID', 'TRANSACTION_AMT', 'NAME']].groupby(\n",
    "    ['CMTE_ID', 'NAME']).count().reset_index().groupby('CMTE_ID').count().reset_index()\n",
    "cmt_count = cmt_count.drop(columns=['NAME']).rename(columns={'TRANSACTION_AMT': 'Num_Unique_Contributions'})\n",
    "\n",
    "# Merge contributions by committee id to linkage data\n",
    "cand_count = pd.merge(cmt_count, ccl, on = 'CMTE_ID').drop(columns=['CMTE_TP', 'CMTE_DSGN', 'LINKAGE_ID', \n",
    "                                                      'CAND_ELECTION_YR', 'FEC_ELECTION_YR']).groupby(\n",
    "                                                    'CAND_ID').sum().reset_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### g) Merge Candidate Endorsements with Number of Unique Donors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Do an inner join based on candidate id \n",
    "final_candidates = pd.merge(cand_fin, cand_count, on='CAND_ID', how='inner')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2) Exploratory Data Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3) Method #1: Multiple Hypothesis Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_df_from_sample(sample, label):\n",
    "    \"\"\"Constructs a dataframe for a permutation test based on sample values and given label\"\"\"\n",
    "    df = pd.DataFrame()\n",
    "    df['Value'] = sample \n",
    "    df['Label'] = label\n",
    "    return df\n",
    "    \n",
    "def compute_KS_stat(sample_A, sample_B):\n",
    "    \"\"\"Computes the KS statistic between two samples A and B\"\"\"\n",
    "    return stats.ks_2samp(samp1, sample_B)[0]\n",
    "\n",
    "def permute_df_by_label(df):\n",
    "    \"\"\"Permutes labels in a given dataframe for the purposes of permutation testing\"\"\"\n",
    "    shuffled_df = df.copy()\n",
    "    shuffled_df['Label'] = np.random.permutation(df['Label'].values)\n",
    "    return shuffled_df\n",
    "    \n",
    "def ks_2_sample_permutation_test(sample_A, sample_B, num_permutations=5000):\n",
    "    \"\"\"Performs a two-sample permutation test based on KS-statistic for two sample A and B.\n",
    "    Returns an array of test statistics and the corresponding p-value of the test. \n",
    "    H_0: The Cumulative Distribution function of A is equal to Cumulative Distribution function of B\n",
    "    H_1: The CDF of A is not equal to the CDF of B (two-sided)\n",
    "    \"\"\"\n",
    "    obs_stat = compute_KS_stat(sample_A, sample_B)\n",
    "    obs_df_A = make_df_from_sample(sample_A, label='A')\n",
    "    obs_df_B = make_df_from_sample(sample_B, label='B')\n",
    "    combined_df = pd.concat([obs_df_A, obs_df_B])\n",
    "    test_statistics = np.array([])\n",
    "    for _ in np.arange(num_permutations):\n",
    "        permuted_df = permute_df_by_label(combined_df)\n",
    "        df_A = permuted_df[permuted_df['Label'] == 'A']\n",
    "        df_B = permuted_df[permuted_df['Label'] == 'B']\n",
    "        ks_stat = compute_KS_stat(df_A['Value'].values, df_B['Value'].values)\n",
    "        test_statistics = np.append(test_statistics, ks_stat)\n",
    "    p_value = np.count_nonzero(np.abs(test_statistics) > obs_stat)/num_permutations\n",
    "    return test_statistics, obs_stat, p_value\n",
    "        \n",
    "\n",
    "def ks_test(sample_A, sample_B):\n",
    "    \"\"\"Performs a two-sample KS-test between sample A and B and returns p-value\n",
    "    H_0: The CDF of A is equal to the CDF of B\n",
    "    H_1: The CDFs of A and B are not equal (two-sided)\n",
    "    \"\"\"\n",
    "    return stats.ks_2samp(samp1, sample_B)[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extract two samples \n",
    "\n",
    "tbl1 = valid_rep_cand.groupby('District').count()['Candidate'].reset_index().groupby('Candidate').count().reset_index()\n",
    "samp1 = np.repeat(tbl1['Candidate'].values, tbl1['District'].values)\n",
    "tbl2 = invalid_rep_cand.groupby('District').count()['Candidate'].reset_index().groupby('Candidate').count().reset_index()\n",
    "samp2 = np.repeat(tbl2['Candidate'].values, tbl2['District'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.22085261842677528"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ks_test(samp1, samp2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This p-value is too big! This is because the KS-test assumes A and B come from continuous distributions, so it is is less powerful in cases where A and B are discrete. Instead, we take a permutation approach as follows: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "data, obs, p = ks_2_sample_permutation_test(samp1, samp2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The p-value was 0.018\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAEICAYAAABGaK+TAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAUX0lEQVR4nO3debRlZX3m8e8TBpFBECkT5oLooiV2Brs0UWzFgB0RtLSXq0MSJ6IhrG6nBFfEKB2cEjK03Ul3JoIKARtFgnGgTTDaNBoBLUyhTCoialkMVQgyxASQX/+x95VThzucqeret/h+1jrrnrPH9z3vPs9593D2TVUhSWrPjyx3ASRJkzHAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYDPQJK/SHLqjJZ1UJJ7kuzQv74kyatnsex+eZ9I8opZLW+M9b4zyeYkt2zrdc/apO093LbLJclNSY7eiss/Lcm5W2v540iyOkkl2bF/PdPP03IzwJfQb+zfT3J3kjuTfC7JSUl++N5V1UlV9Y4Rl7XoB6eqvlVVu1fVD2ZQ9od9kKrqmKo6e9plj1mOA4GTgcOr6sfmGX9kkg0Dr3dOcmGSf0zymCR7JXlvklv6dvhqkjctsr5XJbm+n/bWJBcl2aMfd1aSd45R9lcm+ezgsEnbe5Ztu1IMt91WWP5ZfQA/bWDYE5L4AxYM8FG9oKr2AA4GTgfeBLxn1iuZ6yVshw4Gbq+q25aaMMmjgAuBvYD/UFV3Af8d2B14ErAn8ELg6wvM/2zgd4Ff6tvsScD5M6jDI94ybp/fBUb+0n1EqSofizyAm4Cjh4Y9DXgQeHL/+izgnf3zfYCPA3fSbXifofuiPKef5/vAPcBvAauBAl4FfAu4dGDYjv3yLgF+D/g88D3gI8De/bgjgQ3zlRd4HnAfcH+/vqsGlvfq/vmPAG8FvgncBvw1sGc/bq4cr+jLthl4yyLv0579/Jv65b21X/7RfZ0f7Mtx1jzzHglsAHYFLgb+Hnj0wPirgReN2F5vBP52gXEn9u/HfX1ZPtYPP4XuC+Fu4Frgxf3wJwH/Avygn/7OGbX3XNvuDbwP2AjcsUi5TwPOHXg93zbyDuAf+zpcDOwzMP3L+ja5HXgLA9t0X9a5+t9O92W399B6frh9DpVrt6G2vQfYry/v+f32cDdwDbBmYL79gL/pt5VvAK9bpD3PAt4N3AI8ux/2BKAW+owOvl8LvFevXu5cmdXDHvgEqurzdIHz7+cZfXI/bhXwo8Bvd7PUy+g+BC+objf6DwbmeTZdWPzCAqt8OfCrdBv+A8CfjFDGv6PriX6wX99PzTPZK/vHc4BD6Xq5/2tommcChwFHAf81yZMWWOX/pAvxQ/v6vBw4oar+ATgG2NiX45ULzP8o4BN0gfnCqvr+wLjLgXclOSHJExesdOcK4BeSvC3JEX2PHoCqOgN4P/AHfVle0I/6Ol1b7gm8DTg3yb5VdR1wEnBZP/1e86xvkvaecw7dl9ZPAI+n29OY1C8DJ/TL2Znui4wkhwN/Thfi+wGPAw4YmO91wIvo2mw/ui+SPx1a9rzbZ1Xdy5Ztu3tVbexHvxD4AN2e1Efpt6v+0OPHgKuA/em2qzckWWjbB/hnum35XUu+C48wBvjkNtL1oIbdD+wLHFxV91fVZ6r/6l/EaVV171BoDTqnqq7uPzCnAv9pRifCfgV4d1XdWFX3AG8Gjh/aVX5bVX2/qq6i+9A97IugL8svAm+uqrur6ibgv9GFxqj2AJ4OnF1V/zo07rV0wfsa4NokNyQ5Zr6FVNVngP8IPAW4CLg9ybsXe7+q6kNVtbGqHqyqDwJfo9vLGsUk7U2SfenC76SquqOf9/+NuM75vK+qvtpvQ+cDP90Pfwnw8aq6tH9fT6XrMc/5dbo9qw39+NOAlwxtA0ttn/P5bFX9n+qO95/DQ9vNU4FVVfX2qrqvqm4E/go4fonl/SVw0ELt/khlgE9uf7pd5mF/CNwAXJzkxiSnjLCsb48x/pvATnS77tPar1/e4LJ3pOtJzhm8auSf6Xrpw/ah6/UNL2v/Mcqyme5DfPZwb6z/Avndqvp3dD3I84EPJZnvC5Sq+kTfu94bWEu3l7HglQdJXp5kfX+S+k7gyYz+/k7S3gAHAt+tqjtGnH4pC7XTfgxsP30n4PaBaQ8GPjxQ9+voDhkNbgNLbZ+jlGeX/kvhYGC/ufX16/ztofU9TP/l8o7+kQnKs10ywCeQ5Kl04fTZ4XF9D/TkqjoUeAHwm0mOmhu9wCKX6rEdOPD8ILpe32bgXrpd8Lly7UC3Kz/qcjfSfaAGl/0AcOsS8w3b3JdpeFnfGWchVXUh8GvABUmes8A0d9HtTu8GHLLE8h6sqk8Bn6YLZRh6T5IcTNcDfA3wuP4wydU8FBKLvocTtjd0obh3kr0WW35vi3YGHnYlzyJuZmD7SbIr3ZfgYDmOqaq9Bh67VNVg2y1Wj3GvBvk28I2h9e1RVc8fYd730R3mevHQ8Gnen6YZ4GPoL2k7ju7Y3rlV9eV5pjmuv8wpwF10vZm5y8ZupTtGPK6XJjm8//C9Hbig3zX9Kl3P5tgkO9GdOHzUwHy3AqsHL3kcch7wG0kOSbI7Dx0zf2CcwvVlOZ/uOPUefSj+JjD2tcBVdR5dmH4kyREASU5N8tT+8sJdgNfTnTT8yvD8SdYmOT7JY9N5Gt0x3Mv7SYbbYDe6ENrUz38CD4X93PQHJNl5vvJO2t5VdTPdMf8/68u6U5JnLfC2rAeele468j3pDnWN6gLguCTP7Ovwdrb83P8FXbsd3NdnVZK1Yyz/VuBxfblG8XngriRvSvLoJDskeXLfKVpUv12eRncV2KD1dIf+dkqyhu6w0SOCAT6ajyW5m6738Ba6s+InLDDtE4F/oDsjfxnwZ1V1ST/u94C39ruObxxj/efQnY2/BdiF7sQTVfU94D8DZ9L1du+lO6E250P939uTfHGe5b63X/aldFcD/Avd8eZJvLZf/410eyb/u1/+2Kq7Tv1k4KI+gIuu97WZbq/hucCx/XH7YXfQ9eK/Rheo5wJ/WFXv78e/Bzi8b4O/rapr6Y7XX0YXRv+W7mqOOZ+mu4riliSb51nfNO39Mro9l+vprgJ6wwLvxyeBDwJfAq6ku+plJFV1DfBf6NrjZrr3Z3Ab+WO6k4wX99v45cDPjrH86+k6Ajf29dxviel/QLen8tN029xmuu131C+A8/p6DDoV+HG6ur2Nrq6PCBnhfIskaQWyBy5JjTLAJalRBrgkNcoAl6RGbdOb0+yzzz61evXqbblKafa+0l+9eNhhy1sOPWJceeWVm6tq1fDwbRrgq1evZt26ddtyldLsHXlk9/eSS5azFHoESfLN+YZ7CEWSGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhq1TX+J2arVp1y0LOu96fRjl2W9ktpgD1ySGmWAS1KjDHBJapQBLkmNWjLAk7w3yW1Jrh4YtneSTyb5Wv/3sVu3mJKkYaP0wM8Cnjc07BTgU1X1ROBT/WtJ0ja0ZIBX1aXAd4cGrwXO7p+fDbxotsWSJC1l0mPgP1pVNwP0fx+/0IRJTkyyLsm6TZs2Tbg6SdKwrX4Ss6rOqKo1VbVm1aqH/Us3SdKEJg3wW5PsC9D/vW12RZIkjWLSAP8o8Ir++SuAj8ymOJKkUY1yGeF5wGXAYUk2JHkVcDrw3CRfA57bv5YkbUNL3syqqn5pgVFHzbgskqQx+EtMSWqUt5NdwbyNraTF2AOXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWrUVAGe5DeSXJPk6iTnJdllVgWTJC1u4gBPsj/wOmBNVT0Z2AE4flYFkyQtbtpDKDsCj06yI7ArsHH6IkmSRjFxgFfVd4A/Ar4F3Ax8r6ouHp4uyYlJ1iVZt2nTpslLKknawjSHUB4LrAUOAfYDdkvy0uHpquqMqlpTVWtWrVo1eUklSVuY5hDK0cA3qmpTVd0PXAg8YzbFkiQtZZoA/xbwc0l2TRLgKOC62RRLkrSUaY6BXwFcAHwR+HK/rDNmVC5J0hJ2nGbmqvod4HdmVBZJ0hj8JaYkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDVqql9iavu0+pSLlm3dN51+7LKtW2qNPXBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDVqqgBPsleSC5Jcn+S6JE+fVcEkSYub9r/S/zHwd1X1kiQ7A7vOoEySpBFMHOBJHgM8C3glQFXdB9w3m2JJkpYyzSGUQ4FNwPuS/FOSM5PsNjxRkhOTrEuybtOmTVOsTpI0aJoA3xF4CvDnVfUzwL3AKcMTVdUZVbWmqtasWrVqitVJkgZNE+AbgA1VdUX/+gK6QJckbQMTB3hV3QJ8O8lh/aCjgGtnUipJ0pKmvQrltcD7+ytQbgROmL5IkqRRTBXgVbUeWDObokiSxuEvMSWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJatTUAZ5khyT/lOTjsyiQJGk0s+iBvx64bgbLkSSNYaoAT3IAcCxw5myKI0ka1bQ98P8B/Bbw4EITJDkxybok6zZt2jTl6iRJcyYO8CTHAbdV1ZWLTVdVZ1TVmqpas2rVqklXJ0kaMk0P/AjghUluAj4A/HySc2dSKknSkiYO8Kp6c1UdUFWrgeOBT1fVS2dWMknSorwOXJIateMsFlJVlwCXzGJZkqTR2AOXpEYZ4JLUqJkcQtkWVp9y0XIXQZJWFHvgktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJatTEAZ7kwCT/N8l1Sa5J8vpZFkyStLgdp5j3AeDkqvpikj2AK5N8sqqunVHZJEmLmLgHXlU3V9UX++d3A9cB+8+qYJKkxU3TA/+hJKuBnwGumGfcicCJAAcddNAsVqft2OpTLlqW9d50+rHLsl5pGlOfxEyyO/A3wBuq6q7h8VV1RlWtqao1q1atmnZ1kqTeVAGeZCe68H5/VV04myJJkkYxzVUoAd4DXFdV755dkSRJo5imB34E8DLg55Os7x/Pn1G5JElLmPgkZlV9FsgMyyJJGoO/xJSkRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUqJncD1xq3Tj3If/AjbcDcPyM7l3uvcg1KXvgktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcr7gUvLbJx7kc+S9yFvnz1wSWqUAS5JjTLAJalRBrgkNWqqAE/yvCRfSXJDklNmVShJ0tImvgolyQ7AnwLPBTYAX0jy0aq6dlaFk7T1LNfVL+AVMLMyTQ/8acANVXVjVd0HfABYO5tiSZKWMs114PsD3x54vQH42eGJkpwInNi/vCfJV6ZY50qxD7B5uQsxY9ZpRE+fe/L7x8160UvZbtoov//Dp9tNnQZsjTodPN/AaQI88wyrhw2oOgM4Y4r1rDhJ1lXVmuUuxyxZp5Vve6sPWKdpTXMIZQNw4MDrA4CN0xVHkjSqaQL8C8ATkxySZGfgeOCjsymWJGkpEx9CqaoHkrwG+HtgB+C9VXXNzEq2sm1Xh4R61mnl297qA9ZpKql62GFrSVID/CWmJDXKAJekRhngQ5a6PUA6f9KP/1KSpwyMuynJl5OsT7Ju25Z8fiPU598kuSzJvyZ54zjzLpcp67Ti2ghGqtOv9Nvbl5J8LslPjTrvcpiyPq220dq+PuuTrEvyzFHnnVhV+egfdCdjvw4cCuwMXAUcPjTN84FP0F0H/3PAFQPjbgL2We56jFmfxwNPBd4FvHGceVur00psozHq9Azgsf3zY+a2u5XYTtPUp/E22p2Hziv+JHD91m4je+BbGuX2AGuBv67O5cBeSfbd1gUd0ZL1qarbquoLwP3jzrtMpqnTSjVKnT5XVXf0Ly+n+93FSPMug2nqs1KNUqd7qk9sYDce+mHjVmsjA3xL890eYP8xping4iRX9rcQWG6j1GdrzLs1TVuuldZGMH6dXkW3FzjJvNvCNPWBhtsoyYuTXA9cBPzqOPNOwv+JuaVRbg+w2DRHVNXGJI8HPpnk+qq6dKYlHM9ItzvYCvNuTdOWa6W1EYxRpyTPoQu8ueOrK7GdpqkPNNxGVfVh4MNJngW8Azh61HknYQ98S6PcHmDBaapq7u9twIfpdp2W0zS3O1ipt0qYqlwrsI1gxDol+UngTGBtVd0+zrzb2DT1abqN5vRfOD+eZJ9x5x3Lcp8cWEkPuj2SG4FDeOhkw08MTXMsW57E/Hw/fDdgj4HnnwOet9LrMzDtaWx5EnPkeRuq04prozG2u4OAG4BnTPp+NFKfltvoCTx0EvMpwHf6nNhqbbSsb8pKfNBdZfJVurPGb+mHnQSc1D8P3T+y+DrwZWBNP/zQvmGuAq6Zm3e5HyPU58foegh3AXf2zx+z0Lwr4TFpnVZqG41YpzOBO4D1/WPdYvMu92PS+jTeRm/qy7weuAx45tZuI39KL0mN8hi4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmN+v90TtKZmcO81QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.title('Distribution of KS Statistic under the Null')\n",
    "plt.hist(data, density=True)\n",
    "plt.axvline(x=obs, color='r');\n",
    "print(f'The p-value was {p}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4) Method #2: Bayesian Inference "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a) Feature Extraction "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_candidates = pd.read_csv('data/all_candidates.csv').drop_duplicates(subset=['Candidate'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Candidate</th>\n",
       "      <th>CAND_ID</th>\n",
       "      <th>District</th>\n",
       "      <th>Party</th>\n",
       "      <th>State</th>\n",
       "      <th>Emily Endorsed?</th>\n",
       "      <th>Partisan Lean</th>\n",
       "      <th>TTL_RECEIPTS</th>\n",
       "      <th>TRANS_FROM_AUTH</th>\n",
       "      <th>CAND_CONTRIB</th>\n",
       "      <th>TTL_INDIV_CONTRIB</th>\n",
       "      <th>OTHER_POL_CMTE_CONTRIB</th>\n",
       "      <th>Primary %</th>\n",
       "      <th>Biden Endorsed?</th>\n",
       "      <th>Warren Endorsed?</th>\n",
       "      <th>Sanders Endorsed?</th>\n",
       "      <th>Our Revolution Endorsed?</th>\n",
       "      <th>Justice Dems Endorsed?</th>\n",
       "      <th>PCCC Endorsed?</th>\n",
       "      <th>Indivisible Endorsed?</th>\n",
       "      <th>WFP Endorsed?</th>\n",
       "      <th>VoteVets Endorsed?</th>\n",
       "      <th>No Labels Support?</th>\n",
       "      <th>Party Support?</th>\n",
       "      <th>Trump Endorsed?</th>\n",
       "      <th>Bannon Endorsed?</th>\n",
       "      <th>Great America Endorsed?</th>\n",
       "      <th>NRA Endorsed?</th>\n",
       "      <th>Right to Life Endorsed?</th>\n",
       "      <th>Susan B. Anthony Endorsed?</th>\n",
       "      <th>Club for Growth Endorsed?</th>\n",
       "      <th>Koch Support?</th>\n",
       "      <th>House Freedom Support?</th>\n",
       "      <th>Tea Party Endorsed?</th>\n",
       "      <th>Main Street Endorsed?</th>\n",
       "      <th>Chamber Endorsed?</th>\n",
       "      <th>Race Type</th>\n",
       "      <th>CAND_ICI</th>\n",
       "      <th>Num_Unique_Contributions</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Wendy Rogers</td>\n",
       "      <td>H2AZ09035</td>\n",
       "      <td>U.S. House Arizona District 1</td>\n",
       "      <td>R</td>\n",
       "      <td>AZ</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-24.559999</td>\n",
       "      <td>1423836.68</td>\n",
       "      <td>0.00</td>\n",
       "      <td>698.53</td>\n",
       "      <td>1302250.00</td>\n",
       "      <td>83750.00</td>\n",
       "      <td>43.400000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Regular</td>\n",
       "      <td>C</td>\n",
       "      <td>382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Michael Eggman</td>\n",
       "      <td>H4CA10075</td>\n",
       "      <td>U.S. House California District 10</td>\n",
       "      <td>D</td>\n",
       "      <td>CA</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.620000</td>\n",
       "      <td>1639589.92</td>\n",
       "      <td>16038.75</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1195080.49</td>\n",
       "      <td>375444.76</td>\n",
       "      <td>21.340000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>Regular</td>\n",
       "      <td>C</td>\n",
       "      <td>246</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Wendy Reed</td>\n",
       "      <td>H6CA23206</td>\n",
       "      <td>U.S. House California District 23</td>\n",
       "      <td>D</td>\n",
       "      <td>CA</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-25.379999</td>\n",
       "      <td>40357.40</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1887.97</td>\n",
       "      <td>22601.93</td>\n",
       "      <td>7500.00</td>\n",
       "      <td>34.250000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>Regular</td>\n",
       "      <td>C</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Justin Fareed</td>\n",
       "      <td>H4CA24142</td>\n",
       "      <td>U.S. House California District 24</td>\n",
       "      <td>R</td>\n",
       "      <td>CA</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-7.729107</td>\n",
       "      <td>2353070.71</td>\n",
       "      <td>0.00</td>\n",
       "      <td>739.26</td>\n",
       "      <td>2096376.98</td>\n",
       "      <td>231850.00</td>\n",
       "      <td>78.370000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Regular</td>\n",
       "      <td>O</td>\n",
       "      <td>342</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Bryan Caforio</td>\n",
       "      <td>H6CA25177</td>\n",
       "      <td>U.S. House California District 25</td>\n",
       "      <td>D</td>\n",
       "      <td>CA</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>2.020000</td>\n",
       "      <td>1706520.29</td>\n",
       "      <td>22679.24</td>\n",
       "      <td>1974.17</td>\n",
       "      <td>1173006.17</td>\n",
       "      <td>456702.21</td>\n",
       "      <td>38.259998</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>Regular</td>\n",
       "      <td>C</td>\n",
       "      <td>1078</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Candidate    CAND_ID                           District Party State  \\\n",
       "0    Wendy Rogers  H2AZ09035      U.S. House Arizona District 1     R    AZ   \n",
       "1  Michael Eggman  H4CA10075  U.S. House California District 10     D    CA   \n",
       "2      Wendy Reed  H6CA23206  U.S. House California District 23     D    CA   \n",
       "3   Justin Fareed  H4CA24142  U.S. House California District 24     R    CA   \n",
       "4   Bryan Caforio  H6CA25177  U.S. House California District 25     D    CA   \n",
       "\n",
       "   Emily Endorsed?  Partisan Lean  TTL_RECEIPTS  TRANS_FROM_AUTH  \\\n",
       "0             -1.0     -24.559999    1423836.68             0.00   \n",
       "1             -1.0       0.620000    1639589.92         16038.75   \n",
       "2              0.0     -25.379999      40357.40             0.00   \n",
       "3             -1.0      -7.729107    2353070.71             0.00   \n",
       "4             -1.0       2.020000    1706520.29         22679.24   \n",
       "\n",
       "   CAND_CONTRIB  TTL_INDIV_CONTRIB  OTHER_POL_CMTE_CONTRIB  Primary %  \\\n",
       "0        698.53         1302250.00                83750.00  43.400000   \n",
       "1          0.00         1195080.49               375444.76  21.340000   \n",
       "2       1887.97           22601.93                 7500.00  34.250000   \n",
       "3        739.26         2096376.98               231850.00  78.370000   \n",
       "4       1974.17         1173006.17               456702.21  38.259998   \n",
       "\n",
       "   Biden Endorsed?  Warren Endorsed?   Sanders Endorsed?  \\\n",
       "0             -1.0               -1.0               -1.0   \n",
       "1              0.0                0.0                0.0   \n",
       "2              0.0                0.0                0.0   \n",
       "3             -1.0               -1.0               -1.0   \n",
       "4              0.0                0.0                0.0   \n",
       "\n",
       "   Our Revolution Endorsed?  Justice Dems Endorsed?  PCCC Endorsed?  \\\n",
       "0                      -1.0                    -1.0            -1.0   \n",
       "1                      -1.0                    -1.0             0.0   \n",
       "2                       0.0                     0.0             0.0   \n",
       "3                      -1.0                    -1.0            -1.0   \n",
       "4                      -1.0                     1.0             0.0   \n",
       "\n",
       "   Indivisible Endorsed?  WFP Endorsed?  VoteVets Endorsed?  \\\n",
       "0                   -1.0           -1.0                -1.0   \n",
       "1                   -1.0            0.0                 0.0   \n",
       "2                    0.0            0.0                 0.0   \n",
       "3                   -1.0           -1.0                -1.0   \n",
       "4                    0.0            0.0                 0.0   \n",
       "\n",
       "   No Labels Support?  Party Support?  Trump Endorsed?  Bannon Endorsed?  \\\n",
       "0                   0               1              0.0               0.0   \n",
       "1                   0              -1             -1.0              -1.0   \n",
       "2                   0               0             -1.0              -1.0   \n",
       "3                   0               1              0.0               0.0   \n",
       "4                   0              -1             -1.0              -1.0   \n",
       "\n",
       "   Great America Endorsed?  NRA Endorsed?  Right to Life Endorsed?  \\\n",
       "0                      0.0            0.0                      0.0   \n",
       "1                     -1.0           -1.0                     -1.0   \n",
       "2                     -1.0           -1.0                     -1.0   \n",
       "3                      0.0            1.0                      0.0   \n",
       "4                     -1.0           -1.0                     -1.0   \n",
       "\n",
       "   Susan B. Anthony Endorsed?  Club for Growth Endorsed?  Koch Support?  \\\n",
       "0                         0.0                       -1.0            0.0   \n",
       "1                        -1.0                       -1.0           -1.0   \n",
       "2                        -1.0                       -1.0           -1.0   \n",
       "3                         0.0                        0.0            0.0   \n",
       "4                        -1.0                       -1.0           -1.0   \n",
       "\n",
       "   House Freedom Support?  Tea Party Endorsed?  Main Street Endorsed?  \\\n",
       "0                     0.0                 -1.0                    0.0   \n",
       "1                    -1.0                 -1.0                   -1.0   \n",
       "2                    -1.0                 -1.0                   -1.0   \n",
       "3                     0.0                  0.0                    0.0   \n",
       "4                    -1.0                 -1.0                   -1.0   \n",
       "\n",
       "   Chamber Endorsed? Race Type CAND_ICI  Num_Unique_Contributions  \n",
       "0                0.0   Regular        C                       382  \n",
       "1               -1.0   Regular        C                       246  \n",
       "2               -1.0   Regular        C                        14  \n",
       "3                0.0   Regular        O                       342  \n",
       "4               -1.0   Regular        C                      1078  "
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    U.S. House Arizona District 1\n",
      "Name: District, dtype: object 0    R\n",
      "Name: Party, dtype: object\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Can only compare identically-labeled Series objects",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-59-106df1b10652>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      9\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mnum_competitors\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 11\u001b[1;33m \u001b[0mfinal_candidates\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'Candidate'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mget_num_competitors\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\series.py\u001b[0m in \u001b[0;36mapply\u001b[1;34m(self, func, convert_dtype, args, **kwds)\u001b[0m\n\u001b[0;32m   4198\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4199\u001b[0m                 \u001b[0mvalues\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobject\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_values\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 4200\u001b[1;33m                 \u001b[0mmapped\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmap_infer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mconvert\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mconvert_dtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   4201\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4202\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmapped\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmapped\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mSeries\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\lib.pyx\u001b[0m in \u001b[0;36mpandas._libs.lib.map_infer\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32m<ipython-input-59-106df1b10652>\u001b[0m in \u001b[0;36mget_num_competitors\u001b[1;34m(cand_name, cand_df, ref_df)\u001b[0m\n\u001b[0;32m      6\u001b[0m     \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdistrict_name\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparty\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m     num_competitors = ref_df[(\n\u001b[1;32m----> 8\u001b[1;33m         ref_df['District'] == district_name) & (ref_df['Party'] == party)].shape[0] - 1\n\u001b[0m\u001b[0;32m      9\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mnum_competitors\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\ops\\common.py\u001b[0m in \u001b[0;36mnew_method\u001b[1;34m(self, other)\u001b[0m\n\u001b[0;32m     63\u001b[0m         \u001b[0mother\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mitem_from_zerodim\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mother\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     64\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 65\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mother\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     66\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     67\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mnew_method\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\ops\\__init__.py\u001b[0m in \u001b[0;36mwrapper\u001b[1;34m(self, other)\u001b[0m\n\u001b[0;32m    363\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    364\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mother\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mABCSeries\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mand\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_indexed_same\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mother\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 365\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Can only compare identically-labeled Series objects\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    366\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    367\u001b[0m         \u001b[0mlvalues\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mextract_array\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mextract_numpy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Can only compare identically-labeled Series objects"
     ]
    }
   ],
   "source": [
    "def get_num_competitors(cand_name, cand_df=final_candidates, ref_df=all_candidates):\n",
    "    \"\"\"For a specified candidate name in the candidate dataframe, gets the number of \n",
    "    opposing candidates in the same district with respect to the reference dataframe.\"\"\"\n",
    "    district_name = cand_df[cand_df['Candidate'] == cand_name]#.loc[0, 'District']\n",
    "    party = cand_df[cand_df['Candidate'] == cand_name]#.loc[0, 'Party']\n",
    "    print(district_name, party)\n",
    "    num_competitors = ref_df[(\n",
    "        ref_df['District'] == district_name) & (ref_df['Party'] == party)].shape[0] - 1\n",
    "    return num_competitors \n",
    "\n",
    "final_candidates['Candidate'].apply(get_num_competitors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       2\n",
       "1       0\n",
       "2       0\n",
       "3       4\n",
       "4       2\n",
       "       ..\n",
       "744     0\n",
       "745     0\n",
       "746     0\n",
       "747     0\n",
       "748    10\n",
       "Length: 749, dtype: int64"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Initialize candidate feature dataframe\n",
    "cand_features = pd.DataFrame()\n",
    "\n",
    "# Read in final candidate data\n",
    "final_candidates = pd.read_csv('data/final_candidates.csv')\n",
    "\n",
    "# Set candidate names, districts, parties, and political leans based on final candidate data\n",
    "cand_features['Candidate'] = final_candidates['Candidate']\n",
    "cand_features['District'] =  final_candidates['District']\n",
    "cand_features['Party'] =  final_candidates['Party']\n",
    "cand_features['Partisan Lean'] =  final_candidates['Partisan Lean']\n",
    "\n",
    "# Find the number of positive endorsements for each candidate\n",
    "dem_endorse_support_cols = [x for x in final_candidates.columns if 'Support' in x or 'Endorsed' in x]\n",
    "rep_endorse_support_cols = [x for x in final_candidates.columns if 'Support' in x or 'Endorsed' in x]\n",
    "endorse_cols = dem_endorse_support_cols + rep_endorse_support_cols\n",
    "pos_endorsements = final_candidates.loc[:, endorse_cols].applymap(\n",
    "    lambda x: 0 if x == -1 else x).apply(np.sum, axis=1).apply(int)\n",
    "cand_features['Positive Endorsement Count'] =  pos_endorsements\n",
    "\n",
    "# Find the number of competitors each candidate is opposing in their primary\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "\n",
    "cand_features['Number of Competitors'] = ...\n",
    "cand_features['Number of Unique Donors'] = final_candidates['Num_Unique_Contributions']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}