code/scripts/mimic-omop_hot_encoding.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "590b9824-5db4-49c7-9f1f-f7395d376c5d",
   "metadata": {},
   "source": [
    "Reproducible conda environment used: file db_analysis-linux-64.lock "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ee1f8bb2-1e0d-48d4-b149-207139dad3f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "from pathlib import Path\n",
    "import os\n",
    "sys.path.insert(0,str(Path(os.path.abspath('')).parent))\n",
    "from proj_config import *\n",
    "from sklearn import preprocessing as preproc\n",
    "import scipy as sp\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import gc\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "import hot_encoding as oh\n",
    "import concepts_toolbox\n",
    "import dtypecheck as dc\n",
    "import pandas_utils\n",
    "import csv\n",
    "import timeit\n",
    "from importlib import reload\n",
    "\n",
    "datadir = get_datadir() / \"mimic-omop/pre-processed/\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee1a3349-6436-4f64-b768-7f4ae19f0bce",
   "metadata": {},
   "source": [
    "# Diagnoses one hot encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e3dd0d5f-ab5e-41cd-84cf-fdfc7d530fef",
   "metadata": {},
   "outputs": [],
   "source": [
    "diagnoses = pd.read_csv(datadir/\"diagnoses.csv.gz\",usecols=[\"visit_occurrence_id\",\"icd9_code\"])\n",
    "#diagnoses = diagnoses.sample(n=100000)\n",
    "icd9_concepts = pd.read_csv(datadir/\"icd9_concepts.csv.gz\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "af9b80f2-e0d5-48a9-b112-dc4ffbb8416c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>concept_id</th>\n",
       "      <th>concept_name</th>\n",
       "      <th>domain_id</th>\n",
       "      <th>concept_class_id</th>\n",
       "      <th>standard_concept</th>\n",
       "      <th>concept_code</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>44829696</td>\n",
       "      <td>Cholera</td>\n",
       "      <td>Condition</td>\n",
       "      <td>3-dig nonbill code</td>\n",
       "      <td>NaN</td>\n",
       "      <td>001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>44835564</td>\n",
       "      <td>Cholera due to vibrio cholerae</td>\n",
       "      <td>Condition</td>\n",
       "      <td>4-dig billing code</td>\n",
       "      <td>NaN</td>\n",
       "      <td>001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>44823922</td>\n",
       "      <td>Cholera due to vibrio cholerae el tor</td>\n",
       "      <td>Condition</td>\n",
       "      <td>4-dig billing code</td>\n",
       "      <td>NaN</td>\n",
       "      <td>001.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>44827441</td>\n",
       "      <td>Cholera, unspecified</td>\n",
       "      <td>Condition</td>\n",
       "      <td>4-dig billing code</td>\n",
       "      <td>NaN</td>\n",
       "      <td>001.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>44829697</td>\n",
       "      <td>Typhoid and paratyphoid fevers</td>\n",
       "      <td>Condition</td>\n",
       "      <td>3-dig nonbill code</td>\n",
       "      <td>NaN</td>\n",
       "      <td>002</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   concept_id                           concept_name  domain_id  \\\n",
       "0    44829696                                Cholera  Condition   \n",
       "1    44835564         Cholera due to vibrio cholerae  Condition   \n",
       "2    44823922  Cholera due to vibrio cholerae el tor  Condition   \n",
       "3    44827441                   Cholera, unspecified  Condition   \n",
       "4    44829697         Typhoid and paratyphoid fevers  Condition   \n",
       "\n",
       "     concept_class_id  standard_concept concept_code  \n",
       "0  3-dig nonbill code               NaN          001  \n",
       "1  4-dig billing code               NaN        001.0  \n",
       "2  4-dig billing code               NaN        001.1  \n",
       "3  4-dig billing code               NaN        001.9  \n",
       "4  3-dig nonbill code               NaN          002  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "icd9_concepts.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97a1b358-4339-4c19-aa7f-4e4d95764645",
   "metadata": {},
   "source": [
    "## Using Sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "95f21fe4-05a2-42a5-9d61-6568805f4175",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "The truth value of a Series is ambiguous. Use a.empty, a.bool(), a.item(), a.any() or a.all().",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_8533/3397305825.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0menc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpreproc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOneHotEncoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcategories\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0micd9_concepts\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcept_code\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0menc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdiagnoses\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0micd9_code\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/envs/pmsi_db_analysis/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y)\u001b[0m\n\u001b[1;32m    459\u001b[0m         \"\"\"\n\u001b[1;32m    460\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_validate_keywords\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 461\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle_unknown\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhandle_unknown\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mforce_all_finite\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"allow-nan\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    462\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdrop_idx_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_compute_drop_idx\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    463\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pmsi_db_analysis/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py\u001b[0m in \u001b[0;36m_fit\u001b[0;34m(self, X, handle_unknown, force_all_finite)\u001b[0m\n\u001b[1;32m     80\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_features_in_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mn_features\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     81\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 82\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcategories\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m\"auto\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     83\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcategories\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mn_features\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     84\u001b[0m                 raise ValueError(\n",
      "\u001b[0;32m~/anaconda3/envs/pmsi_db_analysis/lib/python3.10/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m__nonzero__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1535\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mfinal\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1536\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__nonzero__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1537\u001b[0;31m         raise ValueError(\n\u001b[0m\u001b[1;32m   1538\u001b[0m             \u001b[0;34mf\"The truth value of a {type(self).__name__} is ambiguous. \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1539\u001b[0m             \u001b[0;34m\"Use a.empty, a.bool(), a.item(), a.any() or a.all().\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: The truth value of a Series is ambiguous. Use a.empty, a.bool(), a.item(), a.any() or a.all()."
     ]
    }
   ],
   "source": [
    "enc = preproc.OneHotEncoder(categories=icd9_concepts.concept_code)\n",
    "enc.fit(diagnoses.icd9_code.reshape(-1,1))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "438cee95-532e-4cdf-8536-eaafac7228ba",
   "metadata": {},
   "source": [
    "## Using pandas get_dummies and then aggregate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "322d61a3-a53c-4581-827d-e27ba5902f95",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 73.6 ms, sys: 8.01 ms, total: 81.6 ms\n",
      "Wall time: 79 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "oh_icd_codes = pd.get_dummies(diagnoses.icd9_code,columns=\"icd9_code\",sparse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c98e7f1-947b-4076-86d6-13b9318f5762",
   "metadata": {},
   "outputs": [],
   "source": [
    "oh_icd_codes.info()\n",
    "oh_icd_codes.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd36d2b8-bb0f-4f04-bd66-393ac93c9cd1",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "%time\n",
    "oh_diagnoses_df = pd.merge(diagnoses['visit_occurrence_id'],oh_icd_codes,left_index=True,right_index=True)\n",
    "test = oh_diagnoses_df.groupby(by=\"visit_occurrence_id\",as_index=False).sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1709f9fc-521a-406d-8f0d-259353a2faac",
   "metadata": {},
   "source": [
    "**This method is intractable even with small samples**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c7d8e450-0bd5-48ca-820d-eb1545b37430",
   "metadata": {},
   "source": [
    "## Using pandas string aggregation and then get_dummies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8f9ee955-79da-4e6b-8be7-3b4f4e7910b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Monkey patch the _str_get_dummies to enforce uint8 return type\n",
    "\n",
    "def _str_get_dummies_uint8(self, sep=\"|\"):\n",
    "    from pandas import Series\n",
    "    import pandas._libs.lib as lib\n",
    "\n",
    "    arr = Series(self).fillna(\"\")\n",
    "    try:\n",
    "        arr = sep + arr + sep\n",
    "    except TypeError:\n",
    "        arr = sep + arr.astype(str) + sep\n",
    "\n",
    "    tags: set[str] = set()\n",
    "    for ts in Series(arr).str.split(sep):\n",
    "        tags.update(ts)\n",
    "    tags2 = sorted(tags - {\"\"})\n",
    "\n",
    "    dummies = np.empty((len(arr), len(tags2)), dtype=np.uint8)\n",
    "\n",
    "    for i, t in enumerate(tags2):\n",
    "        pat = sep + t + sep\n",
    "        dummies[:, i] = lib.map_infer(arr.to_numpy(), lambda x: pat in x)\n",
    "    return dummies, tags2\n",
    "\n",
    "pd.core.strings.object_array.ObjectStringArrayMixin._str_get_dummies = _str_get_dummies_uint8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d41a84ff-0d6c-4324-a900-00027a15d262",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 401 ms, sys: 19.4 ms, total: 420 ms\n",
      "Wall time: 425 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "agg_diagnoses_df = diagnoses[['visit_occurrence_id','icd9_code']].groupby(by=\"visit_occurrence_id\",as_index=False).agg(\"|\".join)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a30064e-1781-49fa-b085-fce19311b49b",
   "metadata": {},
   "outputs": [],
   "source": [
    "agg_diagnoses_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5e844123-f693-47c7-b045-fa444e23b9a7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 57s, sys: 4.16 s, total: 2min 1s\n",
      "Wall time: 2min 5s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "oh_icd_codes = agg_diagnoses_df.icd9_code.str.get_dummies(sep='|')#.astype(dtype=np.uint8)#.astype(pd.SparseDtype(dtype='uint8',fill_value=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0bea04eb-0160-418f-8f96-348e423afc26",
   "metadata": {},
   "outputs": [],
   "source": [
    "oh_icd_codes.info()\n",
    "oh_icd_codes.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "978c28e2-d172-46ea-b724-c39419466b35",
   "metadata": {},
   "outputs": [],
   "source": [
    "oh_diagnoses_df = pd.merge(agg_diagnoses_df['visit_occurrence_id'],oh_icd_codes,left_index=True,right_index=True)\n",
    "oh_diagnoses_df.info()\n",
    "oh_diagnoses_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "eb8b9db1-3a1e-4755-a72c-57b0cdb82008",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.93 s, sys: 2.37 s, total: 4.3 s\n",
      "Wall time: 4.44 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "visit_occurrence_s = pd.read_csv(datadir/\"../visit_occurrence.csv.gz\",usecols=['visit_occurrence_id']).squeeze()\n",
    "tmp = visit_occurrence_s[np.logical_not(visit_occurrence_s.isin(oh_diagnoses_df.visit_occurrence_id))]\n",
    "colvals_dict = {colname:0 for colname in oh_diagnoses_df.columns}\n",
    "colvals_dict['visit_occurrence_id']=tmp\n",
    "oh_diagnoses_df = pd.concat([oh_diagnoses_df,pd.DataFrame(colvals_dict).astype(oh_diagnoses_df.dtypes.to_dict())])\n",
    "oh_diagnoses_df.fillna(0,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "b7147966-823b-4afc-b0e4-9c0dbfdd4baa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 58s, sys: 140 ms, total: 1min 58s\n",
      "Wall time: 1min 58s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "oh_diagnoses_df.to_csv(datadir/\"oh_diagnoses.csv.gz\",index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "11f662d9-5b11-43c8-a72d-d45704bb34a2",
   "metadata": {},
   "source": [
    "Caveat: this is not really portable because there's no way to incorporate unseen codes"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4a4adf67-2dc2-44a9-bf02-c5f518895a73",
   "metadata": {},
   "source": [
    "# Diagnoses hierarchy unrolling and hot encode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ae592045-5047-4eba-a1e6-48559168bc9f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>visit_occurrence_id</th>\n",
       "      <th>icd9_concept_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>30549</td>\n",
       "      <td>44825520</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>33647</td>\n",
       "      <td>44833580</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10854</td>\n",
       "      <td>44826039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>17475</td>\n",
       "      <td>44824251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>30550</td>\n",
       "      <td>44835139</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   visit_occurrence_id  icd9_concept_id\n",
       "0                30549         44825520\n",
       "1                33647         44833580\n",
       "2                10854         44826039\n",
       "3                17475         44824251\n",
       "4                30550         44835139"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "diagnoses = pd.read_csv(datadir/\"diagnoses.csv.gz\",usecols=[\"visit_occurrence_id\",\"icd9_concept_id\"])\n",
    "icd9_concepts = pd.read_csv(datadir/\"icd9_concepts.csv.gz\")\n",
    "diagnoses.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6438e50e-e7e8-494b-8d9b-88b1edab33c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "icd9_relationships = pd.read_csv(datadir/\"icd9_concept_relationships.csv.gz\")\n",
    "diagnoses_rollup = concepts_toolbox.unroll_relationship(diagnoses,\n",
    "                                     concept_colname=\"icd9_concept_id\",\n",
    "                                     relationship=\"Is a\",\n",
    "                                     concept_relationship=icd9_relationships,max_depth=-1)\n",
    "diagnoses_rollup.drop_duplicates(inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6d3ad2e0-4e4f-4bbd-8ea2-467f895f8f8d",
   "metadata": {},
   "source": [
    "~~**I have to drop duplicates since the concept relationship table contains parents relationship to several levels**~~\n",
    "This is no longer true since I have filtered the DAG to make a tree, but taking unique will not hurt."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "dee32a7c-31bb-4dd5-af46-5835505ff1d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "diagnoses_rollup = pd.merge(diagnoses_rollup,\n",
    "         icd9_concepts[['concept_id',\"concept_code\"]],\n",
    "         left_on=\"icd9_concept_id\",\n",
    "         right_on=\"concept_id\").drop(columns=['icd9_concept_id','concept_id']).sort_values('visit_occurrence_id')\n",
    "diagnoses_rollup.rename(columns={'concept_code': \"icd9_code\"}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "894c3ae1-c61c-481f-b744-182d2075c3b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "oh_diagnoses_rollup = oh.hot_encode_df_col(diagnoses_rollup,colname=\"icd9_code\",by=\"visit_occurrence_id\",sep=\"|\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "fe33d488-aedb-4425-971c-d96309d64792",
   "metadata": {},
   "outputs": [],
   "source": [
    "visit_occurrence_s = pd.read_csv(datadir/\"../visit_occurrence.csv.gz\",usecols=['visit_occurrence_id']).squeeze()\n",
    "tmp = visit_occurrence_s[np.logical_not(visit_occurrence_s.isin(oh_diagnoses_rollup.visit_occurrence_id))]\n",
    "colvals_dict = {colname:0 for colname in oh_diagnoses_rollup.columns}\n",
    "colvals_dict['visit_occurrence_id']=tmp\n",
    "oh_diagnoses_rollup = pd.concat([oh_diagnoses_rollup,pd.DataFrame(colvals_dict).astype(oh_diagnoses_rollup.dtypes.to_dict())])\n",
    "oh_diagnoses_rollup.fillna(0,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "521e85de-69be-4714-a7dc-938a92071c1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "oh_diagnoses_rollup.to_csv(datadir/\"oh_diagnoses_rollup.csv.gz\",index=False) # drop a csv file for archiving\n",
    "oh_diagnoses_rollup.to_parquet(datadir/\"oh_diagnoses_rollup.parquet\",index=False) # parquet file as working file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "353d17a3-5295-485b-8752-172150e74036",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 58929 entries, 0 to 58928\n",
      "Columns: 9337 entries, visit_occurrence_id to V91.03\n",
      "dtypes: int64(1), uint8(9336)\n",
      "memory usage: 525.1 MB\n"
     ]
    }
   ],
   "source": [
    "oh_diagnoses_rollup.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5a2659f7-69d3-4d34-8902-8e03df220811",
   "metadata": {},
   "source": [
    "# Prescriptions hot encode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a7f7f66-446d-4a70-b7f8-443899d97855",
   "metadata": {},
   "outputs": [],
   "source": [
    "ingredients = pd.read_csv(datadir/\"unique_ingredient_prescription.csv.gz\",usecols=['visit_occurrence_id','rxnorm_ingredient_name'])\n",
    "ingredients.fillna(value={'rxnorm_ingredient_name':\"NA\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9eeadcd-4826-4fe6-87d3-2a13cf7f4088",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "oh_ingredients_df = oh.hot_encode_df_col(ingredients.dropna(),colname=\"rxnorm_ingredient_name\",by=\"visit_occurrence_id\",sep=\"|\")\n",
    "oh_ingredients_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "537a7c9b-ed3f-417a-a1c2-4d7a7452fffa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 50211 entries, 0 to 50210\n",
      "Columns: 1165 entries, visit_occurrence_id to zonisamide\n",
      "dtypes: int64(1), uint8(1164)\n",
      "memory usage: 56.1 MB\n"
     ]
    }
   ],
   "source": [
    "oh_ingredients_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "15a84b1e-541f-497e-9531-d3fdb1ac97b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "visit_occurrence_s = pd.read_csv(datadir/\"../visit_occurrence.csv.gz\",usecols=['visit_occurrence_id']).squeeze()\n",
    "tmp = visit_occurrence_s[np.logical_not(visit_occurrence_s.isin(oh_ingredients_df.visit_occurrence_id))]\n",
    "colvals_dict = {colname:0 for colname in oh_ingredients_df.columns}\n",
    "colvals_dict['visit_occurrence_id']=tmp\n",
    "oh_ingredients_df = pd.concat([oh_ingredients_df,pd.DataFrame(colvals_dict).astype(oh_ingredients_df.dtypes.to_dict())])\n",
    "oh_ingredients_df.fillna(0,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "de571151-1c95-4991-9e20-97f91aa08ef4",
   "metadata": {},
   "outputs": [],
   "source": [
    "oh_ingredients_df.to_csv(datadir/\"oh_ingredients.csv.gz\",index=False) # drop a csv file for archiving\n",
    "oh_ingredients_df.to_parquet(datadir/\"oh_ingredients.parquet\",index=False) # parquet file as working file"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "96b9e221-c9fc-4c62-a3a7-e1ecd93cdc60",
   "metadata": {},
   "source": [
    "# Performance checks"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "315b3cdf-8788-4142-aa15-ce40dc13f878",
   "metadata": {},
   "source": [
    "## Some memory experiments: convert_dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "eef1e109-6af4-41d3-8b87-a95777c2b4e5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 500000 entries, 0 to 499999\n",
      "Data columns (total 1 columns):\n",
      " #   Column  Non-Null Count   Dtype\n",
      "---  ------  --------------   -----\n",
      " 0   0       500000 non-null  bool \n",
      "dtypes: bool(1)\n",
      "memory usage: 488.4 KB\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 500000 entries, 0 to 499999\n",
      "Data columns (total 1 columns):\n",
      " #   Column  Non-Null Count   Dtype\n",
      "---  ------  --------------   -----\n",
      " 0   0       500000 non-null  bool \n",
      "dtypes: bool(1)\n",
      "memory usage: 488.4 KB\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 500000 entries, 0 to 499999\n",
      "Data columns (total 1 columns):\n",
      " #   Column  Non-Null Count   Dtype\n",
      "---  ------  --------------   -----\n",
      " 0   0       500000 non-null  uint8\n",
      "dtypes: uint8(1)\n",
      "memory usage: 488.4 KB\n"
     ]
    }
   ],
   "source": [
    "init_array=np.zeros(500000)\n",
    "pd.DataFrame(init_array,dtype=bool).info()\n",
    "pd.DataFrame(init_array,dtype=np.bool_).info()\n",
    "pd.DataFrame(init_array,dtype='uint8').info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "136bcd9f-84a8-4f3c-b8b9-173d00289d9d",
   "metadata": {},
   "source": [
    "## OH file reading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "fc66db1b-fa08-4f7d-891f-362947499685",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 38s, sys: 4.01 s, total: 1min 42s\n",
      "Wall time: 1min 57s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "oh_diagnoses_df = pandas_utils.read_csv_defaultdtype(datadir/\"oh_diagnoses_rollup.csv.gz\",\n",
    "                                                     defaultdtype='uint8',dtypes_dict={'visit_occurrence_id':'uint64'},\n",
    "                                                    engine='c')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "71fd0227-339b-4f23-8286-70d2e877c429",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4.23 s, sys: 154 ms, total: 4.39 s\n",
      "Wall time: 4.39 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "oh_ingredients_df = pandas_utils.read_csv_defaultdtype(datadir/\"oh_ingredients.csv.gz\",\n",
    "                                                       defaultdtype='uint8',dtypes_dict={'visit_occurrence_id':'uint64'},\n",
    "                                                      engine='c')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1d2657af-d8c4-4d2a-bc27-faeac173db8c",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# Using the faster csv reader from pandas, introduced in 1.4.0\n",
    "#del oh_diagnoses_df\n",
    "# WILL EXHAUST MY COMPUTER RAM\n",
    "oh_diagnoses_df = pandas_utils.read_csv_defaultdtype(datadir/\"oh_diagnoses_rollup.csv.gz\",\n",
    "                                                     defaultdtype='uint8',dtypes_dict={'visit_occurrence_id':'uint64'},\n",
    "                                                    engine='pyarrow')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "733af8fb-3f38-4397-8c1d-249e42435f0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gc\n",
    "from multiprocessing import Process, Event, Value\n",
    "import os\n",
    "import time\n",
    "from timeit import default_timer\n",
    "from collections import defaultdict\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import psutil\n",
    "\n",
    "\n",
    "class Timer:\n",
    "    \"\"\"Simple util to measure execution time.\n",
    "    Examples\n",
    "    --------\n",
    "    >>> import time\n",
    "    >>> with Timer() as timer:\n",
    "    ...     time.sleep(1)\n",
    "    >>> print(timer)\n",
    "    00:00:01\n",
    "    \"\"\"\n",
    "    def __init__(self):\n",
    "        self.start = None\n",
    "        self.elapsed = None\n",
    "\n",
    "    def __enter__(self):\n",
    "        self.start = default_timer()\n",
    "        return self\n",
    "\n",
    "    def __exit__(self, exc_type, exc_val, exc_tb):\n",
    "        self.elapsed = default_timer() - self.start\n",
    "\n",
    "    def __str__(self):\n",
    "        return self.verbose()\n",
    "\n",
    "    def __float__(self):\n",
    "        return self.elapsed\n",
    "\n",
    "    def verbose(self):\n",
    "        if self.elapsed is None:\n",
    "            return '<not-measured>'\n",
    "        return self.format_elapsed_time(self.elapsed)\n",
    "\n",
    "    @staticmethod\n",
    "    def format_elapsed_time(value: float):\n",
    "        return time.strftime('%H:%M:%S', time.gmtime(value))\n",
    "    \n",
    "    \n",
    "class MemoryTrackingProcess(Process):\n",
    "    \"\"\"A process that periodically measures the amount of RAM consumed by another process.\n",
    "    \n",
    "    This process is stopped as soon as the event is set.\n",
    "    \"\"\"\n",
    "    def __init__(self, pid, event, **kwargs):\n",
    "        super().__init__()\n",
    "        self.p = psutil.Process(pid)\n",
    "        self.event = event\n",
    "        self.max_mem = Value('f', 0.0)\n",
    "        \n",
    "    def run(self):\n",
    "        mem_usage = []\n",
    "        while not self.event.is_set():\n",
    "            info = self.p.memory_info()\n",
    "            mem_bytes = info.rss\n",
    "            mem_usage.append(mem_bytes)\n",
    "            time.sleep(0.05)\n",
    "        self.max_mem.value = np.max(mem_usage)\n",
    "\n",
    "\n",
    "class MemoryTracker:\n",
    "    \"\"\"A context manager that runs MemoryTrackingProcess in background and collects \n",
    "    the information about used memory when the context is exited.\n",
    "    \"\"\"\n",
    "    def __init__(self, pid=None):\n",
    "        pid = pid or os.getpid()\n",
    "        self.start_mem = psutil.Process(pid).memory_info().rss\n",
    "        self.event = Event()\n",
    "        self.p = MemoryTrackingProcess(pid, self.event)\n",
    "    \n",
    "    @property\n",
    "    def memory(self):\n",
    "        return self.p.max_mem.value - self.start_mem\n",
    "    \n",
    "    def __enter__(self):\n",
    "        self.p.start()\n",
    "        return self\n",
    "    \n",
    "    def __exit__(self, exc_type, exc_val, exc_tb):\n",
    "        self.event.set()\n",
    "        self.p.join()\n",
    "\n",
    "        \n",
    "class GC:\n",
    "    def __enter__(self):\n",
    "        return self\n",
    "    def __exit__(self, exc_type, exc_val, ext_tb):\n",
    "        self.collected = gc.collect()\n",
    "        \n",
    "        \n",
    "def size_of(filename, unit=1024**2):\n",
    "    return round(os.stat(filename).st_size / unit, 2)\n",
    "\n",
    "\n",
    "def get_save_load(df, fmt):\n",
    "    def csv_reader(filename):\n",
    "        return pandas_utils.read_csv_defaultdtype(filename, defaultdtype='uint8',dtypes_dict={'visit_occurrence_id':'uint64'},\n",
    "                                                    engine='c')\n",
    "    save = getattr(df, f'to_csv') if fmt=='csv.gz' else getattr(df, f'to_{fmt}')\n",
    "    load = csv_reader if fmt in ['csv','csv.gz'] else getattr(pd, f'read_{fmt}')\n",
    "    return save, load\n",
    "\n",
    "\n",
    "def benchmark(dataset, list_of_formats, n_rounds=1):\n",
    "    \"\"\"Runs dataset saving/loading benchamrk using formts from the list_of_formats.\n",
    "    \n",
    "    Each round a new random dataset is generated with data_size observations. \n",
    "    The measurements for each of the rounds are concatenated together and returned\n",
    "    as a single data frame.\n",
    "    \n",
    "    Parameters:\n",
    "        list_of_formats: A list of tuples in the format (<format_name>, [<params_dict>]). \n",
    "            The <format_name> should be one of the pandas supported formats.\n",
    "        data_size: A number of samples in the generated dataset.\n",
    "        n_num: A number of numerical columns in the generated dataset.\n",
    "        n_cat: A number of categorical columns in the generated dataset.\n",
    "        n_rounds: A number of randomly generated datasets to test the formats.\n",
    "        as_category: If True, then categorical columns will be converted into \n",
    "            pandas.Category type before saving.\n",
    "            \n",
    "    \"\"\"\n",
    "    runs = []\n",
    "    \n",
    "    for i in range(n_rounds):\n",
    "        print(f'Benchmarking round #{i + 1:d}')\n",
    "        \n",
    "        benchmark = []\n",
    "        \n",
    "        for case in list_of_formats:\n",
    "            fmt, params = case if len(case) == 2 else (case[0], {})\n",
    "            \n",
    "            with GC():\n",
    "                print('\\ttesting format:', fmt)\n",
    "                filename = f'/home/quentin/Desktop/filebench.{fmt}'\n",
    "                save, load = get_save_load(dataset, fmt)\n",
    "                results = defaultdict(int)\n",
    "                results['format'] = fmt\n",
    "                \n",
    "                with MemoryTracker() as tracker:\n",
    "                    with Timer() as timer:\n",
    "                        save(filename, **params)\n",
    "                results['size_mb'] = size_of(filename)\n",
    "                results['save_ram_delta_mb'] = tracker.memory / (1024 ** 2)\n",
    "                results['save_time'] = float(timer)\n",
    "                \n",
    "                with MemoryTracker() as tracker:\n",
    "                    with Timer() as timer:\n",
    "                        _ = load(filename)\n",
    "                results['load_ram_delta_mb'] = tracker.memory / (1024 ** 2)\n",
    "                results['load_time'] = float(timer)\n",
    "                \n",
    "                benchmark.append(results)\n",
    "                os.remove(filename)\n",
    "                \n",
    "            run = pd.DataFrame(benchmark)\n",
    "            run['run_no'] = i\n",
    "            runs.append(run)\n",
    "            \n",
    "    benchmark = pd.concat(runs, axis=0)\n",
    "    benchmark.reset_index(inplace=True, drop=True)\n",
    "    return benchmark\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8edda0fd-7228-42d9-afc2-97a5077af962",
   "metadata": {},
   "source": [
    "Try some binary formats:\n",
    "- Feather\n",
    "- Parquet\n",
    "- msgpack\n",
    "- hdf5\n",
    "- pickle\n",
    "\n",
    "Some existing benchmarks:\n",
    "- https://towardsdatascience.com/the-best-format-to-save-pandas-data-414dca023e0d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "7be7665a-5821-45f6-a113-3c5e281c4f9d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Benchmarking round #1\n",
      "\ttesting format: csv.gz\n",
      "\ttesting format: pickle\n",
      "\ttesting format: feather\n",
      "\ttesting format: parquet\n",
      "Benchmarking round #2\n",
      "\ttesting format: csv.gz\n",
      "\ttesting format: pickle\n",
      "\ttesting format: feather\n",
      "\ttesting format: parquet\n"
     ]
    }
   ],
   "source": [
    "formats = [\n",
    "    #('csv',), # this simply takes way too much memory\n",
    "    ('csv.gz',),\n",
    "    #('hdf', {'key': 'data', 'format': 'table'}),\n",
    "    ('pickle',),\n",
    "    #('msgpack',),\n",
    "    ('feather',),\n",
    "    ('parquet',)\n",
    "]\n",
    "results = benchmark(oh_diagnoses_df,list_of_formats=formats,n_rounds=2)\n",
    "results.drop_duplicates(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "67173367-9f6c-4495-829f-da16ee8fe715",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>format</th>\n",
       "      <th>size_mb</th>\n",
       "      <th>save_ram_delta_mb</th>\n",
       "      <th>save_time</th>\n",
       "      <th>load_ram_delta_mb</th>\n",
       "      <th>load_time</th>\n",
       "      <th>run_no</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>csv.gz</td>\n",
       "      <td>4.91</td>\n",
       "      <td>0.042969</td>\n",
       "      <td>148.332947</td>\n",
       "      <td>1942.574219</td>\n",
       "      <td>305.204614</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>pickle</td>\n",
       "      <td>525.62</td>\n",
       "      <td>525.292969</td>\n",
       "      <td>25.216944</td>\n",
       "      <td>522.023438</td>\n",
       "      <td>0.268633</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>feather</td>\n",
       "      <td>12.74</td>\n",
       "      <td>95.980469</td>\n",
       "      <td>0.640301</td>\n",
       "      <td>829.636719</td>\n",
       "      <td>5.407113</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>parquet</td>\n",
       "      <td>8.89</td>\n",
       "      <td>-125.617188</td>\n",
       "      <td>6.190886</td>\n",
       "      <td>1161.753906</td>\n",
       "      <td>1.181758</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>csv.gz</td>\n",
       "      <td>4.91</td>\n",
       "      <td>-158.871094</td>\n",
       "      <td>144.419550</td>\n",
       "      <td>1080.207031</td>\n",
       "      <td>293.954210</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>pickle</td>\n",
       "      <td>525.62</td>\n",
       "      <td>505.742188</td>\n",
       "      <td>16.025387</td>\n",
       "      <td>489.945312</td>\n",
       "      <td>0.362092</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>feather</td>\n",
       "      <td>12.74</td>\n",
       "      <td>94.089844</td>\n",
       "      <td>0.512138</td>\n",
       "      <td>804.367188</td>\n",
       "      <td>11.903225</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>parquet</td>\n",
       "      <td>8.89</td>\n",
       "      <td>-108.304688</td>\n",
       "      <td>6.180116</td>\n",
       "      <td>1176.902344</td>\n",
       "      <td>2.212960</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     format  size_mb  save_ram_delta_mb   save_time  load_ram_delta_mb  \\\n",
       "0    csv.gz     4.91           0.042969  148.332947        1942.574219   \n",
       "2    pickle   525.62         525.292969   25.216944         522.023438   \n",
       "5   feather    12.74          95.980469    0.640301         829.636719   \n",
       "9   parquet     8.89        -125.617188    6.190886        1161.753906   \n",
       "10   csv.gz     4.91        -158.871094  144.419550        1080.207031   \n",
       "12   pickle   525.62         505.742188   16.025387         489.945312   \n",
       "15  feather    12.74          94.089844    0.512138         804.367188   \n",
       "19  parquet     8.89        -108.304688    6.180116        1176.902344   \n",
       "\n",
       "     load_time  run_no  \n",
       "0   305.204614       0  \n",
       "2     0.268633       0  \n",
       "5     5.407113       0  \n",
       "9     1.181758       0  \n",
       "10  293.954210       1  \n",
       "12    0.362092       1  \n",
       "15   11.903225       1  \n",
       "19    2.212960       1  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}