code/scripts/mimic-omop_checks_and_preproc.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "01240e40-8fb7-40bf-9d85-7a8683db1291",
   "metadata": {},
   "source": [
    "This notebook presents a summary analysis, sanity checks and preprocessing of the mimic-OMOP database.\n",
    "The database has been obtained using the postgreSQL ETLs from: https://github.com/MIT-LCP/mimic-omop. \n",
    "\n",
    "Reproducible conda environment used: file db_analysis-linux-64.lock "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cbb8047a-eda8-4e82-8639-04345f15db66",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "from pathlib import Path\n",
    "import os\n",
    "sys.path.insert(0,str(Path(os.path.abspath('')).parent))\n",
    "from proj_config import *\n",
    "import pandas as pd\n",
    "import pandas_profiling as pp\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import subprocess\n",
    "import pandas_utils\n",
    "import concepts_toolbox\n",
    "# Define local env variables\n",
    "datadir = get_datadir() / \"mimic-omop\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "286355ab-10e2-459e-9deb-0ab3ea18c3b4",
   "metadata": {},
   "source": [
    "# Concept tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7bc04dd4-f3cf-477a-bbe8-db499235f5f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/quentin/anaconda3/envs/pmsi_db_analysis/lib/python3.10/site-packages/IPython/core/interactiveshell.py:3251: DtypeWarning: Columns (5,6,9) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  exec(code_obj, self.user_global_ns, self.user_ns)\n"
     ]
    }
   ],
   "source": [
    "concept_df = pd.read_csv(datadir/\"concept.csv.gz\")\n",
    "concept_class_df = pd.read_csv(datadir/\"concept_class.csv.gz\")\n",
    "concept_relationship_df = pd.read_csv(datadir/\"concept_relationship.csv.gz\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a64036a-856a-41d9-8b53-2b476e2e987e",
   "metadata": {},
   "source": [
    "The concept_relationship table contains:\n",
    "- hierarchical relationships between concepts: relationship_id 'Subsumes' (parent), or 'Is a' (child)\n",
    "- mappings between different ontologies: relationship_id 'Mapped from' and 'Maps to'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3d8aa14b-5efa-4093-a27e-33188ef9a0b3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEFCAYAAADjUZCuAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAVP0lEQVR4nO3dfYxd9X3n8fenuLHoNiQED6zXNjUhJitArVN7vWjbVHTZFpdUMWlDa1QVV0VyQKRq2s3uwkbaRJGsDXlYVqiFyFkQOJvykBAWS4U2lKRhV+IhY2oeDCEM4JTBFrghS1ilYWXy3T/ub7LX4+sZ+z7MjM37JR3dc7/n/M79ncNlPvec37nXqSokSfqp+e6AJGlhMBAkSYCBIElqDARJEmAgSJKaRfPdgX4tWbKkVq5cOd/dkKSjyo4dO/6hqsZ6LTtqA2HlypWMj4/Pdzck6aiS5LuHWuYlI0kSYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJamYNhCQ3Jnk5yRNdtduS7GzT7iQ7W31lkn/sWvb5rjZrkjyeZCLJtUnS6ovb9iaSPJRk5fB3U5I0m8P5pvJNwJ8B26YKVfW7U/NJPge82rX+s1W1usd2rgc2Aw8CdwPrgXuAS4HvV9W7kmwErgZ+t0f7Y8LKK/+yZ333p943xz2RpAPNeoZQVfcDr/Ra1j7l/w5wy0zbSLIUOKGqHqjOP9G2DbiwLd4A3NzmvwKcN3X2IEmaO4OOIbwXeKmqnumqnZbk75J8M8l7W20ZMNm1zmSrTS17AaCq9tM52zhpwH5Jko7QoD9udzEHnh3sBU6tqu8lWQP8jyRnAb0+8U/9Y84zLTtAks10Ljtx6qmn9t1pSdLB+j5DSLII+C3gtqlaVb1eVd9r8zuAZ4Ez6JwRLO9qvhzY0+YngRVd23wbh7hEVVVbq2ptVa0dG+v5662SpD4Ncsno3wDfrqqfXApKMpbkuDb/TmAV8FxV7QVeS3JOGx+4BLirNdsObGrzHwS+3sYZJElz6HBuO70FeAB4d5LJJJe2RRs5eDD5V4DHkjxKZ4D4sqqa+rR/OfDfgAk6Zw73tPoNwElJJoA/Ba4cYH8kSX2adQyhqi4+RP0PetTuAO44xPrjwNk96j8CLpqtH5Kk0fKbypIkwECQJDUGgiQJMBAkSY2BIEkCDARJUmMgSJIAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElSYyBIkgADQZLUGAiSJMBAkCQ1BoIkCTAQJEmNgSBJAg4jEJLcmOTlJE901T6R5MUkO9t0Qdeyq5JMJHk6yfld9TVJHm/Lrk2SVl+c5LZWfyjJyiHvoyTpMBzOGcJNwPoe9WuqanWb7gZIciawETirtbkuyXFt/euBzcCqNk1t81Lg+1X1LuAa4Oo+90WSNIBZA6Gq7gdeOcztbQBurarXq+p5YAJYl2QpcEJVPVBVBWwDLuxqc3Ob/wpw3tTZgyRp7gwyhvDhJI+1S0onttoy4IWudSZbbVmbn14/oE1V7QdeBU7q9YJJNicZTzK+b9++AbouSZqu30C4HjgdWA3sBT7X6r0+2dcM9ZnaHFys2lpVa6tq7djY2BF1WJI0s74Coapeqqo3qurHwBeAdW3RJLCia9XlwJ5WX96jfkCbJIuAt3H4l6gkSUPSVyC0MYEpHwCm7kDaDmxsdw6dRmfw+OGq2gu8luScNj5wCXBXV5tNbf6DwNfbOIMkaQ4tmm2FJLcA5wJLkkwCHwfOTbKazqWd3cCHAKpqV5LbgSeB/cAVVfVG29TldO5YOh64p00ANwBfTDJB58xg4xD2S5J0hGYNhKq6uEf5hhnW3wJs6VEfB87uUf8RcNFs/ZAkjZbfVJYkAQaCJKkxECRJgIEgSWoMBEkSYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSpMRAkSYCBIElqZg2EJDcmeTnJE121zyT5dpLHktyZ5O2tvjLJPybZ2abPd7VZk+TxJBNJrk2SVl+c5LZWfyjJyuHvpiRpNodzhnATsH5a7V7g7Kr6eeA7wFVdy56tqtVtuqyrfj2wGVjVpqltXgp8v6reBVwDXH3EeyFJGtisgVBV9wOvTKt9rar2t6cPAstn2kaSpcAJVfVAVRWwDbiwLd4A3NzmvwKcN3X2IEmaO8MYQ/hD4J6u56cl+bsk30zy3lZbBkx2rTPZalPLXgBoIfMqcFKvF0qyOcl4kvF9+/YNoeuSpCkDBUKSjwH7gS+10l7g1Kp6D/CnwF8kOQHo9Ym/pjYzw7IDi1Vbq2ptVa0dGxsbpOuSpGkW9dswySbgN4Hz2mUgqup14PU2vyPJs8AZdM4Iui8rLQf2tPlJYAUwmWQR8DamXaKSJI1eX2cISdYD/wF4f1X9sKs+luS4Nv9OOoPHz1XVXuC1JOe08YFLgLtas+3Apjb/QeDrUwEjSZo7s54hJLkFOBdYkmQS+Didu4oWA/e28d8H2x1FvwJ8Msl+4A3gsqqa+rR/OZ07lo6nM+YwNe5wA/DFJBN0zgw2DmXPJElHZNZAqKqLe5RvOMS6dwB3HGLZOHB2j/qPgItm64ckabT8prIkCTAQJEmNgSBJAgwESVJjIEiSAANBktQYCJIkwECQJDUGgiQJMBAkSY2BIEkCDARJUmMgSJIAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElSM2sgJLkxyctJnuiqvSPJvUmeaY8ndi27KslEkqeTnN9VX5Pk8bbs2iRp9cVJbmv1h5KsHPI+SpIOw+GcIdwErJ9WuxK4r6pWAfe15yQ5E9gInNXaXJfkuNbmemAzsKpNU9u8FPh+Vb0LuAa4ut+dkST1b9ZAqKr7gVemlTcAN7f5m4ELu+q3VtXrVfU8MAGsS7IUOKGqHqiqArZNazO1ra8A502dPUiS5k6/YwinVNVegPZ4cqsvA17oWm+y1Za1+en1A9pU1X7gVeCkXi+aZHOS8STj+/bt67PrkqRehj2o3OuTfc1Qn6nNwcWqrVW1tqrWjo2N9dlFSVIv/QbCS+0yEO3x5VafBFZ0rbcc2NPqy3vUD2iTZBHwNg6+RCVJGrF+A2E7sKnNbwLu6qpvbHcOnUZn8PjhdlnptSTntPGBS6a1mdrWB4Gvt3EGSdIcWjTbCkluAc4FliSZBD4OfAq4PcmlwN8DFwFU1a4ktwNPAvuBK6rqjbapy+ncsXQ8cE+bAG4Avphkgs6Zwcah7Jkk6YjMGghVdfEhFp13iPW3AFt61MeBs3vUf0QLFEnS/PGbypIkwECQJDUGgiQJMBAkSY2BIEkCDARJUmMgSJIAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElSYyBIkgADQZLUGAiSJMBAkCQ1BoIkCTAQJElN34GQ5N1JdnZNP0jykSSfSPJiV/2CrjZXJZlI8nSS87vqa5I83pZdmySD7pgk6cj0HQhV9XRVra6q1cAa4IfAnW3xNVPLqupugCRnAhuBs4D1wHVJjmvrXw9sBla1aX2//ZIk9WdYl4zOA56tqu/OsM4G4Naqer2qngcmgHVJlgInVNUDVVXANuDCIfVLknSYhhUIG4Fbup5/OMljSW5McmKrLQNe6FpnstWWtfnp9YMk2ZxkPMn4vn37htR1SRIMIRCSvAV4P/DlVroeOB1YDewFPje1ao/mNUP94GLV1qpaW1Vrx8bGBum2JGmaYZwh/AbwSFW9BFBVL1XVG1X1Y+ALwLq23iSwoqvdcmBPqy/vUZckzaFhBMLFdF0uamMCUz4APNHmtwMbkyxOchqdweOHq2ov8FqSc9rdRZcAdw2hX5KkI7BokMZJfgb4NeBDXeVPJ1lN57LP7qllVbUrye3Ak8B+4IqqeqO1uRy4CTgeuKdNkqQ5NFAgVNUPgZOm1X5/hvW3AFt61MeBswfpiyRpMH5TWZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSpMRAkSYCBIElqDARJEmAgSJIaA0GSBBgIkqTGQJAkAQaCJKkZKBCS7E7yeJKdScZb7R1J7k3yTHs8sWv9q5JMJHk6yfld9TVtOxNJrk2SQfolSTpywzhD+NWqWl1Va9vzK4H7qmoVcF97TpIzgY3AWcB64Lokx7U21wObgVVtWj+EfkmSjsAoLhltAG5u8zcDF3bVb62q16vqeWACWJdkKXBCVT1QVQVs62ojSZojgwZCAV9LsiPJ5lY7par2ArTHk1t9GfBCV9vJVlvW5qfXD5Jkc5LxJOP79u0bsOuSpG6LBmz/S1W1J8nJwL1Jvj3Dur3GBWqG+sHFqq3AVoC1a9f2XEeS1J+BzhCqak97fBm4E1gHvNQuA9EeX26rTwIrupovB/a0+vIedUnSHOo7EJL8kyRvnZoHfh14AtgObGqrbQLuavPbgY1JFic5jc7g8cPtstJrSc5pdxdd0tVGkjRHBrlkdApwZ7tDdBHwF1X1V0m+Bdye5FLg74GLAKpqV5LbgSeB/cAVVfVG29blwE3A8cA9bZIkzaG+A6GqngN+oUf9e8B5h2izBdjSoz4OnN1vXyRJg/ObypIkwECQJDUGgiQJMBAkSY2BIEkCDARJUmMgSJIAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElSYyBIkgADQZLUGAiSJMBAkCQ1BoIkCTAQJElN34GQZEWSbyR5KsmuJH/c6p9I8mKSnW26oKvNVUkmkjyd5Pyu+pokj7dl1ybJYLslSTpSiwZoux/4t1X1SJK3AjuS3NuWXVNVn+1eOcmZwEbgLOCfAX+T5IyqegO4HtgMPAjcDawH7hmgb5KkI9T3GUJV7a2qR9r8a8BTwLIZmmwAbq2q16vqeWACWJdkKXBCVT1QVQVsAy7st1+SpP4MZQwhyUrgPcBDrfThJI8luTHJia22DHihq9lkqy1r89PrkqQ5NHAgJPlZ4A7gI1X1AzqXf04HVgN7gc9Nrdqjec1Q7/Vam5OMJxnft2/foF2XJHUZKBCS/DSdMPhSVX0VoKpeqqo3qurHwBeAdW31SWBFV/PlwJ5WX96jfpCq2lpVa6tq7djY2CBdlyRNM8hdRgFuAJ6qqv/SVV/atdoHgCfa/HZgY5LFSU4DVgEPV9Ve4LUk57RtXgLc1W+/JEn9GeQuo18Cfh94PMnOVvuPwMVJVtO57LMb+BBAVe1KcjvwJJ07lK5odxgBXA7cBBxP5+4i7zCSpDnWdyBU1f+i9/X/u2doswXY0qM+Dpzdb18kSYPzm8qSJMBAkCQ1BoIkCTAQJEmNgSBJAgwESVJjIEiSAANBktQYCJIkwECQJDUGgiQJMBAkSY2BIEkCDARJUjPIv4cgaZ6svPIve9Z3f+p987KdI93+MF9Dw2MgSAvYTH9QR7mdYb1uP69hUMwfA0E6hszFH3IduxxDkCQBniFI0oI115fVDARJC8qwLnsd6o/mkW7/zTSmYSBIOibN14D80WzBjCEkWZ/k6SQTSa6c7/5I0pvNgjhDSHIc8OfArwGTwLeSbK+qJ+e3Z9LceDN9CtXCtVDOENYBE1X1XFX9X+BWYMM890mS3lQWxBkCsAx4oev5JPAvp6+UZDOwuT39P0me7vP1lgD/0GfbkcjVc/6SC+4YzAOPgccAjsJjMODfi5871IKFEgjpUauDClVbga0Dv1gyXlVrB93O0cxj4DEAjwF4DLotlEtGk8CKrufLgT3z1BdJelNaKIHwLWBVktOSvAXYCGyf5z5J0pvKgrhkVFX7k3wY+GvgOODGqto1wpcc+LLTMcBj4DEAjwF4DH4iVQddqpckvQktlEtGkqR5ZiBIkoBjIBCSrEjyjSRPJdmV5I97rJMk17afxXgsyS92Lev5kxlJ3pHk3iTPtMcT52qfjtQIj8Fnkny7rX9nkrfP0S4dsVEdg67lH01SSZaMel/6NcpjkOSP2rJdST49F/vTjxH+v7A6yYNJdiYZT7JurvZpTlXVUT0BS4FfbPNvBb4DnDltnQuAe+h83+Ec4KFWPw54Fngn8Bbg0am2wKeBK9v8lcDV872v83AMfh1Y1OavfjMeg7Z8BZ0bHr4LLJnvfZ2H98GvAn8DLG7PT57vfZ2HY/A14De62v/tfO/rKKaj/gyhqvZW1SNt/jXgKTrffO62AdhWHQ8Cb0+ylJl/MmMDcHObvxm4cLR70r9RHYOq+lpV7W/tH6Tz/ZAFaYTvA4BrgH9Pjy9LLiQjPAaXA5+qqtfbtl+eg93pywiPQQEntPm3cYx+T+qoD4RuSVYC7wEemrao109jLJuhDnBKVe2FzpsMOHkEXR66IR+Dbn9I51PVgjfMY5Dk/cCLVfXoqPo7CkN+H5wBvDfJQ0m+meRfjKTTQzbkY/AR4DNJXgA+C1w1/B7Pv2MmEJL8LHAH8JGq+sH0xT2a1Az1o9KojkGSjwH7gS8No5+jNMxjkORngI8B/2m4vRytEbwPFgEn0rm88u+A25P0Wn/BGMExuBz4k6paAfwJcMOw+rqQHBOBkOSn6fzH/1JVfbXHKof6aYyZfjLjpXYaSXtcsKfJMLJjQJJNwG8Cv1ftAupCNYJjcDpwGvBokt2t/kiSfzr83g/HiN4Hk8BX2yWWh4Ef0/lBuAVpRMdgEzC1rS/Tubx07JnvQYxBJzqpvg34rzOs8z4OHER6uNUXAc/R+Z9+ahDprLbsMxw4qPzp+d7XeTgG64EngbH53sf5OgbT2u9mYQ8qj+p9cBnwyTZ/Bp3LKpnv/Z3jY/AUcG6bPw/YMd/7OpLjN98dGMIb4JfpnNY9Buxs0wXtTXxZ15vkz+ncQfA4sLar/QV07kR4FvhYV/0k4D7gmfb4jvne13k4BhPtf/6pbX5+vvd1ro/BtNfYzcIOhFG9D94C/HfgCeAR4F/P977OwzH4ZWAHnZB4CFgz3/s6ismfrpAkAcfIGIIkaXAGgiQJMBAkSY2BIEkCDARJOiokuTHJy0meOIx1fy7Jfe3H+/42yWH97IyBIElHh5vofDfocHyWzu81/TzwSeA/H04jA0GSjgJVdT/wSnctyelJ/irJjiT/M8k/b4vOpPP9KYBvcOCPNR6SgSBJR6+twB9V1Rrgo8B1rf4o8Ntt/gPAW5OcNNvGFo2ki5KkkWo/4PevgC93/dbg4vb4UeDPkvwBcD/wIp0fqJyRgSBJR6efAv53Va2evqCq9gC/BT8Jjt+uqlcPZ4OSpKNMdX7W+/kkF8FP/mnQX2jzS5JM/X2/CrjxcLZpIEjSUSDJLcADwLuTTCa5FPg94NIkjwK7+P+Dx+cCTyf5DnAKsOWwXsMft5MkgWcIkqTGQJAkAQaCJKkxECRJgIEgSWoMBEkSYCBIkpr/B6IOQVh1E88SAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "plt.hist(concept_df.concept_id[concept_df.concept_id>200000000],bins=50);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "4a9e611c-34e1-4fc3-b96f-8e3c9487a8f7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2009359722"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concept_df.concept_id[concept_df.concept_id>200000000].max()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c937f3fa-49de-4afe-8885-21aae4baf7c2",
   "metadata": {},
   "source": [
    "# Person and visit tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c6dbcb17-beb1-48ac-89f9-9f7a45724927",
   "metadata": {},
   "outputs": [],
   "source": [
    "person_df = pd.read_csv(datadir/\"person.csv.gz\",usecols= ['person_id','gender_concept_id','birth_datetime'],\n",
    "                        parse_dates=['birth_datetime'],infer_datetime_format=True)\n",
    "visit_df = pd.read_csv(datadir/\"visit_occurrence.csv.gz\",usecols=['visit_occurrence_id','person_id','visit_start_date'],\n",
    "                      parse_dates=['visit_start_date'],infer_datetime_format=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b67bbd50-ae18-4c60-9212-1cfd6a3e3507",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([8507, 8532])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "person_df.gender_concept_id.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "c2b59af7-5f57-4d0f-b1af-280557154da3",
   "metadata": {},
   "outputs": [],
   "source": [
    "preproc_visit_df = pd.merge(visit_df,person_df,how='left',\n",
    "        left_on='person_id',right_on='person_id')\n",
    "preproc_visit_df.gender_concept_id = (preproc_visit_df.gender_concept_id==8532).astype(dtype='uint8')\n",
    "preproc_visit_df.rename(columns={'gender_concept_id':\"sex_F\"},inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "bda12a6b-92a7-4114-a8f9-41d3c294bd68",
   "metadata": {},
   "outputs": [],
   "source": [
    "preproc_visit_df['age'] = (preproc_visit_df.visit_start_date.dt.date-preproc_visit_df.birth_datetime.dt.date)\n",
    "preproc_visit_df.age = preproc_visit_df.age.apply(lambda x: x.days/365.25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "1fc4f8b4-41cc-4141-9311-9c6a98f2f775",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([10781., 19303., 26276.,     0.,     0.,     0.,     0.,     0.,\n",
       "            0.,  2616.]),\n",
       " array([  0.        ,  31.15537303,  62.31074606,  93.4661191 ,\n",
       "        124.62149213, 155.77686516, 186.93223819, 218.08761123,\n",
       "        249.24298426, 280.39835729, 311.55373032]),\n",
       " <BarContainer object of 10 artists>)"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD4CAYAAADsKpHdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAQ+UlEQVR4nO3dX6hd5ZnH8e9vEutIrR3/RAlJmJPWXDTKTFpDJuBQOjjU1F7EgsLxouYikCIRWuhcxBamzkVAB1pBGIUUxSidarAtBqwzFe1QCqI9dqJJTDOeVqemCeZ0dGx6oTNJn7nY76E7J/v8T7L3mfP9wGKv86z1rv28rB5/WWvts5uqQpKkP+l3A5KkwWAgSJIAA0GS1BgIkiTAQJAkNUv73cBcXXHFFTU0NNTvNiRpQXn55Zd/W1XLem1bsIEwNDTEyMhIv9uQpAUlyX9Ots1bRpIkwECQJDUGgiQJMBAkSY2BIEkCDARJUmMgSJIAA0GS1BgIkiRgAf+lsmZnaMfTfXvvN+/5fN/eW9LMeYUgSQIMBElSYyBIkgADQZLUGAiSJMBAkCQ10wZCklVJfpzkUJKDSb7c6ncn+U2SfW25qWvMXUlGkxxOcmNX/bok+9u2+5Ok1S9M8kSrv5hk6BzMVZI0hZlcIZwEvlpVnwA2AtuTrG3b7quqdW35IUDbNgxcA2wCHkiypO3/ILANWNOWTa2+FXi3qq4G7gPunf/UJEmzMW0gVNWxqvp5Wz8BHAJWTDFkM/B4VX1QVW8Ao8CGJMuBS6rqhaoq4FHg5q4xu9v6k8AN41cPkqTzY1bPENqtnE8CL7bSnUleTfJwkktbbQXwVtewI622oq1PrJ82pqpOAu8Bl/d4/21JRpKMjI2NzaZ1SdI0ZhwISS4Gvgd8pap+R+f2z8eBdcAx4Jvju/YYXlPUpxpzeqFqV1Wtr6r1y5Ytm2nrkqQZmFEgJLmAThh8p6q+D1BVb1fVqar6A/BtYEPb/Qiwqmv4SuBoq6/sUT9tTJKlwEeBd+YyIUnS3MzkU0YBHgIOVdW3uurLu3b7AnCgre8Fhtsnh1bTeXj8UlUdA04k2diOeTvwVNeYLW39FuD59pxBknSezOTbTq8HvgjsT7Kv1b4G3JZkHZ1bO28CXwKoqoNJ9gCv0fmE0vaqOtXG3QE8AlwEPNMW6ATOY0lG6VwZDM9nUpKk2Zs2EKrqp/S+x//DKcbsBHb2qI8A1/aovw/cOl0vkqRzx79UliQBBoIkqTEQJEmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSpMRAkSYCBIElqDARJEmAgSJIaA0GSBBgIkqTGQJAkAQaCJKkxECRJgIEgSWoMBEkSYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRJgIEiSmmkDIcmqJD9OcijJwSRfbvXLkjyb5PX2emnXmLuSjCY5nOTGrvp1Sfa3bfcnSatfmOSJVn8xydA5mKskaQozuUI4CXy1qj4BbAS2J1kL7ACeq6o1wHPtZ9q2YeAaYBPwQJIl7VgPAtuANW3Z1OpbgXer6mrgPuDeszA3SdIsTBsIVXWsqn7e1k8Ah4AVwGZgd9ttN3BzW98MPF5VH1TVG8AosCHJcuCSqnqhqgp4dMKY8WM9CdwwfvUgSTo/ZvUMod3K+STwInBVVR2DTmgAV7bdVgBvdQ070mor2vrE+mljquok8B5weY/335ZkJMnI2NjYbFqXJE1jxoGQ5GLge8BXqup3U+3ao1ZT1Kcac3qhaldVra+q9cuWLZuuZUnSLMwoEJJcQCcMvlNV32/lt9ttINrr8VY/AqzqGr4SONrqK3vUTxuTZCnwUeCd2U5GkjR3M/mUUYCHgENV9a2uTXuBLW19C/BUV324fXJoNZ2Hxy+120onkmxsx7x9wpjxY90CPN+eM0iSzpOlM9jneuCLwP4k+1rta8A9wJ4kW4FfA7cCVNXBJHuA1+h8Qml7VZ1q4+4AHgEuAp5pC3QC57Eko3SuDIbnNy1J0mxNGwhV9VN63+MHuGGSMTuBnT3qI8C1Perv0wJFktQfM7lC0Fk0tOPpfrcgST351RWSJMBAkCQ1BoIkCTAQJEmNgSBJAgwESVJjIEiSAANBktQYCJIkwECQJDUGgiQJMBAkSY2BIEkCDARJUmMgSJIAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElSYyBIkgADQZLUGAiSJMBAkCQ1BoIkCTAQJEmNgSBJAmYQCEkeTnI8yYGu2t1JfpNkX1tu6tp2V5LRJIeT3NhVvy7J/rbt/iRp9QuTPNHqLyYZOstzlCTNwEyuEB4BNvWo31dV69ryQ4Aka4Fh4Jo25oEkS9r+DwLbgDVtGT/mVuDdqroauA+4d45zkSTNw7SBUFU/Ad6Z4fE2A49X1QdV9QYwCmxIshy4pKpeqKoCHgVu7hqzu60/CdwwfvUgSTp/5vMM4c4kr7ZbSpe22grgra59jrTairY+sX7amKo6CbwHXD6PviRJczDXQHgQ+DiwDjgGfLPVe/3LvqaoTzXmDEm2JRlJMjI2NjarhiVJU5tTIFTV21V1qqr+AHwb2NA2HQFWde26Ejja6it71E8bk2Qp8FEmuUVVVbuqan1VrV+2bNlcWpckTWJOgdCeCYz7AjD+CaS9wHD75NBqOg+PX6qqY8CJJBvb84Hbgae6xmxp67cAz7fnDJKk82jpdDsk+S7wGeCKJEeAbwCfSbKOzq2dN4EvAVTVwSR7gNeAk8D2qjrVDnUHnU8sXQQ80xaAh4DHkozSuTIYPgvzkiTN0rSBUFW39Sg/NMX+O4GdPeojwLU96u8Dt07XhyTp3PIvlSVJgIEgSWoMBEkSYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSpMRAkSYCBIElqDARJEmAgSJIaA0GSBBgIkqTGQJAkAQaCJKkxECRJgIEgSWoMBEkSYCBIkpql/W6gH4Z2PN3vFiRp4HiFIEkCDARJUjNtICR5OMnxJAe6apcleTbJ6+310q5tdyUZTXI4yY1d9euS7G/b7k+SVr8wyROt/mKSobM8R0nSDMzkCuERYNOE2g7guapaAzzXfibJWmAYuKaNeSDJkjbmQWAbsKYt48fcCrxbVVcD9wH3znUykqS5mzYQquonwDsTypuB3W19N3BzV/3xqvqgqt4ARoENSZYDl1TVC1VVwKMTxowf60nghvGrB0nS+TPXZwhXVdUxgPZ6ZauvAN7q2u9Iq61o6xPrp42pqpPAe8Dlvd40ybYkI0lGxsbG5ti6JKmXs/1Qude/7GuK+lRjzixW7aqq9VW1ftmyZXNsUZLUy1wD4e12G4j2erzVjwCruvZbCRxt9ZU96qeNSbIU+Chn3qKSJJ1jcw2EvcCWtr4FeKqrPtw+ObSazsPjl9ptpRNJNrbnA7dPGDN+rFuA59tzBknSeTTtXyon+S7wGeCKJEeAbwD3AHuSbAV+DdwKUFUHk+wBXgNOAtur6lQ71B10PrF0EfBMWwAeAh5LMkrnymD4rMxMkjQr0wZCVd02yaYbJtl/J7CzR30EuLZH/X1aoEiS+se/VJYkAQaCJKkxECRJgIEgSWoMBEkSYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSpMRAkSYCBIElqDARJEmAgSJIaA0GSBBgIkqTGQJAkAQaCJKkxECRJgIEgSWoMBEkSYCBIkpp5BUKSN5PsT7IvyUirXZbk2SSvt9dLu/a/K8loksNJbuyqX9eOM5rk/iSZT1+SpNk7G1cIf1NV66pqfft5B/BcVa0Bnms/k2QtMAxcA2wCHkiypI15ENgGrGnLprPQlyRpFs7FLaPNwO62vhu4uav+eFV9UFVvAKPAhiTLgUuq6oWqKuDRrjGSpPNkvoFQwI+SvJxkW6tdVVXHANrrla2+Anira+yRVlvR1ifWz5BkW5KRJCNjY2PzbF2S1G3pPMdfX1VHk1wJPJvkF1Ps2+u5QE1RP7NYtQvYBbB+/fqe+0iS5mZeVwhVdbS9Hgd+AGwA3m63gWivx9vuR4BVXcNXAkdbfWWPuiTpPJpzICT5cJKPjK8DnwUOAHuBLW23LcBTbX0vMJzkwiSr6Tw8fqndVjqRZGP7dNHtXWMkSefJfG4ZXQX8oH1CdCnwz1X1L0l+BuxJshX4NXArQFUdTLIHeA04CWyvqlPtWHcAjwAXAc+0RZJ0Hs05EKrqV8Bf9qj/F3DDJGN2Ajt71EeAa+faiyRp/vxLZUkSYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAHz/39Mk6RFaWjH03177zfv+fw5Oa5XCJIkwECQJDUGgiQJMBAkSY2BIEkCDARJUmMgSJIAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElSYyBIkgADQZLUGAiSJGCAAiHJpiSHk4wm2dHvfiRpsRmIQEiyBPgn4HPAWuC2JGv725UkLS4DEQjABmC0qn5VVf8DPA5s7nNPkrSoLO13A80K4K2un48AfzVxpyTbgG3tx98nOTzH97sC+O0cxw6KBTOH3DvppgUzhyk4h8Gw0Ocwq/6n+J2aiT+fbMOgBEJ61OqMQtUuYNe83ywZqar18z1OPzmHweAcBsNCn8Og9D8ot4yOAKu6fl4JHO1TL5K0KA1KIPwMWJNkdZIPAcPA3j73JEmLykDcMqqqk0nuBP4VWAI8XFUHz+Fbzvu20wBwDoPBOQyGhT6Hgeg/VWfcqpckLUKDcstIktRnBoIkCViEgbBQvyIjyZtJ9ifZl2Sk1S5L8myS19vrpf3us1uSh5McT3KgqzZpz0nuauflcJIb+9P1H03S/91JftPOw74kN3VtG6j+AZKsSvLjJIeSHEzy5VZfSOdhsjksmHOR5E+TvJTklTaHf2j1wToPVbVoFjoPrH8JfAz4EPAKsLbffc2w9zeBKybU/hHY0dZ3APf2u88J/X0a+BRwYLqe6XxlySvAhcDqdp6WDGD/dwN/12Pfgeu/9bUc+FRb/wjwH63XhXQeJpvDgjkXdP7W6uK2fgHwIrBx0M7DYrtC+P/2FRmbgd1tfTdwc/9aOVNV/QR4Z0J5sp43A49X1QdV9QYwSud89c0k/U9m4PoHqKpjVfXztn4COETnmwEW0nmYbA6TGcQ5VFX9vv14QVuKATsPiy0Qen1FxlT/wxokBfwoycvtKzwArqqqY9D5pQGu7Ft3MzdZzwvp3NyZ5NV2S2n8En/g+08yBHySzr9OF+R5mDAHWEDnIsmSJPuA48CzVTVw52GxBcKMviJjQF1fVZ+i842w25N8ut8NnWUL5dw8CHwcWAccA77Z6gPdf5KLge8BX6mq3021a4/aQMyjxxwW1LmoqlNVtY7ONzFsSHLtFLv3ZQ6LLRAW7FdkVNXR9noc+AGdy8e3kywHaK/H+9fhjE3W84I4N1X1dvvF/gPwbf54GT+w/Se5gM5/SL9TVd9v5QV1HnrNYSGeC4Cq+m/g34BNDNh5WGyBsCC/IiPJh5N8ZHwd+CxwgE7vW9puW4Cn+tPhrEzW815gOMmFSVYDa4CX+tDflMZ/eZsv0DkPMKD9JwnwEHCoqr7VtWnBnIfJ5rCQzkWSZUn+rK1fBPwt8AsG7Tz088l7PxbgJjqfUvgl8PV+9zPDnj9G5xMHrwAHx/sGLgeeA15vr5f1u9cJfX+XzqX8/9L5F8/WqXoGvt7Oy2HgcwPa/2PAfuBVOr+0ywe1/9bTX9O51fAqsK8tNy2w8zDZHBbMuQD+Avj31usB4O9bfaDOg19dIUkCFt8tI0nSJAwESRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSp+T8dG4vhxlSFLAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "plt.hist(preproc_visit_df.age)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f7200905-b08d-467b-ab6c-338961f86bd0",
   "metadata": {},
   "source": [
    "From MIMIC docs:\n",
    "Dates of birth which occur in the present time are not true dates of birth. Furthermore, dates of birth which occur before the year 1900 occur if the patient is older than 89. In these cases, the patient’s age at their first admission has been fixed to 300."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "783f7f71-f9f4-4319-8f08-516bb40e8a47",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Normalize by max age\n",
    "preproc_visit_df.age = preproc_visit_df.age.clip(upper=89)/89.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "6fd8d56b-ec17-45e6-9b17-b5b1fe63c0df",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export age/sex table\n",
    "preproc_visit_df.to_csv(datadir/\"pre-processed/visits_agesex.csv.gz\",columns=['visit_occurrence_id','person_id','sex_F','age'],index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7497772d-042c-4b2d-899f-bcb7934d9905",
   "metadata": {},
   "source": [
    "# Condition occurrence table preprocessing\n",
    "## EDF initiale"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0123e77d-1b7d-4f6a-89fb-53b831c45253",
   "metadata": {},
   "outputs": [],
   "source": [
    "diagnostics_df = pd.read_csv(datadir/\"condition_occurrence.csv.gz\")\n",
    "diag_prof = diagnostics_df.profile_report()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1224d683-cb49-4290-b8af-1563814a6a8f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>condition_occurrence_id</th>\n",
       "      <th>person_id</th>\n",
       "      <th>condition_concept_id</th>\n",
       "      <th>condition_start_date</th>\n",
       "      <th>condition_start_datetime</th>\n",
       "      <th>condition_end_date</th>\n",
       "      <th>condition_end_datetime</th>\n",
       "      <th>condition_type_concept_id</th>\n",
       "      <th>stop_reason</th>\n",
       "      <th>provider_id</th>\n",
       "      <th>visit_occurrence_id</th>\n",
       "      <th>visit_detail_id</th>\n",
       "      <th>condition_source_value</th>\n",
       "      <th>condition_source_concept_id</th>\n",
       "      <th>condition_status_source_value</th>\n",
       "      <th>condition_status_concept_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>336159169</td>\n",
       "      <td>392799187</td>\n",
       "      <td>198464</td>\n",
       "      <td>2153-01-11</td>\n",
       "      <td>2153-01-11 07:15:00</td>\n",
       "      <td>2153-01-13</td>\n",
       "      <td>2153-01-13 11:00:00</td>\n",
       "      <td>38000184</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>30549</td>\n",
       "      <td>NaN</td>\n",
       "      <td>55321</td>\n",
       "      <td>44825520.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>336176080</td>\n",
       "      <td>392807515</td>\n",
       "      <td>195559</td>\n",
       "      <td>2159-08-27</td>\n",
       "      <td>2159-08-27 08:30:00</td>\n",
       "      <td>2159-09-08</td>\n",
       "      <td>2159-09-08 13:50:00</td>\n",
       "      <td>38000184</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33647</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4413</td>\n",
       "      <td>44833580.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>335984902</td>\n",
       "      <td>392784957</td>\n",
       "      <td>437474</td>\n",
       "      <td>2158-03-21</td>\n",
       "      <td>2158-03-21 21:50:00</td>\n",
       "      <td>2158-04-06</td>\n",
       "      <td>2158-04-06 16:04:00</td>\n",
       "      <td>38000184</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10854</td>\n",
       "      <td>NaN</td>\n",
       "      <td>99859</td>\n",
       "      <td>44826039.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>336063211</td>\n",
       "      <td>392795465</td>\n",
       "      <td>40481042</td>\n",
       "      <td>2169-03-24</td>\n",
       "      <td>2169-03-24 11:09:00</td>\n",
       "      <td>2169-04-03</td>\n",
       "      <td>2169-04-03 14:10:00</td>\n",
       "      <td>38000184</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>17475</td>\n",
       "      <td>NaN</td>\n",
       "      <td>42831</td>\n",
       "      <td>44824251.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>336159162</td>\n",
       "      <td>392799187</td>\n",
       "      <td>73614</td>\n",
       "      <td>2154-06-18</td>\n",
       "      <td>2154-06-18 02:27:00</td>\n",
       "      <td>2154-06-20</td>\n",
       "      <td>2154-06-20 16:25:00</td>\n",
       "      <td>38000184</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>30550</td>\n",
       "      <td>NaN</td>\n",
       "      <td>80707</td>\n",
       "      <td>44835139.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   condition_occurrence_id  person_id  condition_concept_id  \\\n",
       "0                336159169  392799187                198464   \n",
       "1                336176080  392807515                195559   \n",
       "2                335984902  392784957                437474   \n",
       "3                336063211  392795465              40481042   \n",
       "4                336159162  392799187                 73614   \n",
       "\n",
       "  condition_start_date condition_start_datetime condition_end_date  \\\n",
       "0           2153-01-11      2153-01-11 07:15:00         2153-01-13   \n",
       "1           2159-08-27      2159-08-27 08:30:00         2159-09-08   \n",
       "2           2158-03-21      2158-03-21 21:50:00         2158-04-06   \n",
       "3           2169-03-24      2169-03-24 11:09:00         2169-04-03   \n",
       "4           2154-06-18      2154-06-18 02:27:00         2154-06-20   \n",
       "\n",
       "  condition_end_datetime  condition_type_concept_id  stop_reason  provider_id  \\\n",
       "0    2153-01-13 11:00:00                   38000184          NaN          NaN   \n",
       "1    2159-09-08 13:50:00                   38000184          NaN          NaN   \n",
       "2    2158-04-06 16:04:00                   38000184          NaN          NaN   \n",
       "3    2169-04-03 14:10:00                   38000184          NaN          NaN   \n",
       "4    2154-06-20 16:25:00                   38000184          NaN          NaN   \n",
       "\n",
       "   visit_occurrence_id  visit_detail_id condition_source_value  \\\n",
       "0                30549              NaN                  55321   \n",
       "1                33647              NaN                   4413   \n",
       "2                10854              NaN                  99859   \n",
       "3                17475              NaN                  42831   \n",
       "4                30550              NaN                  80707   \n",
       "\n",
       "   condition_source_concept_id  condition_status_source_value  \\\n",
       "0                   44825520.0                            NaN   \n",
       "1                   44833580.0                            NaN   \n",
       "2                   44826039.0                            NaN   \n",
       "3                   44824251.0                            NaN   \n",
       "4                   44835139.0                            NaN   \n",
       "\n",
       "   condition_status_concept_id  \n",
       "0                          NaN  \n",
       "1                          NaN  \n",
       "2                          NaN  \n",
       "3                          NaN  \n",
       "4                          NaN  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "diagnostics_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5076719-eef2-4c01-abc5-20845b244377",
   "metadata": {},
   "outputs": [],
   "source": [
    "diag_prof.to_notebook_iframe()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8efbcd0-d56f-40aa-bccd-593fec32ed82",
   "metadata": {},
   "source": [
    "Some background:\n",
    "- On the different types of ICD9 and their mapping to SNOMED: https://www.ohdsi.org/web/wiki/doku.php?id=documentation:vocabulary:icd9cm\n",
    "\n",
    "Observations:\n",
    "- not all condition occurence are taken from the same source (21 unique values for condition_type_id) => check which information we're interested in.\n",
    "- source ICD-9 value is a mix of ICD9 codes and text => assert valid ICD9 code before use, Some free text data come from the mimic ADMISSION table, maybe filtering on the condition_type_id will be sufficient.\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c590e28f-943c-4e0f-b4d9-73971c906740",
   "metadata": {},
   "source": [
    "## Look for the ICD9 reference concept"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "26d292eb-9186-438c-b7db-c32d905b7e1a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ICD9CM non-bill code,ICD9CM non-billable code,45754823\n",
      "ICD9Proc non-bill,ICD9Proc non-billable code,45754824\n",
      "ICD9CM code,ICD9CM code,44819259\n",
      "ICD9CM E code,ICD9CM E code,44819260\n",
      "ICD9CM V code,ICD9CM V code,44819261\n",
      "ICD9Proc Chapter,ICD9Proc Chapter,32746\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Available ICD9 related concept classes\n",
    "subprocess.check_call(['zgrep ICD9 \"$1\"', \"_\", datadir/\"concept_class.csv.gz\"],shell=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "217bdb8a-22da-46fa-860b-d4d9cb8ed694",
   "metadata": {},
   "source": [
    "We should only be interested in ICD9CM and ICD9 CM non-billable codes (E and V are about external causes of injury and health status or care provider encounter respectively), ICD9Proc are procedures. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "549faeed-588a-4755-8f98-e7356fc80dc0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/concept.csv.gz:45754823,ICD9CM non-billable code,Metadata,Concept Class,Concept Class,,OMOP generated,1970-01-01,2015-01-03,D\n",
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/concept_class.csv.gz:ICD9CM non-bill code,ICD9CM non-billable code,45754823\n",
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/drug_exposure.csv.gz:345754823,392784444,0,2151-02-22,2151-02-22 21:30:00,,,,38000180,,,,,,4128794,,96510,14013,397596982,Promod,2001023363,,,\n",
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/fact_relationship.csv.gz:13,345749672,13,345754823,44818791\n",
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/fact_relationship.csv.gz:13,344505334,13,345754823,44818791\n",
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/fact_relationship.csv.gz:13,345749674,13,345754823,44818791\n",
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/fact_relationship.csv.gz:13,345749671,13,345754823,44818791\n",
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/fact_relationship.csv.gz:13,345749673,13,345754823,44818791\n",
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/fact_relationship.csv.gz:36,445754823,21,364482536,44818854\n",
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/fact_relationship.csv.gz:36,457548230,21,376275943,44818854\n",
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/fact_relationship.csv.gz:36,457548231,21,376275944,44818854\n",
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/fact_relationship.csv.gz:36,457548232,21,376275945,44818854\n",
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/fact_relationship.csv.gz:36,457548233,21,376275946,44818854\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# See how we can use the codes above for easy filtering\n",
    "subprocess.check_call(['zgrep -H -m1 45754823 \"$1/\"*.csv.gz', \"_\", datadir],shell=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "80c67380-3929-4254-9f96-4ffcba07fbb8",
   "metadata": {},
   "source": [
    "So using the concept IDs for the concept classes directly seems useless, as there is no relationship defined outside the entries in the concept and concept_class tables...\n",
    "\n",
    "Let's check using 'ICD9CM' string search:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "7df0fd4d-1e17-4794-a749-b2757c042027",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/concept.csv.gz:44819259,ICD9CM code,Metadata,Concept Class,Concept Class,,OMOP generated,1970-01-01,2015-01-03,D\n",
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/concept.csv.gz:44819260,ICD9CM E code,Metadata,Concept Class,Concept Class,,OMOP generated,1970-01-01,2015-01-03,D\n",
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/concept.csv.gz:44819261,ICD9CM V code,Metadata,Concept Class,Concept Class,,OMOP generated,1970-01-01,2015-01-03,D\n",
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/concept.csv.gz:45754823,ICD9CM non-billable code,Metadata,Concept Class,Concept Class,,OMOP generated,1970-01-01,2015-01-03,D\n",
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/concept.csv.gz:44829696,Cholera,Condition,ICD9CM,3-dig nonbill code,,001,1970-01-01,2099-12-31,\n",
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/concept.csv.gz:44835564,Cholera due to vibrio cholerae,Condition,ICD9CM,4-dig billing code,,001.0,1970-01-01,2099-12-31,\n",
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/concept.csv.gz:44823922,Cholera due to vibrio cholerae el tor,Condition,ICD9CM,4-dig billing code,,001.1,1970-01-01,2099-12-31,\n",
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/concept.csv.gz:44827441,\"Cholera, unspecified\",Condition,ICD9CM,4-dig billing code,,001.9,1970-01-01,2099-12-31,\n",
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/concept.csv.gz:44829697,Typhoid and paratyphoid fevers,Condition,ICD9CM,3-dig nonbill code,,002,1970-01-01,2099-12-31,\n",
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/concept.csv.gz:44825090,Typhoid fever,Condition,ICD9CM,4-dig billing code,,002.0,1970-01-01,2099-12-31,\n",
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/concept_class.csv.gz:ICD9CM non-bill code,ICD9CM non-billable code,45754823\n",
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/concept_class.csv.gz:ICD9CM code,ICD9CM code,44819259\n",
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/concept_class.csv.gz:ICD9CM E code,ICD9CM E code,44819260\n",
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/concept_class.csv.gz:ICD9CM V code,ICD9CM V code,44819261\n",
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/relationship.csv.gz:ICD9CM - MedDRA,ICD-9-CM to MedDRA (MSSO),0,0,MedDRA - ICD9CM,44819286\n",
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/relationship.csv.gz:MedDRA - ICD9CM,MedDRA to ICD-9-CM (MSSO),0,0,ICD9CM - MedDRA,44819287\n",
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/relationship.csv.gz:ICD9CM - FDB Ind,ICD-9-CM to FDB Indication/Contraindication (FDB),0,0,FDB Ind - ICD9CM,44819288\n",
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/relationship.csv.gz:FDB Ind - ICD9CM,FDB Indication/Contraindication to ICD-9-CM (FDB),0,0,ICD9CM - FDB Ind,44819289\n",
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/vocabulary.csv.gz:ICD9CM,\"International Classification of Diseases, Ninth Revision, Clinical Modification, Volume 1 and 2 (NCHS)\",http://www.cms.gov/Medicare/Coding/ICD9ProviderDiagnosticCodes/codes.html,ICD9CM v32 master descriptions,5046\n",
      "/home/quentin/Projects/completion_pmsi/data/mimic-omop/vocabulary.csv.gz:ICD9Proc,\"International Classification of Diseases, Ninth Revision, Clinical Modification, Volume 3 (NCHS)\",http://www.cms.gov/Medicare/Coding/ICD9ProviderDiagnosticCodes/codes.html,ICD9CM v32 master descriptions,44819099\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subprocess.check_call(['zgrep -H -m10 ICD9CM \"$1/\"*.csv.gz', \"_\", datadir],shell=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1cfc4d6c-2c5b-4872-9c7e-c8b92d8b8587",
   "metadata": {},
   "source": [
    "Filter concepts based on vocabulary_id containing the 'ICD9CM' string:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b9178bd8-0d6b-4381-81a7-44a8b56f90fe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Uniques domain_id containing ICD9CM: ['Condition' 'Observation' 'Measurement' 'Procedure']\n",
      "Uniques vocabulary_id containing ICD9CM: ['ICD9CM']\n",
      "Uniques concept_class_id containing ICD9CM: ['3-dig nonbill code' '4-dig billing code' '4-dig nonbill code'\n",
      " '5-dig billing code' '3-dig billing code' 'ICD9CM code'\n",
      " '3-dig nonbill E code' '4-dig billing E code' '3-dig billing E code'\n",
      " '3-dig nonbill V code' '4-dig billing V code' '4-dig nonbill V code'\n",
      " '5-dig billing V code' '3-dig billing V code']\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>concept_id</th>\n",
       "      <th>concept_name</th>\n",
       "      <th>domain_id</th>\n",
       "      <th>vocabulary_id</th>\n",
       "      <th>concept_class_id</th>\n",
       "      <th>standard_concept</th>\n",
       "      <th>concept_code</th>\n",
       "      <th>valid_start_date</th>\n",
       "      <th>valid_end_date</th>\n",
       "      <th>invalid_reason</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>116767</th>\n",
       "      <td>44829696</td>\n",
       "      <td>Cholera</td>\n",
       "      <td>Condition</td>\n",
       "      <td>ICD9CM</td>\n",
       "      <td>3-dig nonbill code</td>\n",
       "      <td>NaN</td>\n",
       "      <td>001</td>\n",
       "      <td>1970-01-01</td>\n",
       "      <td>2099-12-31</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116768</th>\n",
       "      <td>44835564</td>\n",
       "      <td>Cholera due to vibrio cholerae</td>\n",
       "      <td>Condition</td>\n",
       "      <td>ICD9CM</td>\n",
       "      <td>4-dig billing code</td>\n",
       "      <td>NaN</td>\n",
       "      <td>001.0</td>\n",
       "      <td>1970-01-01</td>\n",
       "      <td>2099-12-31</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116769</th>\n",
       "      <td>44823922</td>\n",
       "      <td>Cholera due to vibrio cholerae el tor</td>\n",
       "      <td>Condition</td>\n",
       "      <td>ICD9CM</td>\n",
       "      <td>4-dig billing code</td>\n",
       "      <td>NaN</td>\n",
       "      <td>001.1</td>\n",
       "      <td>1970-01-01</td>\n",
       "      <td>2099-12-31</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116770</th>\n",
       "      <td>44827441</td>\n",
       "      <td>Cholera, unspecified</td>\n",
       "      <td>Condition</td>\n",
       "      <td>ICD9CM</td>\n",
       "      <td>4-dig billing code</td>\n",
       "      <td>NaN</td>\n",
       "      <td>001.9</td>\n",
       "      <td>1970-01-01</td>\n",
       "      <td>2099-12-31</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116771</th>\n",
       "      <td>44829697</td>\n",
       "      <td>Typhoid and paratyphoid fevers</td>\n",
       "      <td>Condition</td>\n",
       "      <td>ICD9CM</td>\n",
       "      <td>3-dig nonbill code</td>\n",
       "      <td>NaN</td>\n",
       "      <td>002</td>\n",
       "      <td>1970-01-01</td>\n",
       "      <td>2099-12-31</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        concept_id                           concept_name  domain_id  \\\n",
       "116767    44829696                                Cholera  Condition   \n",
       "116768    44835564         Cholera due to vibrio cholerae  Condition   \n",
       "116769    44823922  Cholera due to vibrio cholerae el tor  Condition   \n",
       "116770    44827441                   Cholera, unspecified  Condition   \n",
       "116771    44829697         Typhoid and paratyphoid fevers  Condition   \n",
       "\n",
       "       vocabulary_id    concept_class_id standard_concept concept_code  \\\n",
       "116767        ICD9CM  3-dig nonbill code              NaN          001   \n",
       "116768        ICD9CM  4-dig billing code              NaN        001.0   \n",
       "116769        ICD9CM  4-dig billing code              NaN        001.1   \n",
       "116770        ICD9CM  4-dig billing code              NaN        001.9   \n",
       "116771        ICD9CM  3-dig nonbill code              NaN          002   \n",
       "\n",
       "       valid_start_date valid_end_date invalid_reason  \n",
       "116767       1970-01-01     2099-12-31            NaN  \n",
       "116768       1970-01-01     2099-12-31            NaN  \n",
       "116769       1970-01-01     2099-12-31            NaN  \n",
       "116770       1970-01-01     2099-12-31            NaN  \n",
       "116771       1970-01-01     2099-12-31            NaN  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Query the corresponding rows\n",
    "icd9_concepts = concept_df.query('vocabulary_id.str.contains(\"ICD9CM\",na=False)')\n",
    "# Check that it gives a single vocabulary_id\n",
    "print(\"Uniques domain_id containing ICD9CM: \" + str(icd9_concepts.domain_id.unique()))\n",
    "print(\"Uniques vocabulary_id containing ICD9CM: \" + str(icd9_concepts.vocabulary_id.unique()))\n",
    "print(\"Uniques concept_class_id containing ICD9CM: \" + str(icd9_concepts.concept_class_id.unique()))\n",
    "icd9_concepts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "e1a8e5a6-2fd3-4475-870e-3d2f718e7bc6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>concept_class_id</th>\n",
       "      <th>concept_class_name</th>\n",
       "      <th>concept_class_concept_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>3-dig billing code</td>\n",
       "      <td>3-digit billing code</td>\n",
       "      <td>45754868</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>4-dig billing code</td>\n",
       "      <td>4-digit billing code</td>\n",
       "      <td>45754869</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>5-dig billing code</td>\n",
       "      <td>5-digit billing code</td>\n",
       "      <td>45754870</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>4-dig billing E code</td>\n",
       "      <td>4-digit billing E code</td>\n",
       "      <td>45754871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>3-dig billing V code</td>\n",
       "      <td>3-digit billing V code</td>\n",
       "      <td>45754873</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>4-dig billing V code</td>\n",
       "      <td>4-digit billing V code</td>\n",
       "      <td>45754874</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>5-dig billing V code</td>\n",
       "      <td>5-digit billing V code</td>\n",
       "      <td>45754875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>3-dig nonbill code</td>\n",
       "      <td>3-digit non-billing code</td>\n",
       "      <td>45754876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>4-dig nonbill code</td>\n",
       "      <td>4-digit non-billing code</td>\n",
       "      <td>45754877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>3-dig nonbill V code</td>\n",
       "      <td>3-digit non-billing V code</td>\n",
       "      <td>45754878</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>3-dig nonbill E code</td>\n",
       "      <td>3-digit non-billing E code</td>\n",
       "      <td>45754879</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>3-dig billing E code</td>\n",
       "      <td>3-digit billing E code</td>\n",
       "      <td>46233669</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>176</th>\n",
       "      <td>4-dig nonbill V code</td>\n",
       "      <td>4-digit non-billing V code</td>\n",
       "      <td>45756722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>268</th>\n",
       "      <td>ICD9CM code</td>\n",
       "      <td>ICD9CM code</td>\n",
       "      <td>44819259</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         concept_class_id          concept_class_name  \\\n",
       "52     3-dig billing code        3-digit billing code   \n",
       "53     4-dig billing code        4-digit billing code   \n",
       "54     5-dig billing code        5-digit billing code   \n",
       "55   4-dig billing E code      4-digit billing E code   \n",
       "57   3-dig billing V code      3-digit billing V code   \n",
       "58   4-dig billing V code      4-digit billing V code   \n",
       "59   5-dig billing V code      5-digit billing V code   \n",
       "60     3-dig nonbill code    3-digit non-billing code   \n",
       "61     4-dig nonbill code    4-digit non-billing code   \n",
       "62   3-dig nonbill V code  3-digit non-billing V code   \n",
       "63   3-dig nonbill E code  3-digit non-billing E code   \n",
       "97   3-dig billing E code      3-digit billing E code   \n",
       "176  4-dig nonbill V code  4-digit non-billing V code   \n",
       "268           ICD9CM code                 ICD9CM code   \n",
       "\n",
       "     concept_class_concept_id  \n",
       "52                   45754868  \n",
       "53                   45754869  \n",
       "54                   45754870  \n",
       "55                   45754871  \n",
       "57                   45754873  \n",
       "58                   45754874  \n",
       "59                   45754875  \n",
       "60                   45754876  \n",
       "61                   45754877  \n",
       "62                   45754878  \n",
       "63                   45754879  \n",
       "97                   46233669  \n",
       "176                  45756722  \n",
       "268                  44819259  "
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check whether they all existed in the concept_class table\n",
    "assert all(pd.Series(icd9_concepts.concept_class_id.unique()).isin(concept_class_df.concept_class_id))\n",
    "concept_class_df[concept_class_df['concept_class_id'].isin(icd9_concepts.concept_class_id.unique())]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b956dbe-6fc5-4ec3-82a5-23c761d96475",
   "metadata": {},
   "source": [
    "Let's keep all of them for now and see later if we need to remove the E and V classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "9aa64355-6e35-46f2-b83d-e0351fab30cb",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>concept_id</th>\n",
       "      <th>concept_name</th>\n",
       "      <th>domain_id</th>\n",
       "      <th>vocabulary_id</th>\n",
       "      <th>concept_class_id</th>\n",
       "      <th>standard_concept</th>\n",
       "      <th>concept_code</th>\n",
       "      <th>valid_start_date</th>\n",
       "      <th>valid_end_date</th>\n",
       "      <th>invalid_reason</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8396</th>\n",
       "      <td>38000184</td>\n",
       "      <td>Inpatient detail - 1st position</td>\n",
       "      <td>Type Concept</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OMOP4822060</td>\n",
       "      <td>1970-01-01</td>\n",
       "      <td>2099-12-31</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8397</th>\n",
       "      <td>38000185</td>\n",
       "      <td>Inpatient detail - 2nd position</td>\n",
       "      <td>Type Concept</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OMOP4822061</td>\n",
       "      <td>1970-01-01</td>\n",
       "      <td>2099-12-31</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8398</th>\n",
       "      <td>38000186</td>\n",
       "      <td>Inpatient detail - 3rd position</td>\n",
       "      <td>Type Concept</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OMOP4822062</td>\n",
       "      <td>1970-01-01</td>\n",
       "      <td>2099-12-31</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8399</th>\n",
       "      <td>38000187</td>\n",
       "      <td>Inpatient detail - 4th position</td>\n",
       "      <td>Type Concept</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OMOP4822063</td>\n",
       "      <td>1970-01-01</td>\n",
       "      <td>2099-12-31</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8400</th>\n",
       "      <td>38000188</td>\n",
       "      <td>Inpatient detail - 5th position</td>\n",
       "      <td>Type Concept</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OMOP4822064</td>\n",
       "      <td>1970-01-01</td>\n",
       "      <td>2099-12-31</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8401</th>\n",
       "      <td>38000189</td>\n",
       "      <td>Inpatient detail - 6th position</td>\n",
       "      <td>Type Concept</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OMOP4822065</td>\n",
       "      <td>1970-01-01</td>\n",
       "      <td>2099-12-31</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8402</th>\n",
       "      <td>38000190</td>\n",
       "      <td>Inpatient detail - 7th position</td>\n",
       "      <td>Type Concept</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OMOP4822066</td>\n",
       "      <td>1970-01-01</td>\n",
       "      <td>2099-12-31</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8403</th>\n",
       "      <td>38000191</td>\n",
       "      <td>Inpatient detail - 8th position</td>\n",
       "      <td>Type Concept</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OMOP4822067</td>\n",
       "      <td>1970-01-01</td>\n",
       "      <td>2099-12-31</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8404</th>\n",
       "      <td>38000192</td>\n",
       "      <td>Inpatient detail - 9th position</td>\n",
       "      <td>Type Concept</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OMOP4822068</td>\n",
       "      <td>1970-01-01</td>\n",
       "      <td>2099-12-31</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8405</th>\n",
       "      <td>38000193</td>\n",
       "      <td>Inpatient detail - 10th position</td>\n",
       "      <td>Type Concept</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OMOP4822069</td>\n",
       "      <td>1970-01-01</td>\n",
       "      <td>2099-12-31</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8406</th>\n",
       "      <td>38000194</td>\n",
       "      <td>Inpatient detail - 11th position</td>\n",
       "      <td>Type Concept</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OMOP4822070</td>\n",
       "      <td>1970-01-01</td>\n",
       "      <td>2099-12-31</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8407</th>\n",
       "      <td>38000195</td>\n",
       "      <td>Inpatient detail - 12th position</td>\n",
       "      <td>Type Concept</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OMOP4822071</td>\n",
       "      <td>1970-01-01</td>\n",
       "      <td>2099-12-31</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8408</th>\n",
       "      <td>38000196</td>\n",
       "      <td>Inpatient detail - 13th position</td>\n",
       "      <td>Type Concept</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OMOP4822072</td>\n",
       "      <td>1970-01-01</td>\n",
       "      <td>2099-12-31</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8409</th>\n",
       "      <td>38000197</td>\n",
       "      <td>Inpatient detail - 14th position</td>\n",
       "      <td>Type Concept</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OMOP4822073</td>\n",
       "      <td>1970-01-01</td>\n",
       "      <td>2099-12-31</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8410</th>\n",
       "      <td>38000198</td>\n",
       "      <td>Inpatient detail - 15th position</td>\n",
       "      <td>Type Concept</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OMOP4822074</td>\n",
       "      <td>1970-01-01</td>\n",
       "      <td>2099-12-31</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8460</th>\n",
       "      <td>42894222</td>\n",
       "      <td>EHR Chief Complaint</td>\n",
       "      <td>Type Concept</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OMOP4822124</td>\n",
       "      <td>1970-01-01</td>\n",
       "      <td>2099-12-31</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8466</th>\n",
       "      <td>44818709</td>\n",
       "      <td>Inpatient detail - 16th position</td>\n",
       "      <td>Type Concept</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OMOP4822130</td>\n",
       "      <td>1970-01-01</td>\n",
       "      <td>2099-12-31</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8467</th>\n",
       "      <td>44818710</td>\n",
       "      <td>Inpatient detail - 17th position</td>\n",
       "      <td>Type Concept</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OMOP4822131</td>\n",
       "      <td>1970-01-01</td>\n",
       "      <td>2099-12-31</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8468</th>\n",
       "      <td>44818711</td>\n",
       "      <td>Inpatient detail - 18th position</td>\n",
       "      <td>Type Concept</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OMOP4822132</td>\n",
       "      <td>1970-01-01</td>\n",
       "      <td>2099-12-31</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8469</th>\n",
       "      <td>44818712</td>\n",
       "      <td>Inpatient detail - 19th position</td>\n",
       "      <td>Type Concept</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OMOP4822133</td>\n",
       "      <td>1970-01-01</td>\n",
       "      <td>2099-12-31</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8470</th>\n",
       "      <td>44818713</td>\n",
       "      <td>Inpatient detail - 20th position</td>\n",
       "      <td>Type Concept</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>Condition Type</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OMOP4822134</td>\n",
       "      <td>1970-01-01</td>\n",
       "      <td>2099-12-31</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      concept_id                      concept_name     domain_id  \\\n",
       "8396    38000184   Inpatient detail - 1st position  Type Concept   \n",
       "8397    38000185   Inpatient detail - 2nd position  Type Concept   \n",
       "8398    38000186   Inpatient detail - 3rd position  Type Concept   \n",
       "8399    38000187   Inpatient detail - 4th position  Type Concept   \n",
       "8400    38000188   Inpatient detail - 5th position  Type Concept   \n",
       "8401    38000189   Inpatient detail - 6th position  Type Concept   \n",
       "8402    38000190   Inpatient detail - 7th position  Type Concept   \n",
       "8403    38000191   Inpatient detail - 8th position  Type Concept   \n",
       "8404    38000192   Inpatient detail - 9th position  Type Concept   \n",
       "8405    38000193  Inpatient detail - 10th position  Type Concept   \n",
       "8406    38000194  Inpatient detail - 11th position  Type Concept   \n",
       "8407    38000195  Inpatient detail - 12th position  Type Concept   \n",
       "8408    38000196  Inpatient detail - 13th position  Type Concept   \n",
       "8409    38000197  Inpatient detail - 14th position  Type Concept   \n",
       "8410    38000198  Inpatient detail - 15th position  Type Concept   \n",
       "8460    42894222               EHR Chief Complaint  Type Concept   \n",
       "8466    44818709  Inpatient detail - 16th position  Type Concept   \n",
       "8467    44818710  Inpatient detail - 17th position  Type Concept   \n",
       "8468    44818711  Inpatient detail - 18th position  Type Concept   \n",
       "8469    44818712  Inpatient detail - 19th position  Type Concept   \n",
       "8470    44818713  Inpatient detail - 20th position  Type Concept   \n",
       "\n",
       "       vocabulary_id concept_class_id standard_concept concept_code  \\\n",
       "8396  Condition Type   Condition Type              NaN  OMOP4822060   \n",
       "8397  Condition Type   Condition Type              NaN  OMOP4822061   \n",
       "8398  Condition Type   Condition Type              NaN  OMOP4822062   \n",
       "8399  Condition Type   Condition Type              NaN  OMOP4822063   \n",
       "8400  Condition Type   Condition Type              NaN  OMOP4822064   \n",
       "8401  Condition Type   Condition Type              NaN  OMOP4822065   \n",
       "8402  Condition Type   Condition Type              NaN  OMOP4822066   \n",
       "8403  Condition Type   Condition Type              NaN  OMOP4822067   \n",
       "8404  Condition Type   Condition Type              NaN  OMOP4822068   \n",
       "8405  Condition Type   Condition Type              NaN  OMOP4822069   \n",
       "8406  Condition Type   Condition Type              NaN  OMOP4822070   \n",
       "8407  Condition Type   Condition Type              NaN  OMOP4822071   \n",
       "8408  Condition Type   Condition Type              NaN  OMOP4822072   \n",
       "8409  Condition Type   Condition Type              NaN  OMOP4822073   \n",
       "8410  Condition Type   Condition Type              NaN  OMOP4822074   \n",
       "8460  Condition Type   Condition Type              NaN  OMOP4822124   \n",
       "8466  Condition Type   Condition Type              NaN  OMOP4822130   \n",
       "8467  Condition Type   Condition Type              NaN  OMOP4822131   \n",
       "8468  Condition Type   Condition Type              NaN  OMOP4822132   \n",
       "8469  Condition Type   Condition Type              NaN  OMOP4822133   \n",
       "8470  Condition Type   Condition Type              NaN  OMOP4822134   \n",
       "\n",
       "     valid_start_date valid_end_date invalid_reason  \n",
       "8396       1970-01-01     2099-12-31            NaN  \n",
       "8397       1970-01-01     2099-12-31            NaN  \n",
       "8398       1970-01-01     2099-12-31            NaN  \n",
       "8399       1970-01-01     2099-12-31            NaN  \n",
       "8400       1970-01-01     2099-12-31            NaN  \n",
       "8401       1970-01-01     2099-12-31            NaN  \n",
       "8402       1970-01-01     2099-12-31            NaN  \n",
       "8403       1970-01-01     2099-12-31            NaN  \n",
       "8404       1970-01-01     2099-12-31            NaN  \n",
       "8405       1970-01-01     2099-12-31            NaN  \n",
       "8406       1970-01-01     2099-12-31            NaN  \n",
       "8407       1970-01-01     2099-12-31            NaN  \n",
       "8408       1970-01-01     2099-12-31            NaN  \n",
       "8409       1970-01-01     2099-12-31            NaN  \n",
       "8410       1970-01-01     2099-12-31            NaN  \n",
       "8460       1970-01-01     2099-12-31            NaN  \n",
       "8466       1970-01-01     2099-12-31            NaN  \n",
       "8467       1970-01-01     2099-12-31            NaN  \n",
       "8468       1970-01-01     2099-12-31            NaN  \n",
       "8469       1970-01-01     2099-12-31            NaN  \n",
       "8470       1970-01-01     2099-12-31            NaN  "
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Condition type concepts id meaning\n",
    "unique_condition_types = diagnostics_df.condition_type_concept_id.unique()\n",
    "concept_df[concept_df['concept_id'].isin(unique_condition_types)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "f47f9a2b-ecb0-4349-891f-66dc4cfcbcdb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assert all icd9 codes are unique\n",
    "assert all(icd9_concepts.concept_code.value_counts()==1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ecef73d-d18f-49fb-ac77-70092983a1a4",
   "metadata": {},
   "source": [
    "## Filtering ICD9 codes produces a table of similar size to the original DIAGNOSIS_ICD\n",
    "There are many sources of conditions inside the condition_occurrence table let's see if by filtering out broad ICD9 codes we obtain a different set of unique conditions:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3f3eb973-4986-4749-80fd-348830b11b52",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(651000, 16)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "icd_filtered_diagnostics_df = diagnostics_df[diagnostics_df['condition_source_concept_id'].isin(icd9_concepts.concept_id)]\n",
    "unique_condition_types = icd_filtered_diagnostics_df.condition_type_concept_id.unique()\n",
    "concept_df[concept_df['concept_id'].isin(unique_condition_types)]\n",
    "icd_filtered_diagnostics_df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c16fdcce-6bb7-4fa5-8e9c-1f995f0192fa",
   "metadata": {},
   "source": [
    "This is about the size of the DIAGNOSES_ICD table of the original MIMIC (651047)\n",
    "\n",
    "By filtering ICD9 related codes we have eliminated all the 'EHR chief complaint' source of conditions, let's see if doing it the other way around gives the same results:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "dda3a7ac-794c-49ed-9fc5-dbbbe48bb17b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(716595, 16)\n",
      "True\n"
     ]
    }
   ],
   "source": [
    "notEHR_chief_diagnostics_df = diagnostics_df.query('condition_source_concept_id != \"EHR Chief Complaint\"')\n",
    "print(notEHR_chief_diagnostics_df.shape)\n",
    "print(all(icd_filtered_diagnostics_df['condition_occurrence_id'].isin(notEHR_chief_diagnostics_df.condition_occurrence_id)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ddf7609-ecb9-4357-a359-8cad5b6bbcda",
   "metadata": {},
   "source": [
    "So most of the non ICD9 values are 'EHR chief complaint', but some are not. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1668a792-d762-4cea-b7fe-7d1fb4cb964d",
   "metadata": {},
   "outputs": [],
   "source": [
    "notEHR_chief_not_ICD_concept = notEHR_chief_diagnostics_df[~notEHR_chief_diagnostics_df['condition_source_concept_id'].isin(icd9_concepts.concept_id)]\n",
    "notEHR_chief_not_ICD_concept.profile_report().to_notebook_iframe()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f5c43b5b-3714-496e-99f7-554d87b7b9e4",
   "metadata": {},
   "source": [
    "~ 70% of the ~65000 condition reported as text and not 'EHR chief complaint' have been remapped to 950 distinct SNOMED concepts. Quite a few are still more common than the 100 cases bar used in the article from Sagi et al. Let's have a quick look on the existence of such entries in the original mimic table:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "e9b167b1-8438-42fc-847b-1b2abdb2e677",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "47\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROW_ID</th>\n",
       "      <th>SUBJECT_ID</th>\n",
       "      <th>HADM_ID</th>\n",
       "      <th>SEQ_NUM</th>\n",
       "      <th>ICD9_CODE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5251</th>\n",
       "      <td>5029</td>\n",
       "      <td>417</td>\n",
       "      <td>102633</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8354</th>\n",
       "      <td>7969</td>\n",
       "      <td>690</td>\n",
       "      <td>174817</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9691</th>\n",
       "      <td>10924</td>\n",
       "      <td>937</td>\n",
       "      <td>177274</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18667</th>\n",
       "      <td>16813</td>\n",
       "      <td>1490</td>\n",
       "      <td>135580</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27313</th>\n",
       "      <td>37428</td>\n",
       "      <td>3369</td>\n",
       "      <td>126808</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39787</th>\n",
       "      <td>53456</td>\n",
       "      <td>4791</td>\n",
       "      <td>159800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55284</th>\n",
       "      <td>45143</td>\n",
       "      <td>4023</td>\n",
       "      <td>170307</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82223</th>\n",
       "      <td>98541</td>\n",
       "      <td>8818</td>\n",
       "      <td>111608</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92219</th>\n",
       "      <td>84252</td>\n",
       "      <td>7544</td>\n",
       "      <td>116945</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97533</th>\n",
       "      <td>88308</td>\n",
       "      <td>7880</td>\n",
       "      <td>184025</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108374</th>\n",
       "      <td>110935</td>\n",
       "      <td>9896</td>\n",
       "      <td>171183</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108930</th>\n",
       "      <td>112265</td>\n",
       "      <td>9998</td>\n",
       "      <td>108282</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118460</th>\n",
       "      <td>101079</td>\n",
       "      <td>9035</td>\n",
       "      <td>188023</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119063</th>\n",
       "      <td>129512</td>\n",
       "      <td>11519</td>\n",
       "      <td>123220</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123788</th>\n",
       "      <td>131932</td>\n",
       "      <td>11740</td>\n",
       "      <td>129620</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>131059</th>\n",
       "      <td>155602</td>\n",
       "      <td>13890</td>\n",
       "      <td>113170</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132492</th>\n",
       "      <td>128656</td>\n",
       "      <td>11438</td>\n",
       "      <td>154602</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137498</th>\n",
       "      <td>126401</td>\n",
       "      <td>11245</td>\n",
       "      <td>148742</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147765</th>\n",
       "      <td>142003</td>\n",
       "      <td>12607</td>\n",
       "      <td>190435</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151120</th>\n",
       "      <td>148860</td>\n",
       "      <td>13262</td>\n",
       "      <td>160049</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154704</th>\n",
       "      <td>152077</td>\n",
       "      <td>13567</td>\n",
       "      <td>110220</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164332</th>\n",
       "      <td>162208</td>\n",
       "      <td>14497</td>\n",
       "      <td>135312</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>176315</th>\n",
       "      <td>159139</td>\n",
       "      <td>14219</td>\n",
       "      <td>102096</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>188906</th>\n",
       "      <td>192550</td>\n",
       "      <td>17269</td>\n",
       "      <td>176570</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191309</th>\n",
       "      <td>180428</td>\n",
       "      <td>16213</td>\n",
       "      <td>199739</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205851</th>\n",
       "      <td>213073</td>\n",
       "      <td>19092</td>\n",
       "      <td>186323</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>217231</th>\n",
       "      <td>200160</td>\n",
       "      <td>17964</td>\n",
       "      <td>196087</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>217979</th>\n",
       "      <td>206919</td>\n",
       "      <td>18605</td>\n",
       "      <td>144751</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>219132</th>\n",
       "      <td>198343</td>\n",
       "      <td>17796</td>\n",
       "      <td>142890</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>224475</th>\n",
       "      <td>200636</td>\n",
       "      <td>18006</td>\n",
       "      <td>140889</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>232291</th>\n",
       "      <td>219068</td>\n",
       "      <td>19617</td>\n",
       "      <td>194017</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>265402</th>\n",
       "      <td>260938</td>\n",
       "      <td>23407</td>\n",
       "      <td>119146</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>267373</th>\n",
       "      <td>268004</td>\n",
       "      <td>23987</td>\n",
       "      <td>186175</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>272267</th>\n",
       "      <td>270162</td>\n",
       "      <td>24188</td>\n",
       "      <td>163059</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>274944</th>\n",
       "      <td>257173</td>\n",
       "      <td>23081</td>\n",
       "      <td>105970</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>296791</th>\n",
       "      <td>315693</td>\n",
       "      <td>27934</td>\n",
       "      <td>152077</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>305160</th>\n",
       "      <td>277864</td>\n",
       "      <td>24886</td>\n",
       "      <td>166997</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>305335</th>\n",
       "      <td>278820</td>\n",
       "      <td>24975</td>\n",
       "      <td>109963</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>310211</th>\n",
       "      <td>332816</td>\n",
       "      <td>29175</td>\n",
       "      <td>152368</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>313253</th>\n",
       "      <td>331689</td>\n",
       "      <td>29096</td>\n",
       "      <td>182322</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>323891</th>\n",
       "      <td>298691</td>\n",
       "      <td>26690</td>\n",
       "      <td>196643</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>331803</th>\n",
       "      <td>325963</td>\n",
       "      <td>28700</td>\n",
       "      <td>104588</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333000</th>\n",
       "      <td>349152</td>\n",
       "      <td>30405</td>\n",
       "      <td>154166</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333526</th>\n",
       "      <td>321859</td>\n",
       "      <td>28388</td>\n",
       "      <td>173766</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>356738</th>\n",
       "      <td>367323</td>\n",
       "      <td>31928</td>\n",
       "      <td>153208</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>360813</th>\n",
       "      <td>366623</td>\n",
       "      <td>31866</td>\n",
       "      <td>182252</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>386777</th>\n",
       "      <td>355880</td>\n",
       "      <td>30940</td>\n",
       "      <td>110368</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>549256</th>\n",
       "      <td>532929</td>\n",
       "      <td>73686</td>\n",
       "      <td>112990</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>567020</th>\n",
       "      <td>587430</td>\n",
       "      <td>85828</td>\n",
       "      <td>120813</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0778</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ROW_ID  SUBJECT_ID  HADM_ID  SEQ_NUM ICD9_CODE\n",
       "5251      5029         417   102633      NaN       NaN\n",
       "8354      7969         690   174817      NaN       NaN\n",
       "9691     10924         937   177274      NaN       NaN\n",
       "18667    16813        1490   135580      NaN       NaN\n",
       "27313    37428        3369   126808      NaN       NaN\n",
       "39787    53456        4791   159800      NaN       NaN\n",
       "55284    45143        4023   170307      NaN       NaN\n",
       "82223    98541        8818   111608      NaN       NaN\n",
       "92219    84252        7544   116945      NaN       NaN\n",
       "97533    88308        7880   184025      NaN       NaN\n",
       "108374  110935        9896   171183      NaN       NaN\n",
       "108930  112265        9998   108282      NaN       NaN\n",
       "118460  101079        9035   188023      NaN       NaN\n",
       "119063  129512       11519   123220      NaN       NaN\n",
       "123788  131932       11740   129620      NaN       NaN\n",
       "131059  155602       13890   113170      NaN       NaN\n",
       "132492  128656       11438   154602      NaN       NaN\n",
       "137498  126401       11245   148742      NaN       NaN\n",
       "147765  142003       12607   190435      NaN       NaN\n",
       "151120  148860       13262   160049      NaN       NaN\n",
       "154704  152077       13567   110220      NaN       NaN\n",
       "164332  162208       14497   135312      9.0      0778\n",
       "176315  159139       14219   102096      NaN       NaN\n",
       "188906  192550       17269   176570      NaN       NaN\n",
       "191309  180428       16213   199739      NaN       NaN\n",
       "205851  213073       19092   186323      NaN       NaN\n",
       "217231  200160       17964   196087      NaN       NaN\n",
       "217979  206919       18605   144751      NaN       NaN\n",
       "219132  198343       17796   142890      NaN       NaN\n",
       "224475  200636       18006   140889      NaN       NaN\n",
       "232291  219068       19617   194017      NaN       NaN\n",
       "265402  260938       23407   119146      NaN       NaN\n",
       "267373  268004       23987   186175      NaN       NaN\n",
       "272267  270162       24188   163059      NaN       NaN\n",
       "274944  257173       23081   105970      NaN       NaN\n",
       "296791  315693       27934   152077      NaN       NaN\n",
       "305160  277864       24886   166997      NaN       NaN\n",
       "305335  278820       24975   109963      NaN       NaN\n",
       "310211  332816       29175   152368      NaN       NaN\n",
       "313253  331689       29096   182322      NaN       NaN\n",
       "323891  298691       26690   196643      NaN       NaN\n",
       "331803  325963       28700   104588      NaN       NaN\n",
       "333000  349152       30405   154166      NaN       NaN\n",
       "333526  321859       28388   173766      NaN       NaN\n",
       "356738  367323       31928   153208      NaN       NaN\n",
       "360813  366623       31866   182252      NaN       NaN\n",
       "386777  355880       30940   110368      NaN       NaN\n",
       "549256  532929       73686   112990      NaN       NaN\n",
       "567020  587430       85828   120813     10.0      0778"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mimic_diagnostics_df = pd.read_csv(get_datadir()/\"mimic/mimic-iii-clinical-database-1.4/DIAGNOSES_ICD.csv.gz\")\n",
    "print(mimic_diagnostics_df['ICD9_CODE'].isna().sum())\n",
    "mimic_diagnostics_df[mimic_diagnostics_df['ICD9_CODE'].isin(notEHR_chief_not_ICD_concept.condition_source_value)]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e2fffda-58a2-4b60-9637-85efc2f2e1b1",
   "metadata": {},
   "source": [
    "So there are 47 entries with NA diagnostic, and 2 viral conjonctivitis that have been dropped in the ETL for a reason i do not understand, and most likely 1 or 2 things created"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "9d6701ef-5fdc-44d3-a41a-f58fb17328c4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan, '0778'], dtype=object)"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mimic_diagnostics_df[mimic_diagnostics_df['ICD9_CODE'].isin(notEHR_chief_not_ICD_concept.condition_source_value)].ICD9_CODE.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "0c27e068-c3f8-4db6-b232-a598310eb9d4",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>concept_id</th>\n",
       "      <th>concept_name</th>\n",
       "      <th>domain_id</th>\n",
       "      <th>vocabulary_id</th>\n",
       "      <th>concept_class_id</th>\n",
       "      <th>standard_concept</th>\n",
       "      <th>concept_code</th>\n",
       "      <th>valid_start_date</th>\n",
       "      <th>valid_end_date</th>\n",
       "      <th>invalid_reason</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>117785</th>\n",
       "      <td>44825158</td>\n",
       "      <td>Other viral conjunctivitis</td>\n",
       "      <td>Condition</td>\n",
       "      <td>ICD9CM</td>\n",
       "      <td>4-dig billing code</td>\n",
       "      <td>NaN</td>\n",
       "      <td>077.8</td>\n",
       "      <td>1970-01-01</td>\n",
       "      <td>2099-12-31</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        concept_id                concept_name  domain_id vocabulary_id  \\\n",
       "117785    44825158  Other viral conjunctivitis  Condition        ICD9CM   \n",
       "\n",
       "          concept_class_id standard_concept concept_code valid_start_date  \\\n",
       "117785  4-dig billing code              NaN        077.8       1970-01-01   \n",
       "\n",
       "       valid_end_date invalid_reason  \n",
       "117785     2099-12-31            NaN  "
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "icd9_concepts.query('concept_code in \"077.8\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "9a9c6713-01cf-4fe7-8537-1cfe4f350c20",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROW_ID</th>\n",
       "      <th>SUBJECT_ID</th>\n",
       "      <th>HADM_ID</th>\n",
       "      <th>SEQ_NUM</th>\n",
       "      <th>ICD9_CODE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>567020</th>\n",
       "      <td>587430</td>\n",
       "      <td>85828</td>\n",
       "      <td>120813</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>567011</th>\n",
       "      <td>587421</td>\n",
       "      <td>85828</td>\n",
       "      <td>120813</td>\n",
       "      <td>1.0</td>\n",
       "      <td>07999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>567017</th>\n",
       "      <td>587427</td>\n",
       "      <td>85828</td>\n",
       "      <td>120813</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2449</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>567024</th>\n",
       "      <td>587434</td>\n",
       "      <td>85828</td>\n",
       "      <td>120813</td>\n",
       "      <td>14.0</td>\n",
       "      <td>2749</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>567021</th>\n",
       "      <td>587431</td>\n",
       "      <td>85828</td>\n",
       "      <td>120813</td>\n",
       "      <td>11.0</td>\n",
       "      <td>2859</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>567014</th>\n",
       "      <td>587424</td>\n",
       "      <td>85828</td>\n",
       "      <td>120813</td>\n",
       "      <td>4.0</td>\n",
       "      <td>28731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>567022</th>\n",
       "      <td>587432</td>\n",
       "      <td>85828</td>\n",
       "      <td>120813</td>\n",
       "      <td>12.0</td>\n",
       "      <td>2875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>567018</th>\n",
       "      <td>587428</td>\n",
       "      <td>85828</td>\n",
       "      <td>120813</td>\n",
       "      <td>8.0</td>\n",
       "      <td>4589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>567013</th>\n",
       "      <td>587423</td>\n",
       "      <td>85828</td>\n",
       "      <td>120813</td>\n",
       "      <td>3.0</td>\n",
       "      <td>486</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>567015</th>\n",
       "      <td>587425</td>\n",
       "      <td>85828</td>\n",
       "      <td>120813</td>\n",
       "      <td>5.0</td>\n",
       "      <td>515</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>567012</th>\n",
       "      <td>587422</td>\n",
       "      <td>85828</td>\n",
       "      <td>120813</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5849</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>567016</th>\n",
       "      <td>587426</td>\n",
       "      <td>85828</td>\n",
       "      <td>120813</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>567019</th>\n",
       "      <td>587429</td>\n",
       "      <td>85828</td>\n",
       "      <td>120813</td>\n",
       "      <td>9.0</td>\n",
       "      <td>7948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>567023</th>\n",
       "      <td>587433</td>\n",
       "      <td>85828</td>\n",
       "      <td>120813</td>\n",
       "      <td>13.0</td>\n",
       "      <td>V08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164331</th>\n",
       "      <td>162207</td>\n",
       "      <td>14497</td>\n",
       "      <td>135312</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164324</th>\n",
       "      <td>162200</td>\n",
       "      <td>14497</td>\n",
       "      <td>135312</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164332</th>\n",
       "      <td>162208</td>\n",
       "      <td>14497</td>\n",
       "      <td>135312</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164330</th>\n",
       "      <td>162206</td>\n",
       "      <td>14497</td>\n",
       "      <td>135312</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2866</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164327</th>\n",
       "      <td>162203</td>\n",
       "      <td>14497</td>\n",
       "      <td>135312</td>\n",
       "      <td>4.0</td>\n",
       "      <td>51882</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164325</th>\n",
       "      <td>162201</td>\n",
       "      <td>14497</td>\n",
       "      <td>135312</td>\n",
       "      <td>2.0</td>\n",
       "      <td>570</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164326</th>\n",
       "      <td>162202</td>\n",
       "      <td>14497</td>\n",
       "      <td>135312</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5845</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164329</th>\n",
       "      <td>162205</td>\n",
       "      <td>14497</td>\n",
       "      <td>135312</td>\n",
       "      <td>6.0</td>\n",
       "      <td>78552</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164328</th>\n",
       "      <td>162204</td>\n",
       "      <td>14497</td>\n",
       "      <td>135312</td>\n",
       "      <td>5.0</td>\n",
       "      <td>99592</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ROW_ID  SUBJECT_ID  HADM_ID  SEQ_NUM ICD9_CODE\n",
       "567020  587430       85828   120813     10.0      0778\n",
       "567011  587421       85828   120813      1.0     07999\n",
       "567017  587427       85828   120813      7.0      2449\n",
       "567024  587434       85828   120813     14.0      2749\n",
       "567021  587431       85828   120813     11.0      2859\n",
       "567014  587424       85828   120813      4.0     28731\n",
       "567022  587432       85828   120813     12.0      2875\n",
       "567018  587428       85828   120813      8.0      4589\n",
       "567013  587423       85828   120813      3.0       486\n",
       "567015  587425       85828   120813      5.0       515\n",
       "567012  587422       85828   120813      2.0      5849\n",
       "567016  587426       85828   120813      6.0      7101\n",
       "567019  587429       85828   120813      9.0      7948\n",
       "567023  587433       85828   120813     13.0       V08\n",
       "164331  162207       14497   135312      8.0      0521\n",
       "164324  162200       14497   135312      1.0      0527\n",
       "164332  162208       14497   135312      9.0      0778\n",
       "164330  162206       14497   135312      7.0      2866\n",
       "164327  162203       14497   135312      4.0     51882\n",
       "164325  162201       14497   135312      2.0       570\n",
       "164326  162202       14497   135312      3.0      5845\n",
       "164329  162205       14497   135312      6.0     78552\n",
       "164328  162204       14497   135312      5.0     99592"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mimic_diagnostics_df.query('HADM_ID in [120813,135312]').sort_values(by=['HADM_ID','ICD9_CODE']) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "37d110f8-a00f-41e7-958c-f439284d3ad2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>condition_occurrence_id</th>\n",
       "      <th>person_id</th>\n",
       "      <th>condition_concept_id</th>\n",
       "      <th>condition_start_date</th>\n",
       "      <th>condition_start_datetime</th>\n",
       "      <th>condition_end_date</th>\n",
       "      <th>condition_end_datetime</th>\n",
       "      <th>condition_type_concept_id</th>\n",
       "      <th>stop_reason</th>\n",
       "      <th>provider_id</th>\n",
       "      <th>visit_occurrence_id</th>\n",
       "      <th>visit_detail_id</th>\n",
       "      <th>condition_source_value</th>\n",
       "      <th>condition_source_concept_id</th>\n",
       "      <th>condition_status_source_value</th>\n",
       "      <th>condition_status_concept_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>423546</th>\n",
       "      <td>336036941</td>\n",
       "      <td>392796367</td>\n",
       "      <td>380038</td>\n",
       "      <td>2180-04-15</td>\n",
       "      <td>2180-04-15 16:18:00</td>\n",
       "      <td>2180-04-20</td>\n",
       "      <td>2180-04-20 18:00:00</td>\n",
       "      <td>38000192</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>14586</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0778</td>\n",
       "      <td>44825158.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>469842</th>\n",
       "      <td>336439629</td>\n",
       "      <td>392817759</td>\n",
       "      <td>380038</td>\n",
       "      <td>2138-03-27</td>\n",
       "      <td>2138-03-27 17:47:00</td>\n",
       "      <td>2138-04-01</td>\n",
       "      <td>2138-04-01 14:40:00</td>\n",
       "      <td>38000193</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>55467</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0778</td>\n",
       "      <td>44825158.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        condition_occurrence_id  person_id  condition_concept_id  \\\n",
       "423546                336036941  392796367                380038   \n",
       "469842                336439629  392817759                380038   \n",
       "\n",
       "       condition_start_date condition_start_datetime condition_end_date  \\\n",
       "423546           2180-04-15      2180-04-15 16:18:00         2180-04-20   \n",
       "469842           2138-03-27      2138-03-27 17:47:00         2138-04-01   \n",
       "\n",
       "       condition_end_datetime  condition_type_concept_id  stop_reason  \\\n",
       "423546    2180-04-20 18:00:00                   38000192          NaN   \n",
       "469842    2138-04-01 14:40:00                   38000193          NaN   \n",
       "\n",
       "        provider_id  visit_occurrence_id  visit_detail_id  \\\n",
       "423546          NaN                14586              NaN   \n",
       "469842          NaN                55467              NaN   \n",
       "\n",
       "       condition_source_value  condition_source_concept_id  \\\n",
       "423546                   0778                   44825158.0   \n",
       "469842                   0778                   44825158.0   \n",
       "\n",
       "        condition_status_source_value  condition_status_concept_id  \n",
       "423546                            NaN                          NaN  \n",
       "469842                            NaN                          NaN  "
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "icd_filtered_diagnostics_df.query('condition_source_value==\"0778\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "7d73a80f-d817-421c-83d3-2f284aabaabd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>visit_occurrence_id</th>\n",
       "      <th>condition_source_value</th>\n",
       "      <th>condition_source_concept_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>377399</th>\n",
       "      <td>14586</td>\n",
       "      <td>0521</td>\n",
       "      <td>44833234.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44157</th>\n",
       "      <td>14586</td>\n",
       "      <td>0527</td>\n",
       "      <td>44821676.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>423546</th>\n",
       "      <td>14586</td>\n",
       "      <td>0778</td>\n",
       "      <td>44825158.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>345205</th>\n",
       "      <td>14586</td>\n",
       "      <td>2866</td>\n",
       "      <td>44820701.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>227739</th>\n",
       "      <td>14586</td>\n",
       "      <td>51882</td>\n",
       "      <td>44820899.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98999</th>\n",
       "      <td>14586</td>\n",
       "      <td>570</td>\n",
       "      <td>44825528.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164576</th>\n",
       "      <td>14586</td>\n",
       "      <td>5845</td>\n",
       "      <td>44833659.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>290901</th>\n",
       "      <td>14586</td>\n",
       "      <td>78552</td>\n",
       "      <td>44825805.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>230916</th>\n",
       "      <td>14586</td>\n",
       "      <td>99592</td>\n",
       "      <td>44827180.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>469842</th>\n",
       "      <td>55467</td>\n",
       "      <td>0778</td>\n",
       "      <td>44825158.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31270</th>\n",
       "      <td>55467</td>\n",
       "      <td>07999</td>\n",
       "      <td>44822836.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>360562</th>\n",
       "      <td>55467</td>\n",
       "      <td>2449</td>\n",
       "      <td>44828786.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>554903</th>\n",
       "      <td>55467</td>\n",
       "      <td>2749</td>\n",
       "      <td>44829897.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>490340</th>\n",
       "      <td>55467</td>\n",
       "      <td>2859</td>\n",
       "      <td>44825282.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>212821</th>\n",
       "      <td>55467</td>\n",
       "      <td>28731</td>\n",
       "      <td>44834576.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>513342</th>\n",
       "      <td>55467</td>\n",
       "      <td>2875</td>\n",
       "      <td>44825284.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>393426</th>\n",
       "      <td>55467</td>\n",
       "      <td>4589</td>\n",
       "      <td>44833594.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157084</th>\n",
       "      <td>55467</td>\n",
       "      <td>486</td>\n",
       "      <td>44829009.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>264448</th>\n",
       "      <td>55467</td>\n",
       "      <td>515</td>\n",
       "      <td>44832428.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100591</th>\n",
       "      <td>55467</td>\n",
       "      <td>5849</td>\n",
       "      <td>44826731.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>313825</th>\n",
       "      <td>55467</td>\n",
       "      <td>7101</td>\n",
       "      <td>44832613.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>431291</th>\n",
       "      <td>55467</td>\n",
       "      <td>7948</td>\n",
       "      <td>44831608.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>537142</th>\n",
       "      <td>55467</td>\n",
       "      <td>V08</td>\n",
       "      <td>44824945.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        visit_occurrence_id condition_source_value  \\\n",
       "377399                14586                   0521   \n",
       "44157                 14586                   0527   \n",
       "423546                14586                   0778   \n",
       "345205                14586                   2866   \n",
       "227739                14586                  51882   \n",
       "98999                 14586                    570   \n",
       "164576                14586                   5845   \n",
       "290901                14586                  78552   \n",
       "230916                14586                  99592   \n",
       "469842                55467                   0778   \n",
       "31270                 55467                  07999   \n",
       "360562                55467                   2449   \n",
       "554903                55467                   2749   \n",
       "490340                55467                   2859   \n",
       "212821                55467                  28731   \n",
       "513342                55467                   2875   \n",
       "393426                55467                   4589   \n",
       "157084                55467                    486   \n",
       "264448                55467                    515   \n",
       "100591                55467                   5849   \n",
       "313825                55467                   7101   \n",
       "431291                55467                   7948   \n",
       "537142                55467                    V08   \n",
       "\n",
       "        condition_source_concept_id  \n",
       "377399                   44833234.0  \n",
       "44157                    44821676.0  \n",
       "423546                   44825158.0  \n",
       "345205                   44820701.0  \n",
       "227739                   44820899.0  \n",
       "98999                    44825528.0  \n",
       "164576                   44833659.0  \n",
       "290901                   44825805.0  \n",
       "230916                   44827180.0  \n",
       "469842                   44825158.0  \n",
       "31270                    44822836.0  \n",
       "360562                   44828786.0  \n",
       "554903                   44829897.0  \n",
       "490340                   44825282.0  \n",
       "212821                   44834576.0  \n",
       "513342                   44825284.0  \n",
       "393426                   44833594.0  \n",
       "157084                   44829009.0  \n",
       "264448                   44832428.0  \n",
       "100591                   44826731.0  \n",
       "313825                   44832613.0  \n",
       "431291                   44831608.0  \n",
       "537142                   44824945.0  "
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "icd_filtered_diagnostics_df.query('visit_occurrence_id in [14586,55467]')[['visit_occurrence_id','condition_source_value','condition_source_concept_id']].sort_values(by=['visit_occurrence_id','condition_source_value'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f1cd2431-9768-483a-9a76-296c19b1defa",
   "metadata": {},
   "source": [
    "I don't quite understand (probably a mistake of mine) why the two 0778 codes in the whole MIMIC stoud out in the previous cells, but it seems they are correctly mapped to 2 patients with the exact same set of diagnostics and then the 47 missing diags left are NA and could not be mapped."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "47b22fdc-2c0e-4020-bed1-3ce27c4ff794",
   "metadata": {},
   "source": [
    "## Diagnostics are unique in a visit_occurrence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "343d852e-16f5-4610-8a2b-e8ee356120f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert all (icd_filtered_diagnostics_df.groupby(['visit_occurrence_id','condition_source_concept_id']).count()==1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e1a6e631-371a-4729-b791-544360d6cf79",
   "metadata": {},
   "source": [
    "## Export filtered table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fe9078a9-fa6c-4a12-bde7-b573bd4d356a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Join with the icd9 concept table to get consistent ICD9 labels\n",
    "diag_export_table = pd.merge(icd_filtered_diagnostics_df[['condition_occurrence_id', 'person_id', 'condition_concept_id','visit_occurrence_id','condition_source_value',\n",
    "       'condition_source_concept_id']],\n",
    "        icd9_concepts[['concept_id','domain_id','concept_name','concept_class_id','concept_code']],\n",
    "        how='left',\n",
    "        left_on='condition_source_concept_id',\n",
    "        right_on='concept_id').drop(columns='condition_source_concept_id')\n",
    "\n",
    "assert all(diag_export_table.condition_source_value==diag_export_table.concept_code.str.replace('.',\"\",regex=False))\n",
    "diag_export_table.drop(columns='condition_source_value',inplace=True)\n",
    "diag_export_table.rename(columns={'condition_concept_id':'snomed_concept_id','concept_id':'icd9_concept_id',\n",
    "                                 'concept_code':'icd9_code','concept_name':'icd9_name','concept_class_id':'icd9_concept_class_id'},inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5f456485-7a02-467e-b913-47e373c5ab7c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>condition_occurrence_id</th>\n",
       "      <th>person_id</th>\n",
       "      <th>snomed_concept_id</th>\n",
       "      <th>visit_occurrence_id</th>\n",
       "      <th>icd9_concept_id</th>\n",
       "      <th>domain_id</th>\n",
       "      <th>icd9_name</th>\n",
       "      <th>icd9_concept_class_id</th>\n",
       "      <th>icd9_code</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>336159169</td>\n",
       "      <td>392799187</td>\n",
       "      <td>198464</td>\n",
       "      <td>30549</td>\n",
       "      <td>44825520</td>\n",
       "      <td>Condition</td>\n",
       "      <td>Incisional hernia without mention of obstructi...</td>\n",
       "      <td>5-dig billing code</td>\n",
       "      <td>553.21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>336176080</td>\n",
       "      <td>392807515</td>\n",
       "      <td>195559</td>\n",
       "      <td>33647</td>\n",
       "      <td>44833580</td>\n",
       "      <td>Condition</td>\n",
       "      <td>Abdominal aneurysm, ruptured</td>\n",
       "      <td>4-dig billing code</td>\n",
       "      <td>441.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>335984902</td>\n",
       "      <td>392784957</td>\n",
       "      <td>437474</td>\n",
       "      <td>10854</td>\n",
       "      <td>44826039</td>\n",
       "      <td>Condition</td>\n",
       "      <td>Other postoperative infection</td>\n",
       "      <td>5-dig billing code</td>\n",
       "      <td>998.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>336063211</td>\n",
       "      <td>392795465</td>\n",
       "      <td>40481042</td>\n",
       "      <td>17475</td>\n",
       "      <td>44824251</td>\n",
       "      <td>Condition</td>\n",
       "      <td>Acute diastolic heart failure</td>\n",
       "      <td>5-dig billing code</td>\n",
       "      <td>428.31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>336159162</td>\n",
       "      <td>392799187</td>\n",
       "      <td>73614</td>\n",
       "      <td>30550</td>\n",
       "      <td>44835139</td>\n",
       "      <td>Condition</td>\n",
       "      <td>Closed fracture of seven ribs</td>\n",
       "      <td>5-dig billing code</td>\n",
       "      <td>807.07</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   condition_occurrence_id  person_id  snomed_concept_id  visit_occurrence_id  \\\n",
       "0                336159169  392799187             198464                30549   \n",
       "1                336176080  392807515             195559                33647   \n",
       "2                335984902  392784957             437474                10854   \n",
       "3                336063211  392795465           40481042                17475   \n",
       "4                336159162  392799187              73614                30550   \n",
       "\n",
       "   icd9_concept_id  domain_id  \\\n",
       "0         44825520  Condition   \n",
       "1         44833580  Condition   \n",
       "2         44826039  Condition   \n",
       "3         44824251  Condition   \n",
       "4         44835139  Condition   \n",
       "\n",
       "                                           icd9_name icd9_concept_class_id  \\\n",
       "0  Incisional hernia without mention of obstructi...    5-dig billing code   \n",
       "1                       Abdominal aneurysm, ruptured    4-dig billing code   \n",
       "2                      Other postoperative infection    5-dig billing code   \n",
       "3                      Acute diastolic heart failure    5-dig billing code   \n",
       "4                      Closed fracture of seven ribs    5-dig billing code   \n",
       "\n",
       "  icd9_code  \n",
       "0    553.21  \n",
       "1     441.3  \n",
       "2    998.59  \n",
       "3    428.31  \n",
       "4    807.07  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "diag_export_table.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "88773023-826e-4d7b-93b3-99291766d3dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "diag_export_table.to_csv(datadir/\"pre-processed/diagnoses.csv.gz\",index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "411c4e52-666a-4492-9b29-568242b7f6d1",
   "metadata": {},
   "source": [
    "## Section TODO\n",
    "- [ ] ~~check if numbers match with Sagi et al applying the same filters~~ => shall be done in further analysis script\n",
    "- [ ] ~~check if string diagnostics are mapped to precise SNOMED concepts, and if yes, if they can be mapped back to ICD9 non leaf concepts~~ => not in the original mimic diagnosis table\n",
    "- [X] check repeated diagnostics in a single visit_occurrence\n",
    "- [X] check non \"EHR chief complaint\" conditions that have been excluded by the ICD9 filter \n",
    "- [ ] check sejour match between raw mimic and omop(ensemble of dg via group by + hash)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19dfb77a-d196-4ef0-b5c9-840b430199b2",
   "metadata": {},
   "source": [
    "# Drug exposure table preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a3aff5c-c0df-4bb6-addc-f9ecd54d75db",
   "metadata": {},
   "source": [
    "## EDF initiale"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "82f201cb-7215-44d7-ac2f-491b9ce13de0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assign dtypes to reduce memory requirement and get reproducible results upon reading by chunk\n",
    "drug_exp_dtypes = {'drug_exposure_id': np.uint64, 'person_id': np.uint64,'drug_concept_id':'Int64',\n",
    "                   'drug_type_concept_id':'Int64','refills':'Int64',\n",
    "                  'quantity':'Float64','days_supply':'Int64', 'route_concept_id':'Int64',\n",
    "                  'visit_occurrence_id': np.uint64, 'visit_detail_id':'Int64', 'drug_source_concept_id':'Int64',\n",
    "                   'quantity_source_value':str,'drug_source_value':str}\n",
    "date_cols = ['drug_exposure_start_date','drug_exposure_end_date','verbatim_end_date']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8b36e6f7-a5c0-47ff-bc6e-ff4c6c57eabd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_drug_type(df):\n",
    "    return pd.DataFrame.query(df,'drug_type_concept_id==38000177')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "13fb6802-afe8-44ed-9af0-0effd1141d0c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "38000180    20778301\n",
      "38000177     4165275\n",
      "Name: drug_type_concept_id, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "drug_exp_types = pd.read_csv(datadir/\"drug_exposure.csv.gz\",usecols=['drug_type_concept_id'])\n",
    "print(drug_exp_types.drug_type_concept_id.value_counts())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "24f956d0-c949-416e-96d8-e2d2b63bd40b",
   "metadata": {},
   "source": [
    "From the concepts table:\n",
    "- 38000177,Prescription written,Type Concept,Drug Type,Drug Type,,OMOP4822241,1970-01-01,2099-12-31,\n",
    "- 38000180,Inpatient administration,Type Concept,Drug Type,Drug Type,,OMOP4822244,1970-01-01,2099-12-31,\n",
    "\n",
    "Filtering on \"Prescription written\" would thus lead to a table of almost equal size compared with the initial Mimic prescription table (4,156,450 rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "5c931e4f-c892-418f-a66e-f90681398717",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<timed exec>:2: DtypeWarning: Columns (9,19) have mixed types.Specify dtype option on import or set low_memory=False.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 30.8 s, sys: 256 ms, total: 31.1 s\n",
      "Wall time: 31.1 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "## All \"Prescription written\" rows are on top of the file thus there's no point going past 4200000 rows read\n",
    "drug_exposure_df = pandas_utils.read_filter_csv_by_chunk(datadir/\"drug_exposure.csv.gz\",\n",
    "                                                         filter_func=filter_drug_type,\n",
    "                                                         chunksize=100000,nrows=4200000,\n",
    "                                                        dtype=drug_exp_dtypes,parse_dates=date_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "82a7e2d7-dbcc-4fa1-a75b-9de8cfb1c72a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 4165275 entries, 0 to 4165274\n",
      "Data columns (total 24 columns):\n",
      " #   Column                        Dtype  \n",
      "---  ------                        -----  \n",
      " 0   drug_exposure_id              int64  \n",
      " 1   person_id                     int64  \n",
      " 2   drug_concept_id               int64  \n",
      " 3   drug_exposure_start_date      object \n",
      " 4   drug_exposure_start_datetime  object \n",
      " 5   drug_exposure_end_date        object \n",
      " 6   drug_exposure_end_datetime    object \n",
      " 7   verbatim_end_date             float64\n",
      " 8   drug_type_concept_id          int64  \n",
      " 9   stop_reason                   object \n",
      " 10  refills                       float64\n",
      " 11  quantity                      float64\n",
      " 12  days_supply                   float64\n",
      " 13  sig                           float64\n",
      " 14  route_concept_id              float64\n",
      " 15  lot_number                    float64\n",
      " 16  provider_id                   float64\n",
      " 17  visit_occurrence_id           int64  \n",
      " 18  visit_detail_id               float64\n",
      " 19  drug_source_value             object \n",
      " 20  drug_source_concept_id        float64\n",
      " 21  route_source_value            object \n",
      " 22  dose_unit_source_value        object \n",
      " 23  quantity_source_value         object \n",
      "dtypes: float64(10), int64(5), object(9)\n",
      "memory usage: 762.7+ MB\n"
     ]
    }
   ],
   "source": [
    "drug_exposure_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4984a57e-774a-414b-a530-d6a5a14349ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>drug_exposure_id</th>\n",
       "      <th>person_id</th>\n",
       "      <th>drug_concept_id</th>\n",
       "      <th>drug_exposure_start_date</th>\n",
       "      <th>drug_exposure_start_datetime</th>\n",
       "      <th>drug_exposure_end_date</th>\n",
       "      <th>drug_exposure_end_datetime</th>\n",
       "      <th>verbatim_end_date</th>\n",
       "      <th>drug_type_concept_id</th>\n",
       "      <th>stop_reason</th>\n",
       "      <th>...</th>\n",
       "      <th>route_concept_id</th>\n",
       "      <th>lot_number</th>\n",
       "      <th>provider_id</th>\n",
       "      <th>visit_occurrence_id</th>\n",
       "      <th>visit_detail_id</th>\n",
       "      <th>drug_source_value</th>\n",
       "      <th>drug_source_concept_id</th>\n",
       "      <th>route_source_value</th>\n",
       "      <th>dose_unit_source_value</th>\n",
       "      <th>quantity_source_value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>396803253</td>\n",
       "      <td>392815432</td>\n",
       "      <td>0</td>\n",
       "      <td>2106-09-07</td>\n",
       "      <td>2106-09-07 00:00:00</td>\n",
       "      <td>2106-09-08</td>\n",
       "      <td>2106-09-08 00:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>38000177</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>4139962.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>56351</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>SC</td>\n",
       "      <td>VIAL</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>396821890</td>\n",
       "      <td>392819347</td>\n",
       "      <td>0</td>\n",
       "      <td>2187-10-26</td>\n",
       "      <td>2187-10-26 00:00:00</td>\n",
       "      <td>2187-10-28</td>\n",
       "      <td>2187-10-28 00:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>38000177</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>4139962.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>58698</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>SC</td>\n",
       "      <td>VIAL</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>396808816</td>\n",
       "      <td>392815414</td>\n",
       "      <td>0</td>\n",
       "      <td>2149-02-13</td>\n",
       "      <td>2149-02-13 00:00:00</td>\n",
       "      <td>2149-02-13</td>\n",
       "      <td>2149-02-13 00:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>38000177</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>4112421.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>58519</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IV</td>\n",
       "      <td>mL</td>\n",
       "      <td>50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>396821892</td>\n",
       "      <td>392819347</td>\n",
       "      <td>0</td>\n",
       "      <td>2187-10-27</td>\n",
       "      <td>2187-10-27 00:00:00</td>\n",
       "      <td>2187-10-29</td>\n",
       "      <td>2187-10-29 00:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>38000177</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>4112421.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>58698</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IV</td>\n",
       "      <td>mL</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>396808830</td>\n",
       "      <td>392815420</td>\n",
       "      <td>40141424</td>\n",
       "      <td>2116-05-15</td>\n",
       "      <td>2116-05-15 00:00:00</td>\n",
       "      <td>2116-05-17</td>\n",
       "      <td>2116-05-17 00:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>38000177</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>4112421.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>56337</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IV</td>\n",
       "      <td>SYR</td>\n",
       "      <td>0.6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   drug_exposure_id  person_id  drug_concept_id drug_exposure_start_date  \\\n",
       "0         396803253  392815432                0               2106-09-07   \n",
       "1         396821890  392819347                0               2187-10-26   \n",
       "2         396808816  392815414                0               2149-02-13   \n",
       "3         396821892  392819347                0               2187-10-27   \n",
       "4         396808830  392815420         40141424               2116-05-15   \n",
       "\n",
       "  drug_exposure_start_datetime drug_exposure_end_date  \\\n",
       "0          2106-09-07 00:00:00             2106-09-08   \n",
       "1          2187-10-26 00:00:00             2187-10-28   \n",
       "2          2149-02-13 00:00:00             2149-02-13   \n",
       "3          2187-10-27 00:00:00             2187-10-29   \n",
       "4          2116-05-15 00:00:00             2116-05-17   \n",
       "\n",
       "  drug_exposure_end_datetime  verbatim_end_date  drug_type_concept_id  \\\n",
       "0        2106-09-08 00:00:00                NaN              38000177   \n",
       "1        2187-10-28 00:00:00                NaN              38000177   \n",
       "2        2149-02-13 00:00:00                NaN              38000177   \n",
       "3        2187-10-29 00:00:00                NaN              38000177   \n",
       "4        2116-05-17 00:00:00                NaN              38000177   \n",
       "\n",
       "  stop_reason  ...  route_concept_id  lot_number  provider_id  \\\n",
       "0         NaN  ...         4139962.0         NaN          NaN   \n",
       "1         NaN  ...         4139962.0         NaN          NaN   \n",
       "2         NaN  ...         4112421.0         NaN          NaN   \n",
       "3         NaN  ...         4112421.0         NaN          NaN   \n",
       "4         NaN  ...         4112421.0         NaN          NaN   \n",
       "\n",
       "   visit_occurrence_id  visit_detail_id  drug_source_value  \\\n",
       "0                56351              NaN                0.0   \n",
       "1                58698              NaN                0.0   \n",
       "2                58519              NaN                0.0   \n",
       "3                58698              NaN                0.0   \n",
       "4                56337              NaN                0.0   \n",
       "\n",
       "   drug_source_concept_id  route_source_value  dose_unit_source_value  \\\n",
       "0                     NaN                  SC                    VIAL   \n",
       "1                     NaN                  SC                    VIAL   \n",
       "2                     NaN                  IV                      mL   \n",
       "3                     NaN                  IV                      mL   \n",
       "4                     NaN                  IV                     SYR   \n",
       "\n",
       "  quantity_source_value  \n",
       "0                     0  \n",
       "1                     0  \n",
       "2                    50  \n",
       "3                   200  \n",
       "4                   0.6  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "drug_exposure_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "de8130fb-aed1-4a7b-b2ff-3aa08b878134",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4112421.0     2381133\n",
       "4128794.0     1309168\n",
       "4139962.0      214038\n",
       "45956874.0      83428\n",
       "4302612.0       45620\n",
       "4115462.0       44202\n",
       "4231622.0       29624\n",
       "4157760.0       20367\n",
       "45956871.0      11402\n",
       "4225555.0        9590\n",
       "4262099.0        7496\n",
       "4292110.0        4755\n",
       "4156707.0         809\n",
       "4217202.0         716\n",
       "4136280.0         427\n",
       "4023156.0         345\n",
       "4243022.0         338\n",
       "45956879.0        207\n",
       "4186832.0         193\n",
       "4128792.0         159\n",
       "4224886.0          47\n",
       "4181897.0          41\n",
       "4006860.0          12\n",
       "4222259.0           2\n",
       "Name: route_concept_id, dtype: int64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "drug_exposure_df.route_concept_id.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6f313c58-b3c6-49a5-8d9c-93da512381a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "drug_prof = drug_exposure_df.sample(n=200000).profile_report()\n",
    "drug_prof.to_notebook_iframe()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b6c6af4f-b843-46df-ab96-dea061dc9e07",
   "metadata": {},
   "source": [
    "Observations (pandas profiling on 200000 random rows of written prescriptions):\n",
    "- verbatim_end_date has 200000 (100.0%) missing values \n",
    "- stop_reason has 200000 (100.0%) missing values \n",
    "- refills has 200000 (100.0%) missing values \n",
    "- days_supply has 200000 (100.0%) missing values \n",
    "- sig has 200000 (100.0%) missing values \n",
    "- lot_number has 200000 (100.0%) missing values \n",
    "- provider_id has 200000 (100.0%) missing values \n",
    "- visit_detail_id has 200000 (100.0%) missing values \n",
    "- drug_source_concept_id has 200000 (100.0%) missing values \n",
    "- drug_concept_id has 27420 (13.7%) zeros \n",
    "- quantity has 1.9% zeros and 5.7% missing\n",
    "- route_concept_id is almost complete, 23 unique values vs 61 for the route_source_value\n",
    "- 2192 distinct drug_concept_id vs 1602 in Sagi et al"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "983e1524-7a59-4685-9c5d-55b7dba09270",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/quentin/anaconda3/envs/pmsi_db_analysis/lib/python3.10/site-packages/IPython/core/interactiveshell.py:3457: DtypeWarning: Columns (9) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  exec(code_obj, self.user_global_ns, self.user_ns)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# stop_reason_test = \n",
    "all(pd.read_csv(datadir/\"drug_exposure.csv.gz\",usecols=['stop_reason']).stop_reason.isna())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5ebf10e7-5d72-4a79-b645-510eb241c230",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 100 entries, 0 to 99\n",
      "Data columns (total 24 columns):\n",
      " #   Column                        Non-Null Count  Dtype  \n",
      "---  ------                        --------------  -----  \n",
      " 0   drug_exposure_id              100 non-null    int64  \n",
      " 1   person_id                     100 non-null    int64  \n",
      " 2   drug_concept_id               100 non-null    int64  \n",
      " 3   drug_exposure_start_date      100 non-null    object \n",
      " 4   drug_exposure_start_datetime  100 non-null    object \n",
      " 5   drug_exposure_end_date        100 non-null    object \n",
      " 6   drug_exposure_end_datetime    100 non-null    object \n",
      " 7   verbatim_end_date             0 non-null      float64\n",
      " 8   drug_type_concept_id          100 non-null    int64  \n",
      " 9   stop_reason                   0 non-null      float64\n",
      " 10  refills                       0 non-null      float64\n",
      " 11  quantity                      98 non-null     float64\n",
      " 12  days_supply                   0 non-null      float64\n",
      " 13  sig                           0 non-null      float64\n",
      " 14  route_concept_id              100 non-null    int64  \n",
      " 15  lot_number                    0 non-null      float64\n",
      " 16  provider_id                   0 non-null      float64\n",
      " 17  visit_occurrence_id           100 non-null    int64  \n",
      " 18  visit_detail_id               0 non-null      float64\n",
      " 19  drug_source_value             100 non-null    int64  \n",
      " 20  drug_source_concept_id        0 non-null      float64\n",
      " 21  route_source_value            100 non-null    object \n",
      " 22  dose_unit_source_value        100 non-null    object \n",
      " 23  quantity_source_value         100 non-null    object \n",
      "dtypes: float64(10), int64(7), object(7)\n",
      "memory usage: 18.9+ KB\n"
     ]
    }
   ],
   "source": [
    "drug_exposure_df = pd.read_csv(datadir/\"drug_exposure.csv.gz\",nrows=100)\n",
    "drug_exposure_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "363e3cb1-9c8a-4bef-b706-df5cb7d5d707",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 100 entries, 0 to 99\n",
      "Data columns (total 24 columns):\n",
      " #   Column                        Non-Null Count  Dtype         \n",
      "---  ------                        --------------  -----         \n",
      " 0   drug_exposure_id              100 non-null    Int64         \n",
      " 1   person_id                     100 non-null    Int64         \n",
      " 2   drug_concept_id               100 non-null    Int64         \n",
      " 3   drug_exposure_start_date      100 non-null    datetime64[ns]\n",
      " 4   drug_exposure_start_datetime  100 non-null    object        \n",
      " 5   drug_exposure_end_date        100 non-null    datetime64[ns]\n",
      " 6   drug_exposure_end_datetime    100 non-null    object        \n",
      " 7   verbatim_end_date             0 non-null      datetime64[ns]\n",
      " 8   drug_type_concept_id          100 non-null    Int64         \n",
      " 9   stop_reason                   0 non-null      float64       \n",
      " 10  refills                       0 non-null      Int64         \n",
      " 11  quantity                      98 non-null     Float64       \n",
      " 12  days_supply                   0 non-null      Int64         \n",
      " 13  sig                           0 non-null      float64       \n",
      " 14  route_concept_id              100 non-null    Int64         \n",
      " 15  lot_number                    0 non-null      float64       \n",
      " 16  provider_id                   0 non-null      float64       \n",
      " 17  visit_occurrence_id           100 non-null    uint64        \n",
      " 18  visit_detail_id               0 non-null      Int64         \n",
      " 19  drug_source_value             100 non-null    int64         \n",
      " 20  drug_source_concept_id        0 non-null      Int64         \n",
      " 21  route_source_value            100 non-null    object        \n",
      " 22  dose_unit_source_value        100 non-null    object        \n",
      " 23  quantity_source_value         100 non-null    object        \n",
      "dtypes: Float64(1), Int64(9), datetime64[ns](3), float64(4), int64(1), object(5), uint64(1)\n",
      "memory usage: 19.9+ KB\n"
     ]
    }
   ],
   "source": [
    "drug_exposure_df = pd.read_csv(datadir/\"drug_exposure.csv.gz\",nrows=100,dtype=drug_exp_dtypes,parse_dates=date_cols)\n",
    "drug_exposure_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "d229ece1-ce24-4e2c-ae9f-e6ebf8a4e9fa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 23.3 s, sys: 2.81 s, total: 26.1 s\n",
      "Wall time: 30.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "drug_exp_skip_cols = ['lot_number','provider_id','verbatim_end_date','stop_reason','refills','days_supply','sig',\n",
    "                     'visit_detail_id','drug_exposure_start_datetime','drug_exposure_end_datetime']\n",
    "date_cols = ['drug_exposure_start_date','drug_exposure_end_date']\n",
    "\n",
    "## All \"Prescription written\" rows are on top of the file, there's no point going past 4200000 rows read\n",
    "drug_exposure_df = pandas_utils.read_filter_csv_by_chunk(datadir/\"drug_exposure.csv.gz\",\n",
    "                                                         filter_func=filter_drug_type,\n",
    "                                                         chunksize=100000,nrows=4200000,\n",
    "                                                        dtype=drug_exp_dtypes,parse_dates=date_cols,\n",
    "                                                        usecols= lambda x: x not in drug_exp_skip_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "1024fdda-8aa6-4cda-b340-294bac4c4a48",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 4165275 entries, 0 to 4165274\n",
      "Data columns (total 15 columns):\n",
      " #   Column                      Dtype         \n",
      "---  ------                      -----         \n",
      " 0   drug_exposure_id            uint64        \n",
      " 1   person_id                   uint64        \n",
      " 2   drug_concept_id             Int64         \n",
      " 3   drug_exposure_start_date    datetime64[ns]\n",
      " 4   drug_exposure_end_date      datetime64[ns]\n",
      " 5   drug_exposure_end_datetime  datetime64[ns]\n",
      " 6   drug_type_concept_id        Int64         \n",
      " 7   quantity                    Float64       \n",
      " 8   route_concept_id            Int64         \n",
      " 9   visit_occurrence_id         uint64        \n",
      " 10  drug_source_value           object        \n",
      " 11  drug_source_concept_id      Int64         \n",
      " 12  route_source_value          object        \n",
      " 13  dose_unit_source_value      object        \n",
      " 14  quantity_source_value       object        \n",
      "dtypes: Float64(1), Int64(4), datetime64[ns](3), object(4), uint64(3)\n",
      "memory usage: 496.5+ MB\n"
     ]
    }
   ],
   "source": [
    "drug_exposure_df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7e0a6104-66f5-4740-918c-07992e9e2c82",
   "metadata": {},
   "source": [
    "## RxNorm concepts relationships\n",
    "A lot of details on the relationships on the ohdsi website: https://www.ohdsi.org/web/wiki/doku.php?id=documentation:vocabulary:atc\n",
    "\n",
    "And the reference on the NIH website: https://www.nlm.nih.gov/research/umls/rxnorm/overview.html\n",
    "\n",
    "The relationship tables gives relationship between different ontologies, and inside an ontology.\n",
    "The concept_relationship table can be used to standardize RxNorm codes.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "c6e92cd8-5d79-4de8-8325-90b3fb1c9805",
   "metadata": {},
   "outputs": [],
   "source": [
    "rxnorm_concepts = concept_df.query('vocabulary_id.str.contains(\"RxNorm\",na=False)') # get both RxNorm and RxNorm Extension"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "5c25eeb0-2590-44d7-ab79-7c6a63f51c5f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>concept_id</th>\n",
       "      <th>concept_name</th>\n",
       "      <th>domain_id</th>\n",
       "      <th>vocabulary_id</th>\n",
       "      <th>concept_class_id</th>\n",
       "      <th>standard_concept</th>\n",
       "      <th>concept_code</th>\n",
       "      <th>valid_start_date</th>\n",
       "      <th>valid_end_date</th>\n",
       "      <th>invalid_reason</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1684289</th>\n",
       "      <td>40224176</td>\n",
       "      <td>amlodipine 5 MG / hydrochlorothiazide 12.5 MG ...</td>\n",
       "      <td>Drug</td>\n",
       "      <td>RxNorm</td>\n",
       "      <td>Branded Drug</td>\n",
       "      <td>S</td>\n",
       "      <td>1000000</td>\n",
       "      <td>2010-09-05</td>\n",
       "      <td>2099-12-31</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1684290</th>\n",
       "      <td>40224178</td>\n",
       "      <td>amlodipine 5 MG / hydrochlorothiazide 25 MG / ...</td>\n",
       "      <td>Drug</td>\n",
       "      <td>RxNorm</td>\n",
       "      <td>Clinical Drug</td>\n",
       "      <td>S</td>\n",
       "      <td>1000001</td>\n",
       "      <td>2010-09-05</td>\n",
       "      <td>2099-12-31</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1684291</th>\n",
       "      <td>40224677</td>\n",
       "      <td>Tribenzor 40/5/25</td>\n",
       "      <td>Drug</td>\n",
       "      <td>RxNorm</td>\n",
       "      <td>Brand Name</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1000002</td>\n",
       "      <td>2010-09-05</td>\n",
       "      <td>2013-03-31</td>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1684292</th>\n",
       "      <td>40224180</td>\n",
       "      <td>amlodipine 5 MG / hydrochlorothiazide 25 MG / ...</td>\n",
       "      <td>Drug</td>\n",
       "      <td>RxNorm</td>\n",
       "      <td>Branded Drug Comp</td>\n",
       "      <td>S</td>\n",
       "      <td>1000003</td>\n",
       "      <td>2010-09-05</td>\n",
       "      <td>2099-12-31</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1684293</th>\n",
       "      <td>40224158</td>\n",
       "      <td>Amlodipine / Hydrochlorothiazide / Olmesartan ...</td>\n",
       "      <td>Drug</td>\n",
       "      <td>RxNorm</td>\n",
       "      <td>Branded Drug Form</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1000004</td>\n",
       "      <td>2010-09-05</td>\n",
       "      <td>2013-03-31</td>\n",
       "      <td>U</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         concept_id                                       concept_name  \\\n",
       "1684289    40224176  amlodipine 5 MG / hydrochlorothiazide 12.5 MG ...   \n",
       "1684290    40224178  amlodipine 5 MG / hydrochlorothiazide 25 MG / ...   \n",
       "1684291    40224677                                  Tribenzor 40/5/25   \n",
       "1684292    40224180  amlodipine 5 MG / hydrochlorothiazide 25 MG / ...   \n",
       "1684293    40224158  Amlodipine / Hydrochlorothiazide / Olmesartan ...   \n",
       "\n",
       "        domain_id vocabulary_id   concept_class_id standard_concept  \\\n",
       "1684289      Drug        RxNorm       Branded Drug                S   \n",
       "1684290      Drug        RxNorm      Clinical Drug                S   \n",
       "1684291      Drug        RxNorm         Brand Name              NaN   \n",
       "1684292      Drug        RxNorm  Branded Drug Comp                S   \n",
       "1684293      Drug        RxNorm  Branded Drug Form              NaN   \n",
       "\n",
       "        concept_code valid_start_date valid_end_date invalid_reason  \n",
       "1684289      1000000       2010-09-05     2099-12-31            NaN  \n",
       "1684290      1000001       2010-09-05     2099-12-31            NaN  \n",
       "1684291      1000002       2010-09-05     2013-03-31              D  \n",
       "1684292      1000003       2010-09-05     2099-12-31            NaN  \n",
       "1684293      1000004       2010-09-05     2013-03-31              U  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rxnorm_concepts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1650821f-4ada-4f35-b6b7-10805c46c7c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "rxnorm_relationships = pd.read_csv(datadir/\"relationship.csv.gz\").query('relationship_name.str.contains(\"RxNorm\",na=False)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e76b9ef3-927d-4f61-a127-7ce90602ed4a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>relationship_id</th>\n",
       "      <th>relationship_name</th>\n",
       "      <th>is_hierarchical</th>\n",
       "      <th>defines_ancestry</th>\n",
       "      <th>reverse_relationship_id</th>\n",
       "      <th>relationship_concept_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>ETC - RxNorm</td>\n",
       "      <td>ETC to RxNorm (FDB)</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>RxNorm - ETC</td>\n",
       "      <td>44818809</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>Precise ing of</td>\n",
       "      <td>Precise ingredient of (RxNorm)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Has precise ing</td>\n",
       "      <td>44818813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>Tradename of</td>\n",
       "      <td>Tradename of (RxNorm)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Has tradename</td>\n",
       "      <td>44818814</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>RxNorm dose form of</td>\n",
       "      <td>Dose form of (RxNorm)</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>RxNorm has dose form</td>\n",
       "      <td>44818815</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>Form of</td>\n",
       "      <td>Form of (RxNorm)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Has form</td>\n",
       "      <td>44818816</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>RxNorm ing of</td>\n",
       "      <td>Ingredient of (RxNorm)</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>RxNorm has ing</td>\n",
       "      <td>44818817</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>Consists of</td>\n",
       "      <td>Consists of (RxNorm)</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Constitutes</td>\n",
       "      <td>44818818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>Contained in</td>\n",
       "      <td>Is contained in (RxNorm)</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Contains</td>\n",
       "      <td>44818819</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>Reformulated in</td>\n",
       "      <td>Reformulated in (RxNorm)</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Reformulation of</td>\n",
       "      <td>44818820</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>RxNorm - NDFRT eq</td>\n",
       "      <td>RxNorm to NDF-RT equivalent (RxNorm)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NDFRT - RxNorm eq</td>\n",
       "      <td>44818839</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>ATC - RxNorm</td>\n",
       "      <td>ATC to RxNorm</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>RxNorm - ATC</td>\n",
       "      <td>44818810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>127</th>\n",
       "      <td>RxNorm - ETC</td>\n",
       "      <td>RxNorm to ETC (FDB)</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>ETC - RxNorm</td>\n",
       "      <td>44818907</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128</th>\n",
       "      <td>RxNorm - ATC</td>\n",
       "      <td>RxNorm to ATC</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>ATC - RxNorm</td>\n",
       "      <td>44818908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>133</th>\n",
       "      <td>RxNorm - NDFRT name</td>\n",
       "      <td>RxNorm to NDF-RT equivalent by concept_name (O...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NDFRT - RxNorm name</td>\n",
       "      <td>44818922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134</th>\n",
       "      <td>NDFRT - RxNorm name</td>\n",
       "      <td>NDF-RT to RxNorm equivalent by concept_name (O...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>RxNorm - NDFRT name</td>\n",
       "      <td>44818923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>ETC - RxNorm name</td>\n",
       "      <td>ETC to RxNorm equivalent by concept_name (OMOP)</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>RxNorm - ETC name</td>\n",
       "      <td>44818924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136</th>\n",
       "      <td>RxNorm - ETC name</td>\n",
       "      <td>RxNorm to ETC equivalent by concept_name (OMOP)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>ETC - RxNorm name</td>\n",
       "      <td>44818925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137</th>\n",
       "      <td>RxNorm - ATC name</td>\n",
       "      <td>RxNorm to ATC equivalent by concept_name (OMOP)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>ATC - RxNorm name</td>\n",
       "      <td>44818927</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140</th>\n",
       "      <td>DOI - RxNorm</td>\n",
       "      <td>DOI contains RxNorm (OMOP)</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>RxNorm - DOI</td>\n",
       "      <td>44818930</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141</th>\n",
       "      <td>RxNorm - DOI</td>\n",
       "      <td>RxNorm contained in DOI (OMOP)</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>DOI - RxNorm</td>\n",
       "      <td>44818931</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>152</th>\n",
       "      <td>ATC - RxNorm name</td>\n",
       "      <td>ATC to RxNorm equivalent by concept_name (OMOP)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>RxNorm - ATC name</td>\n",
       "      <td>44818926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>158</th>\n",
       "      <td>VAProd - RxNorm eq</td>\n",
       "      <td>VA Product to RxNorm equivalent (NDF-RT)</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>RxNorm - VAProd eq</td>\n",
       "      <td>44818944</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159</th>\n",
       "      <td>RxNorm - VAProd eq</td>\n",
       "      <td>RxNorm to VA Product equivalent (NDF-RT)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>VAProd - RxNorm eq</td>\n",
       "      <td>44818945</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160</th>\n",
       "      <td>RxNorm replaced by</td>\n",
       "      <td>Concept replaced by (RxNorm)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>RxNorm replaces</td>\n",
       "      <td>44818946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>161</th>\n",
       "      <td>RxNorm replaces</td>\n",
       "      <td>Concept replaces (RxNorm)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>RxNorm replaced by</td>\n",
       "      <td>44818947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164</th>\n",
       "      <td>RxNorm - Source eq</td>\n",
       "      <td>RxNorm to Drug Source equivalent (OMOP)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Source - RxNorm eq</td>\n",
       "      <td>44818952</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>165</th>\n",
       "      <td>Source - RxNorm eq</td>\n",
       "      <td>Drug Source to RxNorm equivalent (OMOP)</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>RxNorm - Source eq</td>\n",
       "      <td>44818953</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>214</th>\n",
       "      <td>Has quantified form</td>\n",
       "      <td>Has quantified form (RxNorm)</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Quantified form of</td>\n",
       "      <td>45754827</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>280</th>\n",
       "      <td>Dose form group of</td>\n",
       "      <td>Dose form group of (RxNorm)</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Has dose form group</td>\n",
       "      <td>5026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>281</th>\n",
       "      <td>Has dose form group</td>\n",
       "      <td>Has dose form group (RxNorm)</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Dose form group of</td>\n",
       "      <td>5027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>304</th>\n",
       "      <td>RxNorm - CVX</td>\n",
       "      <td>RxNorm - CVX (OMOP)</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>CVX - RxNorm</td>\n",
       "      <td>581402</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>305</th>\n",
       "      <td>CVX - RxNorm</td>\n",
       "      <td>CVX - RxNorm (OMOP)</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>RxNorm - CVX</td>\n",
       "      <td>581403</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>355</th>\n",
       "      <td>Quantified form of</td>\n",
       "      <td>Quantified form of (RxNorm)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Has quantified form</td>\n",
       "      <td>45754828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>356</th>\n",
       "      <td>RxNorm is a</td>\n",
       "      <td>Is a (RxNorm)</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>RxNorm inverse is a</td>\n",
       "      <td>45754829</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>357</th>\n",
       "      <td>RxNorm inverse is a</td>\n",
       "      <td>Inverse is a (RxNorm)</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>RxNorm is a</td>\n",
       "      <td>45754830</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>366</th>\n",
       "      <td>SNOMED - RxNorm eq</td>\n",
       "      <td>SNOMED to RxNorm equivalent (RxNorm)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>RxNorm - SNOMED eq</td>\n",
       "      <td>45754843</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>367</th>\n",
       "      <td>RxNorm - SNOMED eq</td>\n",
       "      <td>RxNorm to SNOMED equivalent (RxNorm)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>SNOMED - RxNorm eq</td>\n",
       "      <td>45754844</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>368</th>\n",
       "      <td>SNOMED - NDFRT eq</td>\n",
       "      <td>SNOMED to NDF-RT equivalent (RxNorm)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NDFRT - SNOMED eq</td>\n",
       "      <td>45754845</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>369</th>\n",
       "      <td>NDFRT - SNOMED eq</td>\n",
       "      <td>NDF-RT to SNOMED equivalent (RxNorm)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>SNOMED - NDFRT eq</td>\n",
       "      <td>45754846</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>370</th>\n",
       "      <td>SNOMED - VA Class eq</td>\n",
       "      <td>SNOMED to VA Class equivalent (RxNorm)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>VA Class - SNOMED eq</td>\n",
       "      <td>45754847</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>371</th>\n",
       "      <td>VA Class - SNOMED eq</td>\n",
       "      <td>VA Class to SNOMED equivalent (RxNorm)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>SNOMED - VA Class eq</td>\n",
       "      <td>45754848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>372</th>\n",
       "      <td>SNOMED - ATC eq</td>\n",
       "      <td>SNOMED to ATC equivalent (RxNorm)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>ATC - SNOMED eq</td>\n",
       "      <td>45754849</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>373</th>\n",
       "      <td>ATC - SNOMED eq</td>\n",
       "      <td>ATC to SNOMED equivalent (RxNorm)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>SNOMED - ATC eq</td>\n",
       "      <td>45754850</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>376</th>\n",
       "      <td>SPL - RxNorm</td>\n",
       "      <td>SPL to RxNorm (NLM)</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>RxNorm - SPL</td>\n",
       "      <td>45754866</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>377</th>\n",
       "      <td>RxNorm - SPL</td>\n",
       "      <td>RxNorm to SPL (NLM)</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>SPL - RxNorm</td>\n",
       "      <td>45754867</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383</th>\n",
       "      <td>Has precise ing</td>\n",
       "      <td>Has precise ingredient (RxNorm)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Precise ing of</td>\n",
       "      <td>44818715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>384</th>\n",
       "      <td>Has tradename</td>\n",
       "      <td>Has tradename (RxNorm)</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Tradename of</td>\n",
       "      <td>44818716</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>385</th>\n",
       "      <td>RxNorm has dose form</td>\n",
       "      <td>Has dose form (RxNorm)</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>RxNorm dose form of</td>\n",
       "      <td>44818717</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>386</th>\n",
       "      <td>Has form</td>\n",
       "      <td>Has form (RxNorm)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Form of</td>\n",
       "      <td>44818718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>387</th>\n",
       "      <td>RxNorm has ing</td>\n",
       "      <td>Has ingredient (RxNorm)</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>RxNorm ing of</td>\n",
       "      <td>44818719</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>388</th>\n",
       "      <td>Constitutes</td>\n",
       "      <td>Constitutes (RxNorm)</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Consists of</td>\n",
       "      <td>44818720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>389</th>\n",
       "      <td>Contains</td>\n",
       "      <td>Contains (RxNorm)</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Contained in</td>\n",
       "      <td>44818721</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>390</th>\n",
       "      <td>Reformulation of</td>\n",
       "      <td>Reformulation of (RxNorm)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Reformulated in</td>\n",
       "      <td>44818722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>409</th>\n",
       "      <td>NDFRT - RxNorm eq</td>\n",
       "      <td>NDF-RT to RxNorm equivalent (RxNorm)</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>RxNorm - NDFRT eq</td>\n",
       "      <td>44818741</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485</th>\n",
       "      <td>Has antineopl Rx</td>\n",
       "      <td>Has antineoplastic - RxNorm (HemOnc)</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Rx antineopl of</td>\n",
       "      <td>32614</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>486</th>\n",
       "      <td>Rx antineopl of</td>\n",
       "      <td>RxNorm antineoplastic of (HemOnc)</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Has antineopl Rx</td>\n",
       "      <td>32615</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>491</th>\n",
       "      <td>Has immunosuppr Rx</td>\n",
       "      <td>Has immunosuppressor - RxNorm (HemOnc)</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Rx immunosuppr of</td>\n",
       "      <td>32620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>492</th>\n",
       "      <td>Rx immunosuppr of</td>\n",
       "      <td>RxNorm immunosuppressor of (HemOnc)</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Has immunosuppr Rx</td>\n",
       "      <td>32621</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495</th>\n",
       "      <td>Has local therap Rx</td>\n",
       "      <td>Has local therapy - RxNorm (HemOnc)</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Rx local therapy of</td>\n",
       "      <td>32624</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>496</th>\n",
       "      <td>Rx local therapy of</td>\n",
       "      <td>RxNorm local therapy of (HemOnc)</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Has local therap Rx</td>\n",
       "      <td>32625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>530</th>\n",
       "      <td>Rx support med of</td>\n",
       "      <td>RxNorm Supportive medication of (HemOnc)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Has support med Rx</td>\n",
       "      <td>32631</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>575</th>\n",
       "      <td>ATC - RxNorm pr lat</td>\n",
       "      <td>ATC to RxNorm/Extension primary lateral (OMOP)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>RxNorm - ATC pr lat</td>\n",
       "      <td>32795</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>576</th>\n",
       "      <td>RxNorm - ATC pr lat</td>\n",
       "      <td>RxNorm/Extension to ATC pr lateral (OMOP)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>ATC - RxNorm pr lat</td>\n",
       "      <td>32796</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>577</th>\n",
       "      <td>ATC - RxNorm sec lat</td>\n",
       "      <td>ATC to RxNorm/Extension secondary lateral (OMOP)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>RxNorm - ATC sec lat</td>\n",
       "      <td>32797</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>578</th>\n",
       "      <td>RxNorm - ATC sec lat</td>\n",
       "      <td>RxNorm/Extension to ATC secondary lateral (OMOP)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>ATC - RxNorm sec lat</td>\n",
       "      <td>32798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>579</th>\n",
       "      <td>ATC - RxNorm pr up</td>\n",
       "      <td>ATC to RxNorm/Extension primary upwards (OMOP)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>RxNorm - ATC pr up</td>\n",
       "      <td>32799</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>580</th>\n",
       "      <td>RxNorm - ATC pr up</td>\n",
       "      <td>RxNorm/Extension to ATC primary upwards (OMOP)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>ATC - RxNorm pr up</td>\n",
       "      <td>32800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>581</th>\n",
       "      <td>ATC - RxNorm sec up</td>\n",
       "      <td>ATC to RxNorm/Extension secondary upwards (OMOP)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>RxNorm - ATC sec up</td>\n",
       "      <td>32801</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>582</th>\n",
       "      <td>RxNorm - ATC sec up</td>\n",
       "      <td>RxNorm/Extension to ATC secondary upwards (OMOP)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>ATC - RxNorm sec up</td>\n",
       "      <td>32802</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>623</th>\n",
       "      <td>Has AB-drug cjgt Rx</td>\n",
       "      <td>Has antibody-drug conjugate - RxNorm (HemOnc)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Rx AB-drug cjgt of</td>\n",
       "      <td>33017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>628</th>\n",
       "      <td>Has cytotox chemo Rx</td>\n",
       "      <td>Has cytotoxic chemotherapy - RxNorm (HemOnc)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Cytotox chemo RX of</td>\n",
       "      <td>33023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>629</th>\n",
       "      <td>Cytotox chemo RX of</td>\n",
       "      <td>Cytotoxic chemotherapy of - RxNorm (HemOnc)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Has cytotox chemo Rx</td>\n",
       "      <td>33024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>632</th>\n",
       "      <td>Has endocrine tx Rx</td>\n",
       "      <td>Has endocrine therapy - RxNorm (HemOnc)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Rx endocrine tx of</td>\n",
       "      <td>33027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>635</th>\n",
       "      <td>Has immunotherapy Rx</td>\n",
       "      <td>Has immunotherapy - RxNorm (HemOnc)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Rx immunotherapy of</td>\n",
       "      <td>33031</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>638</th>\n",
       "      <td>Has pept-drg cjg Rx</td>\n",
       "      <td>Has peptide-drug conjugate - RxNorm (HemOnc)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Rx pept-drg cjg of</td>\n",
       "      <td>33035</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>641</th>\n",
       "      <td>Has radiocjgt Rx</td>\n",
       "      <td>Has radioconjugate - RxNorm (HemOnc)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Rx radiocjgt of</td>\n",
       "      <td>33039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>644</th>\n",
       "      <td>Has radiotherapy Rx</td>\n",
       "      <td>Has radiotherapy - RxNorm (HemOnc)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Rx radiotherapy of</td>\n",
       "      <td>33043</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>647</th>\n",
       "      <td>Has targeted tx Rx</td>\n",
       "      <td>Has targeted therapy - RxNorm (HemOnc)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Rx targeted tx of</td>\n",
       "      <td>33047</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>648</th>\n",
       "      <td>Has support med Rx</td>\n",
       "      <td>Has supportive medication - RxNorm (HemOnc)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Rx support med of</td>\n",
       "      <td>32630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>649</th>\n",
       "      <td>Rx AB-drug cjgt of</td>\n",
       "      <td>RxNorm antibody-drug conjugate of (HemOnc)</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Has AB-drug cjgt Rx</td>\n",
       "      <td>33018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>650</th>\n",
       "      <td>Rx endocrine tx of</td>\n",
       "      <td>RxNorm endocrine therapy of (HemOnc)</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Has endocrine tx Rx</td>\n",
       "      <td>33028</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>651</th>\n",
       "      <td>Rx immunotherapy of</td>\n",
       "      <td>RxNorm immunotherapy of (HemOnc)</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Has immunotherapy Rx</td>\n",
       "      <td>33032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>652</th>\n",
       "      <td>Rx pept-drg cjg of</td>\n",
       "      <td>RxNorm peptide-drug conjugate of (HemOnc)</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Has pept-drg cjg Rx</td>\n",
       "      <td>33036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>653</th>\n",
       "      <td>Rx radiocjgt of</td>\n",
       "      <td>RxNorm radioconjugate of (HemOnc)</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Has radiocjgt Rx</td>\n",
       "      <td>33040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>654</th>\n",
       "      <td>Rx radiotherapy of</td>\n",
       "      <td>RxNorm radiotherapy of (HemOnc)</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Has radiotherapy Rx</td>\n",
       "      <td>33044</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>655</th>\n",
       "      <td>Rx targeted tx of</td>\n",
       "      <td>RxNorm targeted therapy of (HemOnc)</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Has targeted tx Rx</td>\n",
       "      <td>33048</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          relationship_id                                  relationship_name  \\\n",
       "34           ETC - RxNorm                                ETC to RxNorm (FDB)   \n",
       "37         Precise ing of                     Precise ingredient of (RxNorm)   \n",
       "38           Tradename of                              Tradename of (RxNorm)   \n",
       "39    RxNorm dose form of                              Dose form of (RxNorm)   \n",
       "40                Form of                                   Form of (RxNorm)   \n",
       "41          RxNorm ing of                             Ingredient of (RxNorm)   \n",
       "42            Consists of                               Consists of (RxNorm)   \n",
       "43           Contained in                           Is contained in (RxNorm)   \n",
       "44        Reformulated in                           Reformulated in (RxNorm)   \n",
       "62      RxNorm - NDFRT eq               RxNorm to NDF-RT equivalent (RxNorm)   \n",
       "75           ATC - RxNorm                                      ATC to RxNorm   \n",
       "127          RxNorm - ETC                                RxNorm to ETC (FDB)   \n",
       "128          RxNorm - ATC                                      RxNorm to ATC   \n",
       "133   RxNorm - NDFRT name  RxNorm to NDF-RT equivalent by concept_name (O...   \n",
       "134   NDFRT - RxNorm name  NDF-RT to RxNorm equivalent by concept_name (O...   \n",
       "135     ETC - RxNorm name    ETC to RxNorm equivalent by concept_name (OMOP)   \n",
       "136     RxNorm - ETC name    RxNorm to ETC equivalent by concept_name (OMOP)   \n",
       "137     RxNorm - ATC name    RxNorm to ATC equivalent by concept_name (OMOP)   \n",
       "140          DOI - RxNorm                         DOI contains RxNorm (OMOP)   \n",
       "141          RxNorm - DOI                     RxNorm contained in DOI (OMOP)   \n",
       "152     ATC - RxNorm name    ATC to RxNorm equivalent by concept_name (OMOP)   \n",
       "158    VAProd - RxNorm eq           VA Product to RxNorm equivalent (NDF-RT)   \n",
       "159    RxNorm - VAProd eq           RxNorm to VA Product equivalent (NDF-RT)   \n",
       "160    RxNorm replaced by                       Concept replaced by (RxNorm)   \n",
       "161       RxNorm replaces                          Concept replaces (RxNorm)   \n",
       "164    RxNorm - Source eq            RxNorm to Drug Source equivalent (OMOP)   \n",
       "165    Source - RxNorm eq            Drug Source to RxNorm equivalent (OMOP)   \n",
       "214   Has quantified form                       Has quantified form (RxNorm)   \n",
       "280    Dose form group of                        Dose form group of (RxNorm)   \n",
       "281   Has dose form group                       Has dose form group (RxNorm)   \n",
       "304          RxNorm - CVX                                RxNorm - CVX (OMOP)   \n",
       "305          CVX - RxNorm                                CVX - RxNorm (OMOP)   \n",
       "355    Quantified form of                        Quantified form of (RxNorm)   \n",
       "356           RxNorm is a                                      Is a (RxNorm)   \n",
       "357   RxNorm inverse is a                              Inverse is a (RxNorm)   \n",
       "366    SNOMED - RxNorm eq               SNOMED to RxNorm equivalent (RxNorm)   \n",
       "367    RxNorm - SNOMED eq               RxNorm to SNOMED equivalent (RxNorm)   \n",
       "368     SNOMED - NDFRT eq               SNOMED to NDF-RT equivalent (RxNorm)   \n",
       "369     NDFRT - SNOMED eq               NDF-RT to SNOMED equivalent (RxNorm)   \n",
       "370  SNOMED - VA Class eq             SNOMED to VA Class equivalent (RxNorm)   \n",
       "371  VA Class - SNOMED eq             VA Class to SNOMED equivalent (RxNorm)   \n",
       "372       SNOMED - ATC eq                  SNOMED to ATC equivalent (RxNorm)   \n",
       "373       ATC - SNOMED eq                  ATC to SNOMED equivalent (RxNorm)   \n",
       "376          SPL - RxNorm                                SPL to RxNorm (NLM)   \n",
       "377          RxNorm - SPL                                RxNorm to SPL (NLM)   \n",
       "383       Has precise ing                    Has precise ingredient (RxNorm)   \n",
       "384         Has tradename                             Has tradename (RxNorm)   \n",
       "385  RxNorm has dose form                             Has dose form (RxNorm)   \n",
       "386              Has form                                  Has form (RxNorm)   \n",
       "387        RxNorm has ing                            Has ingredient (RxNorm)   \n",
       "388           Constitutes                               Constitutes (RxNorm)   \n",
       "389              Contains                                  Contains (RxNorm)   \n",
       "390      Reformulation of                          Reformulation of (RxNorm)   \n",
       "409     NDFRT - RxNorm eq               NDF-RT to RxNorm equivalent (RxNorm)   \n",
       "485      Has antineopl Rx               Has antineoplastic - RxNorm (HemOnc)   \n",
       "486       Rx antineopl of                  RxNorm antineoplastic of (HemOnc)   \n",
       "491    Has immunosuppr Rx             Has immunosuppressor - RxNorm (HemOnc)   \n",
       "492     Rx immunosuppr of                RxNorm immunosuppressor of (HemOnc)   \n",
       "495   Has local therap Rx                Has local therapy - RxNorm (HemOnc)   \n",
       "496   Rx local therapy of                   RxNorm local therapy of (HemOnc)   \n",
       "530     Rx support med of           RxNorm Supportive medication of (HemOnc)   \n",
       "575   ATC - RxNorm pr lat     ATC to RxNorm/Extension primary lateral (OMOP)   \n",
       "576   RxNorm - ATC pr lat          RxNorm/Extension to ATC pr lateral (OMOP)   \n",
       "577  ATC - RxNorm sec lat   ATC to RxNorm/Extension secondary lateral (OMOP)   \n",
       "578  RxNorm - ATC sec lat   RxNorm/Extension to ATC secondary lateral (OMOP)   \n",
       "579    ATC - RxNorm pr up     ATC to RxNorm/Extension primary upwards (OMOP)   \n",
       "580    RxNorm - ATC pr up     RxNorm/Extension to ATC primary upwards (OMOP)   \n",
       "581   ATC - RxNorm sec up   ATC to RxNorm/Extension secondary upwards (OMOP)   \n",
       "582   RxNorm - ATC sec up   RxNorm/Extension to ATC secondary upwards (OMOP)   \n",
       "623   Has AB-drug cjgt Rx      Has antibody-drug conjugate - RxNorm (HemOnc)   \n",
       "628  Has cytotox chemo Rx       Has cytotoxic chemotherapy - RxNorm (HemOnc)   \n",
       "629   Cytotox chemo RX of        Cytotoxic chemotherapy of - RxNorm (HemOnc)   \n",
       "632   Has endocrine tx Rx            Has endocrine therapy - RxNorm (HemOnc)   \n",
       "635  Has immunotherapy Rx                Has immunotherapy - RxNorm (HemOnc)   \n",
       "638   Has pept-drg cjg Rx       Has peptide-drug conjugate - RxNorm (HemOnc)   \n",
       "641      Has radiocjgt Rx               Has radioconjugate - RxNorm (HemOnc)   \n",
       "644   Has radiotherapy Rx                 Has radiotherapy - RxNorm (HemOnc)   \n",
       "647    Has targeted tx Rx             Has targeted therapy - RxNorm (HemOnc)   \n",
       "648    Has support med Rx        Has supportive medication - RxNorm (HemOnc)   \n",
       "649    Rx AB-drug cjgt of         RxNorm antibody-drug conjugate of (HemOnc)   \n",
       "650    Rx endocrine tx of               RxNorm endocrine therapy of (HemOnc)   \n",
       "651   Rx immunotherapy of                   RxNorm immunotherapy of (HemOnc)   \n",
       "652    Rx pept-drg cjg of          RxNorm peptide-drug conjugate of (HemOnc)   \n",
       "653       Rx radiocjgt of                  RxNorm radioconjugate of (HemOnc)   \n",
       "654    Rx radiotherapy of                    RxNorm radiotherapy of (HemOnc)   \n",
       "655     Rx targeted tx of                RxNorm targeted therapy of (HemOnc)   \n",
       "\n",
       "     is_hierarchical  defines_ancestry reverse_relationship_id  \\\n",
       "34                 1                 1            RxNorm - ETC   \n",
       "37                 0                 0         Has precise ing   \n",
       "38                 0                 0           Has tradename   \n",
       "39                 1                 0    RxNorm has dose form   \n",
       "40                 0                 0                Has form   \n",
       "41                 1                 1          RxNorm has ing   \n",
       "42                 1                 0             Constitutes   \n",
       "43                 1                 1                Contains   \n",
       "44                 0                 1        Reformulation of   \n",
       "62                 0                 0       NDFRT - RxNorm eq   \n",
       "75                 0                 0            RxNorm - ATC   \n",
       "127                1                 0            ETC - RxNorm   \n",
       "128                0                 0            ATC - RxNorm   \n",
       "133                0                 0     NDFRT - RxNorm name   \n",
       "134                1                 1     RxNorm - NDFRT name   \n",
       "135                1                 1       RxNorm - ETC name   \n",
       "136                0                 0       ETC - RxNorm name   \n",
       "137                0                 0       ATC - RxNorm name   \n",
       "140                1                 1            RxNorm - DOI   \n",
       "141                1                 0            DOI - RxNorm   \n",
       "152                0                 0       RxNorm - ATC name   \n",
       "158                0                 1      RxNorm - VAProd eq   \n",
       "159                0                 0      VAProd - RxNorm eq   \n",
       "160                0                 0         RxNorm replaces   \n",
       "161                0                 0      RxNorm replaced by   \n",
       "164                0                 0      Source - RxNorm eq   \n",
       "165                0                 1      RxNorm - Source eq   \n",
       "214                0                 1      Quantified form of   \n",
       "280                1                 1     Has dose form group   \n",
       "281                1                 0      Dose form group of   \n",
       "304                1                 0            CVX - RxNorm   \n",
       "305                1                 1            RxNorm - CVX   \n",
       "355                0                 0     Has quantified form   \n",
       "356                1                 0     RxNorm inverse is a   \n",
       "357                1                 1             RxNorm is a   \n",
       "366                0                 0      RxNorm - SNOMED eq   \n",
       "367                0                 0      SNOMED - RxNorm eq   \n",
       "368                0                 0       NDFRT - SNOMED eq   \n",
       "369                0                 0       SNOMED - NDFRT eq   \n",
       "370                0                 0    VA Class - SNOMED eq   \n",
       "371                0                 0    SNOMED - VA Class eq   \n",
       "372                0                 0         ATC - SNOMED eq   \n",
       "373                0                 0         SNOMED - ATC eq   \n",
       "376                1                 1            RxNorm - SPL   \n",
       "377                1                 0            SPL - RxNorm   \n",
       "383                0                 0          Precise ing of   \n",
       "384                0                 1            Tradename of   \n",
       "385                1                 0     RxNorm dose form of   \n",
       "386                0                 0                 Form of   \n",
       "387                1                 0           RxNorm ing of   \n",
       "388                1                 1             Consists of   \n",
       "389                1                 0            Contained in   \n",
       "390                0                 0         Reformulated in   \n",
       "409                1                 1       RxNorm - NDFRT eq   \n",
       "485                1                 0         Rx antineopl of   \n",
       "486                1                 1        Has antineopl Rx   \n",
       "491                1                 0       Rx immunosuppr of   \n",
       "492                1                 1      Has immunosuppr Rx   \n",
       "495                1                 0     Rx local therapy of   \n",
       "496                1                 1     Has local therap Rx   \n",
       "530                0                 0      Has support med Rx   \n",
       "575                0                 0     RxNorm - ATC pr lat   \n",
       "576                0                 0     ATC - RxNorm pr lat   \n",
       "577                0                 0    RxNorm - ATC sec lat   \n",
       "578                0                 0    ATC - RxNorm sec lat   \n",
       "579                0                 0      RxNorm - ATC pr up   \n",
       "580                0                 0      ATC - RxNorm pr up   \n",
       "581                0                 0     RxNorm - ATC sec up   \n",
       "582                0                 0     ATC - RxNorm sec up   \n",
       "623                0                 0      Rx AB-drug cjgt of   \n",
       "628                0                 0     Cytotox chemo RX of   \n",
       "629                0                 0    Has cytotox chemo Rx   \n",
       "632                0                 0      Rx endocrine tx of   \n",
       "635                0                 0     Rx immunotherapy of   \n",
       "638                0                 0      Rx pept-drg cjg of   \n",
       "641                0                 0         Rx radiocjgt of   \n",
       "644                0                 0      Rx radiotherapy of   \n",
       "647                0                 0       Rx targeted tx of   \n",
       "648                0                 0       Rx support med of   \n",
       "649                1                 1     Has AB-drug cjgt Rx   \n",
       "650                1                 1     Has endocrine tx Rx   \n",
       "651                1                 1    Has immunotherapy Rx   \n",
       "652                1                 1     Has pept-drg cjg Rx   \n",
       "653                1                 1        Has radiocjgt Rx   \n",
       "654                1                 1     Has radiotherapy Rx   \n",
       "655                1                 1      Has targeted tx Rx   \n",
       "\n",
       "     relationship_concept_id  \n",
       "34                  44818809  \n",
       "37                  44818813  \n",
       "38                  44818814  \n",
       "39                  44818815  \n",
       "40                  44818816  \n",
       "41                  44818817  \n",
       "42                  44818818  \n",
       "43                  44818819  \n",
       "44                  44818820  \n",
       "62                  44818839  \n",
       "75                  44818810  \n",
       "127                 44818907  \n",
       "128                 44818908  \n",
       "133                 44818922  \n",
       "134                 44818923  \n",
       "135                 44818924  \n",
       "136                 44818925  \n",
       "137                 44818927  \n",
       "140                 44818930  \n",
       "141                 44818931  \n",
       "152                 44818926  \n",
       "158                 44818944  \n",
       "159                 44818945  \n",
       "160                 44818946  \n",
       "161                 44818947  \n",
       "164                 44818952  \n",
       "165                 44818953  \n",
       "214                 45754827  \n",
       "280                     5026  \n",
       "281                     5027  \n",
       "304                   581402  \n",
       "305                   581403  \n",
       "355                 45754828  \n",
       "356                 45754829  \n",
       "357                 45754830  \n",
       "366                 45754843  \n",
       "367                 45754844  \n",
       "368                 45754845  \n",
       "369                 45754846  \n",
       "370                 45754847  \n",
       "371                 45754848  \n",
       "372                 45754849  \n",
       "373                 45754850  \n",
       "376                 45754866  \n",
       "377                 45754867  \n",
       "383                 44818715  \n",
       "384                 44818716  \n",
       "385                 44818717  \n",
       "386                 44818718  \n",
       "387                 44818719  \n",
       "388                 44818720  \n",
       "389                 44818721  \n",
       "390                 44818722  \n",
       "409                 44818741  \n",
       "485                    32614  \n",
       "486                    32615  \n",
       "491                    32620  \n",
       "492                    32621  \n",
       "495                    32624  \n",
       "496                    32625  \n",
       "530                    32631  \n",
       "575                    32795  \n",
       "576                    32796  \n",
       "577                    32797  \n",
       "578                    32798  \n",
       "579                    32799  \n",
       "580                    32800  \n",
       "581                    32801  \n",
       "582                    32802  \n",
       "623                    33017  \n",
       "628                    33023  \n",
       "629                    33024  \n",
       "632                    33027  \n",
       "635                    33031  \n",
       "638                    33035  \n",
       "641                    33039  \n",
       "644                    33043  \n",
       "647                    33047  \n",
       "648                    32630  \n",
       "649                    33018  \n",
       "650                    33028  \n",
       "651                    33032  \n",
       "652                    33036  \n",
       "653                    33040  \n",
       "654                    33044  \n",
       "655                    33048  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.display import display as dsp\n",
    "with pd.option_context('display.max_rows', None,):\n",
    "    dsp(rxnorm_relationships)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "03d223a1-0141-4180-912b-982167e82210",
   "metadata": {},
   "source": [
    "## Mapping to different Rxnorm concept_class\n",
    "### Initial concept_class distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "be1af30c-184b-476d-8a9b-f9994f1ffdc3",
   "metadata": {},
   "outputs": [],
   "source": [
    "drugs_concept_classes = pd.merge(drug_exposure_df[['drug_concept_id','visit_occurrence_id']],rxnorm_concepts[['concept_id','concept_class_id','concept_name']],\n",
    "                                 left_on='drug_concept_id',right_on='concept_id',how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "dc6c811f-62e7-4945-9ddf-36ed1efa3477",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Clinical Drug          0.3304\n",
       "Quant Clinical Drug    0.3024\n",
       "Branded Drug           0.1601\n",
       "NaN                    0.1377\n",
       "Quant Branded Drug     0.0384\n",
       "Clinical Drug Form     0.0228\n",
       "Ingredient             0.0065\n",
       "Branded Drug Form      0.0010\n",
       "Clinical Drug Comp     0.0004\n",
       "Branded Drug Comp      0.0001\n",
       "Branded Pack           0.0001\n",
       "Precise Ingredient     0.0000\n",
       "Clinical Pack          0.0000\n",
       "Name: concept_class_id, dtype: float64"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Number of occurences\n",
    "round(drugs_concept_classes.concept_class_id.value_counts(dropna=False)/float(drugs_concept_classes.shape[0]),4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "de8a32ff-f6c4-40cf-9e0e-5244c91122f4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Clinical Drug          1295\n",
       "Branded Drug           1164\n",
       "Quant Branded Drug      301\n",
       "Quant Clinical Drug     282\n",
       "Branded Drug Comp       246\n",
       "Clinical Drug Comp      204\n",
       "Ingredient              167\n",
       "Branded Drug Form        82\n",
       "Clinical Drug Form       41\n",
       "Branded Pack             20\n",
       "Precise Ingredient        2\n",
       "NaN                       1\n",
       "Clinical Pack             1\n",
       "Name: concept_class_id, dtype: int64"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Number of unique codes\n",
    "drugs_concept_classes.groupby(by='drug_concept_id').first().concept_class_id.value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3055c13e-51fd-4b3f-99bc-3f84c8a54a0c",
   "metadata": {},
   "source": [
    "### Mapping to ingredients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "181676a7-d259-4d93-ba90-845d99c879de",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/quentin/anaconda3/envs/pmsi_db_analysis/lib/python3.10/site-packages/pandas/core/indexing.py:1667: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.obj[key] = value\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 7.03 s, sys: 3.72 s, total: 10.8 s\n",
      "Wall time: 13.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "drug_ing = concepts_toolbox.map_to_rxnorm_ingredient(drug_exposure_df[['visit_occurrence_id','drug_concept_id']],\n",
    "                                                     concept_relationship_df,rxnorm_concepts,strict=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "377e4ad5-360f-40d2-8d9a-1c65daed71a7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ingredient            0.8826\n",
      "NaN                   0.1168\n",
      "Clinical Drug Comp    0.0005\n",
      "Branded Drug Comp     0.0000\n",
      "Name: concept_class_id, dtype: float64\n",
      "Ingredient            4333794\n",
      "NaN                    573632\n",
      "Clinical Drug Comp       2574\n",
      "Branded Drug Comp           4\n",
      "Name: concept_class_id, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "drugs_ing_concept_classes = pd.merge(drug_ing[['rxnorm_ingredient','drug_concept_id']],rxnorm_concepts[['concept_id','concept_class_id','concept_name']],\n",
    "                                 left_on='rxnorm_ingredient',right_on='concept_id',how='left')\n",
    "\n",
    "print(round(\n",
    "drugs_ing_concept_classes.concept_class_id.value_counts(dropna=False)/float(drugs_ing_concept_classes.shape[0]),4))\n",
    "print(drugs_ing_concept_classes.concept_class_id.value_counts(dropna=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "03314f75-d940-4178-9c5c-2714982e5f01",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1165"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Number of unique concept_id when loosely brought back to ingredients\n",
    "len(drugs_ing_concept_classes.rxnorm_ingredient.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "6bd5c733-25c5-44f1-b7dc-d7599d8d2cb6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Ingredient            1158\n",
       "Clinical Drug Comp       5\n",
       "NaN                      1\n",
       "Branded Drug Comp        1\n",
       "Name: concept_class_id, dtype: int64"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# and their partition\n",
    "drugs_ing_concept_classes.groupby(by='rxnorm_ingredient').first().concept_class_id.value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8b727400-94ab-4d1d-9112-b32a6640df7f",
   "metadata": {},
   "source": [
    "Some drugs cannot be brought back to ingredients, e.g some immunological products (e.g concept_id 19098759 corresponds to \"Rabbit anti-human T-lymphocyte globulin 5 MG/ML,Drug,RxNorm,Clinical Drug Comp\", and only has 2 relations (consistitutes and tradename).\n",
    "For such examples we keep the concept_id with concept_class closest to the ingredient concept_class."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "7df46818-9e8f-43a7-b9ec-9058a416f72d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of visits: 50216\n",
      "Average number of drugs entries per visit: 97.77768042058308\n",
      "Average number of unique ingredient entries per visit: 34.946292018480165\n",
      "Total number of visits left after filtering out visits containing unknown drugs: 2167\n",
      "Average number of drugs entries per visit left after filtering: 23.333179510844484\n",
      "Average number of unique ingredient entries per visit left after filtering: 14.938163359483157\n"
     ]
    }
   ],
   "source": [
    "# check number of visits removed when dropping visits with unkown drug\n",
    "print(\"Total number of visits: \" + str(drug_ing.visit_occurrence_id.unique().shape[0]))\n",
    "print(\"Average number of drugs entries per visit: \" + str(drug_ing.groupby('visit_occurrence_id').count().mean().rxnorm_ingredient))\n",
    "print(\"Average number of unique ingredient entries per visit: \" + str(drug_ing.groupby('visit_occurrence_id')['rxnorm_ingredient'].nunique().mean()))\n",
    "\n",
    "print(\"Total number of visits left after filtering out visits containing unknown drugs: \" +\n",
    "      str(drug_ing.groupby('visit_occurrence_id').filter(lambda x: (x['rxnorm_ingredient']!=0).all()).visit_occurrence_id.unique().shape[0]))\n",
    "print(\"Average number of drugs entries per visit left after filtering: \" +\n",
    "      str(drug_ing.groupby('visit_occurrence_id').filter(lambda x: (x['rxnorm_ingredient']!=0).all()).groupby(by='visit_occurrence_id').count().mean().rxnorm_ingredient))\n",
    "print(\"Average number of unique ingredient entries per visit left after filtering: \" + str(drug_ing.groupby('visit_occurrence_id').filter(lambda x: (x['rxnorm_ingredient']!=0).all()).groupby('visit_occurrence_id')['rxnorm_ingredient'].nunique().mean()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "4e45da6f-4332-4b94-bbe2-4e52b481f7fb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "573632\n",
      "586586\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0              494387\n",
       "61553008348     19940\n",
       "17314931102      9142\n",
       "62584078833      8266\n",
       "66689036430      5912\n",
       "                ...  \n",
       "54569188401         1\n",
       "72140085731         1\n",
       "49281000610         1\n",
       "51079029072         1\n",
       "49281000950         1\n",
       "Name: drug_source_value, Length: 125, dtype: int64"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check number of unmapped drugs with missing source\n",
    "print(drug_exposure_df.query('drug_concept_id==0').shape[0])\n",
    "print(drug_exposure_df.query('drug_source_value==\"0\"').shape[0])\n",
    "drug_exposure_df.query('drug_concept_id==0').drug_source_value.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04d42136-25f7-47e8-bab0-c375b2e04990",
   "metadata": {},
   "source": [
    "The source value is missing for most of the unmapped drugs, not much to gain to try and remap the ones with none missing source value."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "142d6955-8300-42c9-bd02-b76b5c1d0665",
   "metadata": {},
   "source": [
    "## Export filtered table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "d46d8ff1-d3a6-4948-b9f7-74220eb9fe7d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>drug_exposure_id</th>\n",
       "      <th>person_id</th>\n",
       "      <th>drug_concept_id</th>\n",
       "      <th>drug_exposure_start_date</th>\n",
       "      <th>drug_exposure_end_date</th>\n",
       "      <th>drug_type_concept_id</th>\n",
       "      <th>quantity</th>\n",
       "      <th>route_concept_id</th>\n",
       "      <th>visit_occurrence_id</th>\n",
       "      <th>drug_source_value</th>\n",
       "      <th>drug_source_concept_id</th>\n",
       "      <th>route_source_value</th>\n",
       "      <th>dose_unit_source_value</th>\n",
       "      <th>quantity_source_value</th>\n",
       "      <th>rxnorm_ingredient_id</th>\n",
       "      <th>rxnorm_ingredient_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>396803253</td>\n",
       "      <td>392815432</td>\n",
       "      <td>0</td>\n",
       "      <td>2106-09-07</td>\n",
       "      <td>2106-09-08</td>\n",
       "      <td>38000177</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4139962</td>\n",
       "      <td>56351</td>\n",
       "      <td>0</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>SC</td>\n",
       "      <td>VIAL</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>396821890</td>\n",
       "      <td>392819347</td>\n",
       "      <td>0</td>\n",
       "      <td>2187-10-26</td>\n",
       "      <td>2187-10-28</td>\n",
       "      <td>38000177</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4139962</td>\n",
       "      <td>58698</td>\n",
       "      <td>0</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>SC</td>\n",
       "      <td>VIAL</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>396808816</td>\n",
       "      <td>392815414</td>\n",
       "      <td>0</td>\n",
       "      <td>2149-02-13</td>\n",
       "      <td>2149-02-13</td>\n",
       "      <td>38000177</td>\n",
       "      <td>50.0</td>\n",
       "      <td>4112421</td>\n",
       "      <td>58519</td>\n",
       "      <td>0</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>IV</td>\n",
       "      <td>mL</td>\n",
       "      <td>50</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>396821892</td>\n",
       "      <td>392819347</td>\n",
       "      <td>0</td>\n",
       "      <td>2187-10-27</td>\n",
       "      <td>2187-10-29</td>\n",
       "      <td>38000177</td>\n",
       "      <td>200.0</td>\n",
       "      <td>4112421</td>\n",
       "      <td>58698</td>\n",
       "      <td>0</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>IV</td>\n",
       "      <td>mL</td>\n",
       "      <td>200</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>396808830</td>\n",
       "      <td>392815420</td>\n",
       "      <td>40141424</td>\n",
       "      <td>2116-05-15</td>\n",
       "      <td>2116-05-17</td>\n",
       "      <td>38000177</td>\n",
       "      <td>0.6</td>\n",
       "      <td>4112421</td>\n",
       "      <td>56337</td>\n",
       "      <td>0</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>IV</td>\n",
       "      <td>SYR</td>\n",
       "      <td>0.6</td>\n",
       "      <td>967823.0</td>\n",
       "      <td>sodium chloride</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   drug_exposure_id  person_id  drug_concept_id drug_exposure_start_date  \\\n",
       "0         396803253  392815432                0               2106-09-07   \n",
       "1         396821890  392819347                0               2187-10-26   \n",
       "2         396808816  392815414                0               2149-02-13   \n",
       "3         396821892  392819347                0               2187-10-27   \n",
       "4         396808830  392815420         40141424               2116-05-15   \n",
       "\n",
       "  drug_exposure_end_date  drug_type_concept_id  quantity  route_concept_id  \\\n",
       "0             2106-09-08              38000177       0.0           4139962   \n",
       "1             2187-10-28              38000177       0.0           4139962   \n",
       "2             2149-02-13              38000177      50.0           4112421   \n",
       "3             2187-10-29              38000177     200.0           4112421   \n",
       "4             2116-05-17              38000177       0.6           4112421   \n",
       "\n",
       "   visit_occurrence_id drug_source_value  drug_source_concept_id  \\\n",
       "0                56351                 0                    <NA>   \n",
       "1                58698                 0                    <NA>   \n",
       "2                58519                 0                    <NA>   \n",
       "3                58698                 0                    <NA>   \n",
       "4                56337                 0                    <NA>   \n",
       "\n",
       "  route_source_value dose_unit_source_value quantity_source_value  \\\n",
       "0                 SC                   VIAL                     0   \n",
       "1                 SC                   VIAL                     0   \n",
       "2                 IV                     mL                    50   \n",
       "3                 IV                     mL                   200   \n",
       "4                 IV                    SYR                   0.6   \n",
       "\n",
       "   rxnorm_ingredient_id rxnorm_ingredient_name  \n",
       "0                   0.0                    NaN  \n",
       "1                   0.0                    NaN  \n",
       "2                   0.0                    NaN  \n",
       "3                   0.0                    NaN  \n",
       "4              967823.0        sodium chloride  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "drug_ing = concepts_toolbox.map_to_rxnorm_ingredient(drug_exposure_df,\n",
    "                                                     concept_relationship_df,rxnorm_concepts,strict=False,drug_concept_id_col='drug_concept_id')\n",
    "drug_ing = pd.merge(drug_ing,rxnorm_concepts[['concept_id','concept_name']],\n",
    "                                 left_on='rxnorm_ingredient',right_on='concept_id',how='left')\n",
    "drug_ing.drop(columns=\"concept_id\",inplace=True)\n",
    "drug_ing.rename(columns={\"concept_name\":\"rxnorm_ingredient_name\",\"rxnorm_ingredient\":\"rxnorm_ingredient_id\"},inplace=True)\n",
    "drug_ing.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "fe2f265f-d7c6-40d9-b87a-e6296514e2a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "keep_cols_list = ['person_id','visit_occurrence_id','drug_concept_id','drug_exposure_start_date','drug_exposure_end_date',\n",
    "                  'rxnorm_ingredient_id','rxnorm_ingredient_name']\n",
    "unique_ingredients = drug_ing.groupby(by=['visit_occurrence_id',\n",
    "                                                'rxnorm_ingredient_id']).agg({'drug_exposure_start_date':'min',\n",
    "                                                                              'drug_exposure_end_date':'max',\n",
    "                                                                              'rxnorm_ingredient_name':'first'})\n",
    "unique_ingredients.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "6aa15c24-c902-45f8-8c97-8504ac1a4a58",
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_ingredients.to_csv(datadir/'pre-processed/unique_ingredient_prescription.csv.gz',index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d170e613-ee02-453c-98d4-450445c711bf",
   "metadata": {},
   "source": [
    "# Train/Dev/Test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c9a17449-30d7-4728-9e46-747814d104ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "visit_occurrence_s = pd.read_csv(datadir/\"visit_occurrence.csv.gz\",usecols=['visit_occurrence_id']).squeeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "0dc7fb3b-230c-4702-8b27-9857250c667f",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert not visit_occurrence_s.duplicated().any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "f6582a7e-9c33-4c69-85b1-06157823c456",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(58976,)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "visit_occurrence_s.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "f069c80e-0290-4117-b508-b7685aee0b05",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "visit_occurrence_s.isnull().any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "9c7f3ecf-0605-4c58-afc7-c8ee41369dcf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "visit_occurrence_s.isna().any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "ef45b867-fb86-45fe-8c2e-ea402a958765",
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_visits = visit_occurrence_s.sample(5000,replace=False).sort_values()\n",
    "visit_occurrence_s=visit_occurrence_s[np.logical_not(visit_occurrence_s.isin(dev_visits))]\n",
    "\n",
    "test_visits = visit_occurrence_s.sample(5000,replace=False).sort_values()\n",
    "train_visits = visit_occurrence_s[np.logical_not(visit_occurrence_s.isin(test_visits))].sort_values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "ae537a69-9c53-40d3-bf83-fd71aa5e4e74",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(58976,)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.concat([dev_visits,train_visits,test_visits]).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "0196981b-09bd-4c64-9407-c7fdc6ad008e",
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_visits.to_csv(datadir/'pre-processed/dev_visits_id.csv.gz',index=False,header=True)\n",
    "test_visits.to_csv(datadir/'pre-processed/test_visits_id.csv.gz',index=False,header=True)\n",
    "train_visits.to_csv(datadir/'pre-processed/train_visits_id.csv.gz',index=False,header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "156cf7bc-c638-44e8-8498-43ca92bb00ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(1,5):\n",
    "    dev_visits = visit_occurrence_s.sample(5000,replace=False).sort_values()\n",
    "    visit_occurrence_s=visit_occurrence_s[np.logical_not(visit_occurrence_s.isin(dev_visits))]\n",
    "\n",
    "    test_visits = visit_occurrence_s.sample(5000,replace=False).sort_values()\n",
    "    train_visits = visit_occurrence_s[np.logical_not(visit_occurrence_s.isin(test_visits))].sort_values()\n",
    "    dev_visits.to_csv(datadir/(\"pre-processed/dev_visits_id_\"+str(i)+\".csv.gz\"),index=False,header=True)\n",
    "    test_visits.to_csv(datadir/('pre-processed/test_visits_id_'+str(i)+'.csv.gz'),index=False,header=True)\n",
    "    train_visits.to_csv(datadir/('pre-processed/train_visits_id_'+str(i)+'.csv.gz'),index=False,header=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e56993b3-6f9a-4a92-940e-98a3c940f424",
   "metadata": {},
   "source": [
    "# Create concept relationships for icd9 (sub)chapters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "b434644d-e5ba-4a09-a2e3-2937a397ec62",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#icd9_concepts = pd.read_csv(datadir/'pre-processed/icd9_concepts.csv.gz')\n",
    "icd9_concepts = concept_df.query('vocabulary_id.str.contains(\"ICD9CM\",na=False)')\n",
    "icd9_chapters_concepts = pd.read_csv(datadir/'icd9chapters_concepts.csv.gz')\n",
    "icd9_concepts = pd.concat([icd9_concepts,icd9_chapters_concepts])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "1fab9bd9-453a-469c-9d3c-0c4f91cc21b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "icd9_relationships = concept_relationship_df.query('concept_id_1 in '+str(icd9_concepts.concept_id.to_list()))\n",
    "icd9_relationships = icd9_relationships.query('concept_id_2 in '+str(icd9_concepts.concept_id.to_list()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "96fa5d58-135c-4083-bee8-c6ac98d4730e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>concept_id</th>\n",
       "      <th>concept_name</th>\n",
       "      <th>domain_id</th>\n",
       "      <th>vocabulary_id</th>\n",
       "      <th>concept_class_id</th>\n",
       "      <th>standard_concept</th>\n",
       "      <th>concept_code</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>201000000</td>\n",
       "      <td>Infectious And Parasitic Diseases</td>\n",
       "      <td>Condition</td>\n",
       "      <td>ICD9CM</td>\n",
       "      <td>Chapter</td>\n",
       "      <td>NaN</td>\n",
       "      <td>001-139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>201000001</td>\n",
       "      <td>Neoplasms</td>\n",
       "      <td>Condition</td>\n",
       "      <td>ICD9CM</td>\n",
       "      <td>Chapter</td>\n",
       "      <td>NaN</td>\n",
       "      <td>140-239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>201000002</td>\n",
       "      <td>Endocrine, Nutritional And Metabolic Diseases,...</td>\n",
       "      <td>Condition</td>\n",
       "      <td>ICD9CM</td>\n",
       "      <td>Chapter</td>\n",
       "      <td>NaN</td>\n",
       "      <td>240-279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>201000003</td>\n",
       "      <td>Diseases Of The Blood And Blood-Forming Organs</td>\n",
       "      <td>Condition</td>\n",
       "      <td>ICD9CM</td>\n",
       "      <td>Chapter</td>\n",
       "      <td>NaN</td>\n",
       "      <td>280-289</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>201000004</td>\n",
       "      <td>Mental Disorders</td>\n",
       "      <td>Condition</td>\n",
       "      <td>ICD9CM</td>\n",
       "      <td>Chapter</td>\n",
       "      <td>NaN</td>\n",
       "      <td>290-319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>168</th>\n",
       "      <td>201000168</td>\n",
       "      <td>Suicide And Self-Inflicted Injury</td>\n",
       "      <td>Condition</td>\n",
       "      <td>ICD9CM</td>\n",
       "      <td>SubChapter</td>\n",
       "      <td>NaN</td>\n",
       "      <td>E950-E959</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>169</th>\n",
       "      <td>201000169</td>\n",
       "      <td>Homicide And Injury Purposely Inflicted By Oth...</td>\n",
       "      <td>Condition</td>\n",
       "      <td>ICD9CM</td>\n",
       "      <td>SubChapter</td>\n",
       "      <td>NaN</td>\n",
       "      <td>E960-E969</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>170</th>\n",
       "      <td>201000170</td>\n",
       "      <td>Legal Intervention</td>\n",
       "      <td>Condition</td>\n",
       "      <td>ICD9CM</td>\n",
       "      <td>SubChapter</td>\n",
       "      <td>NaN</td>\n",
       "      <td>E970-E979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>171</th>\n",
       "      <td>201000171</td>\n",
       "      <td>Injury Undetermined Whether Accidentally Or Pu...</td>\n",
       "      <td>Condition</td>\n",
       "      <td>ICD9CM</td>\n",
       "      <td>SubChapter</td>\n",
       "      <td>NaN</td>\n",
       "      <td>E980-E989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>172</th>\n",
       "      <td>201000172</td>\n",
       "      <td>Injury Resulting From Operations Of War</td>\n",
       "      <td>Condition</td>\n",
       "      <td>ICD9CM</td>\n",
       "      <td>SubChapter</td>\n",
       "      <td>NaN</td>\n",
       "      <td>E990-E999</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>173 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     concept_id                                       concept_name  domain_id  \\\n",
       "0     201000000                  Infectious And Parasitic Diseases  Condition   \n",
       "1     201000001                                          Neoplasms  Condition   \n",
       "2     201000002  Endocrine, Nutritional And Metabolic Diseases,...  Condition   \n",
       "3     201000003     Diseases Of The Blood And Blood-Forming Organs  Condition   \n",
       "4     201000004                                   Mental Disorders  Condition   \n",
       "..          ...                                                ...        ...   \n",
       "168   201000168                  Suicide And Self-Inflicted Injury  Condition   \n",
       "169   201000169  Homicide And Injury Purposely Inflicted By Oth...  Condition   \n",
       "170   201000170                                 Legal Intervention  Condition   \n",
       "171   201000171  Injury Undetermined Whether Accidentally Or Pu...  Condition   \n",
       "172   201000172            Injury Resulting From Operations Of War  Condition   \n",
       "\n",
       "    vocabulary_id concept_class_id  standard_concept concept_code  \n",
       "0          ICD9CM          Chapter               NaN      001-139  \n",
       "1          ICD9CM          Chapter               NaN      140-239  \n",
       "2          ICD9CM          Chapter               NaN      240-279  \n",
       "3          ICD9CM          Chapter               NaN      280-289  \n",
       "4          ICD9CM          Chapter               NaN      290-319  \n",
       "..            ...              ...               ...          ...  \n",
       "168        ICD9CM       SubChapter               NaN    E950-E959  \n",
       "169        ICD9CM       SubChapter               NaN    E960-E969  \n",
       "170        ICD9CM       SubChapter               NaN    E970-E979  \n",
       "171        ICD9CM       SubChapter               NaN    E980-E989  \n",
       "172        ICD9CM       SubChapter               NaN    E990-E999  \n",
       "\n",
       "[173 rows x 7 columns]"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "icd9_chapters_concepts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "7dc2390f-5342-4cf8-84f1-3ff94047ee41",
   "metadata": {},
   "outputs": [],
   "source": [
    "# One missing E849 node in the Athena vocabulary, i've opened an issue: https://github.com/OHDSI/Vocabulary-v5.0/issues/608\n",
    "# Add it to the concept tables\n",
    "new_concept_code = icd9_concepts.concept_id.max()+1\n",
    "icd9_concepts =  icd9_concepts.append({'concept_id':new_concept_code,\n",
    "                                          'concept_name':\"Place of occurrence\",\n",
    "                                          'domain_id':\"Observation\",\n",
    "                                          'vocabulary_id':\"ICD9CM\",\n",
    "                                          'concept_class_id':\"3-dig nonbill E code\",\n",
    "                                          'standard_concept':np.NaN,\n",
    "                                          'concept_code':'E849',\n",
    "                                          'valid_start_date':\"1970-01-01\",\n",
    "                                          'valid_end_date':'2099-12-31',\n",
    "                                          'invalid_reason':np.NaN},ignore_index=True)\n",
    "# Add it to the relationships table\n",
    "new_concept_children = icd9_concepts.query('concept_code.str.startswith(\"E849.\")')\n",
    "is_a_df = pd.DataFrame({\"concept_id_1\":new_concept_children.concept_id,\n",
    "                   \"concept_id_2\":new_concept_code,\n",
    "                   \"relationship_id\":\"Is a\",\n",
    "                   \"valid_start_date\":\"1970-01-01\",\n",
    "                   \"valid_end_date\":\"2099-12-31\",\n",
    "                   \"invalid_reason\":np.nan})\n",
    "subsumes_df = pd.DataFrame({\"concept_id_1\":new_concept_code,\n",
    "                   \"concept_id_2\":new_concept_children.concept_id,\n",
    "                   \"relationship_id\":\"Subsumes\",\n",
    "                   \"valid_start_date\":\"1970-01-01\",\n",
    "                   \"valid_end_date\":\"2099-12-31\",\n",
    "                   \"invalid_reason\":np.nan})\n",
    "icd9_relationships = pd.concat([icd9_relationships,is_a_df,subsumes_df])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "d80059e1-7a55-414b-a3b5-72f5f5266285",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_icd_codes_from_range(x):\n",
    "    if x[0]==\"V\":\n",
    "        num_len=2\n",
    "    else:\n",
    "        num_len=3\n",
    "    start,end = x.split(sep=\"-\")\n",
    "    codes_list=[i for i in range(int(start[-num_len:]),int(end[-num_len:])+1)]\n",
    "    codes_list = [str(s).rjust(num_len,'0') for s in codes_list]\n",
    "    if x[0].isalpha():\n",
    "        codes_list = [x[0]+s for s in codes_list]\n",
    "    return codes_list\n",
    "\n",
    "def get_relationships_from_concept(x,icd9_concepts):\n",
    "    implied_concept_codes = str(get_icd_codes_from_range(x.concept_code))\n",
    "    implied_concepts_ids = icd9_concepts.query(\"concept_code in \" + implied_concept_codes).concept_id\n",
    "    subsumes_df = pd.DataFrame({\"concept_id_1\":x.concept_id,\n",
    "                           \"concept_id_2\":implied_concepts_ids,\n",
    "                           \"relationship_id\":\"Subsumes\",\n",
    "                           \"valid_start_date\":\"1970-01-01\",\n",
    "                           \"valid_end_date\":\"2099-12-31\",\n",
    "                           \"invalid_reason\":np.nan})\n",
    "    isa_df = pd.DataFrame({\"concept_id_1\":implied_concepts_ids,\n",
    "                       \"concept_id_2\":x.concept_id,\n",
    "                       \"relationship_id\":\"Is a\",\n",
    "                       \"valid_start_date\":\"1970-01-01\",\n",
    "                       \"valid_end_date\":\"2099-12-31\",\n",
    "                       \"invalid_reason\":np.nan})\n",
    "    \n",
    "    return pd.concat([subsumes_df,isa_df])\n",
    "\n",
    "def get_chapter_subchapter_relationships(icd9_chapters_concepts):\n",
    "    icd9_chapters_concepts['code_range'] = icd9_chapters_concepts.apply(lambda x: set(get_icd_codes_from_range(x.concept_code)), axis=1)\n",
    "    #icd9_chapters_concepts['code_range_min'] = icd9_chapters_concepts.apply(lambda x: min(x.code_range), axis=1)\n",
    "    #icd9_chapters_concepts['code_range_max'] = icd9_chapters_concepts.apply(lambda x: max(x.code_range), axis=1)\n",
    "    def get_relationships_from_concept_range(x,icd9_concepts):\n",
    "        implied_concepts_ids = icd9_concepts[icd9_concepts.apply(lambda y: y.code_range.issubset(x.code_range),axis=1)].concept_id\n",
    "        subsumes_df = pd.DataFrame({\"concept_id_1\":x.concept_id,\n",
    "                               \"concept_id_2\":implied_concepts_ids,\n",
    "                               \"relationship_id\":\"Subsumes\",\n",
    "                               \"valid_start_date\":\"1970-01-01\",\n",
    "                               \"valid_end_date\":\"2099-12-31\",\n",
    "                               \"invalid_reason\":np.nan})\n",
    "        isa_df = pd.DataFrame({\"concept_id_1\":implied_concepts_ids,\n",
    "                           \"concept_id_2\":x.concept_id,\n",
    "                           \"relationship_id\":\"Is a\",\n",
    "                           \"valid_start_date\":\"1970-01-01\",\n",
    "                           \"valid_end_date\":\"2099-12-31\",\n",
    "                           \"invalid_reason\":np.nan})\n",
    "        return pd.concat([subsumes_df,isa_df]) \n",
    "    chapter_relationships = pd.concat(icd9_chapters_concepts.apply(lambda x : get_relationships_from_concept_range(x,icd9_chapters_concepts),\n",
    "                                                      axis=1).to_list()).reset_index(drop=True)\n",
    "    return chapter_relationships.query('concept_id_1!=concept_id_2')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "75cd1345-4ba7-48b6-8c94-fca7b34daf6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create relationships between chapter/subchapters and >=3 dig codes\n",
    "icd9_chapters_relationships  = pd.concat(icd9_chapters_concepts.apply(lambda x : get_relationships_from_concept(x,icd9_concepts),axis=1).to_list()).reset_index(drop=True)\n",
    "# Create relationships between chapters and subchapters\n",
    "icd9_chapters_relationships = pd.concat([icd9_chapters_relationships,get_chapter_subchapter_relationships(icd9_chapters_concepts)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "cb26d914-0f25-4cae-a088-09d2ff32ed84",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export the icd9 concepts\n",
    "icd9_concepts = icd9_concepts.query(\"invalid_reason!='D'\") # Remove deprecated ICD9 codes\n",
    "icd9_export = icd9_concepts.sort_values(by=\"concept_code\",inplace=False)\n",
    "icd9_export.drop(columns=['valid_start_date','valid_end_date','invalid_reason'],inplace=True)\n",
    "icd9_export.to_csv(datadir/'pre-processed/icd9_concepts.csv.gz',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "83ec9d87-694c-4f6c-a998-b301bb2d7596",
   "metadata": {},
   "outputs": [],
   "source": [
    "icd9_relationships = pd.concat([icd9_relationships,icd9_chapters_relationships])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "6ffe7bf4-d680-4a73-be55-2ff4ce4e56b2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>concept_id</th>\n",
       "      <th>concept_name</th>\n",
       "      <th>domain_id</th>\n",
       "      <th>vocabulary_id</th>\n",
       "      <th>concept_class_id</th>\n",
       "      <th>standard_concept</th>\n",
       "      <th>concept_code</th>\n",
       "      <th>valid_start_date</th>\n",
       "      <th>valid_end_date</th>\n",
       "      <th>invalid_reason</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [concept_id, concept_name, domain_id, vocabulary_id, concept_class_id, standard_concept, concept_code, valid_start_date, valid_end_date, invalid_reason]\n",
       "Index: []"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check that all nodes are correcly connected to the tree, all nodes should have a parent except roots\n",
    "icd9_concepts.query('concept_class_id!=\"Chapter\"')[~icd9_concepts.query('concept_class_id!=\"Chapter\"').concept_id.isin(icd9_relationships.query('relationship_id==\"Is a\"').concept_id_1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "b6184b49-0676-4c44-8ae0-c904ca1d92b1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(55514, 6)\n",
      "(45464, 6)\n",
      "(35414, 6)\n"
     ]
    }
   ],
   "source": [
    "from importlib import reload\n",
    "reload(concepts_toolbox)\n",
    "# Prune redundant relationships\n",
    "print(icd9_relationships.shape)\n",
    "icd9_relationships = concepts_toolbox.hierarchy_dag_to_tree(icd9_relationships,'Is a',concept_id_col=1)\n",
    "print(icd9_relationships.shape)\n",
    "icd9_relationships = concepts_toolbox.hierarchy_dag_to_tree(icd9_relationships,'Subsumes',concept_id_col=2)\n",
    "print(icd9_relationships.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "445e49d9-cdce-4a67-8fee-aded6d60c4b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assert that all not chapter nodes have exactly 1 hierarchical relation (factor 2 for the opposite relation direction)\n",
    "assert concepts_toolbox.test_relationship_tree_coherence(icd9_concepts,icd9_relationships,\n",
    "                                                         relationship='Is a', reciprocal_relationship=\"Subsumes\",\n",
    "                                                  root_concepts = icd9_concepts.query('concept_class_id==\"Chapter\"')['concept_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "2dbf2e4f-34b4-4d07-8433-356af3dbe83e",
   "metadata": {},
   "outputs": [],
   "source": [
    "icd9_relationships.to_csv(datadir/'pre-processed/icd9_concept_relationships.csv.gz',index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6e3a8ea2-8487-42d3-b7d8-843d4a95bad7",
   "metadata": {},
   "source": [
    "# General TODO:\n",
    "\n",
    "- [ ] check content of the relationship and vocabulary tables\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2f923ad-89e1-442c-9970-a21d074fd1ae",
   "metadata": {},
   "source": [
    "# Perf checks\n",
    "## pandas chunk reading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "a52cc7c5-07a6-426f-a429-dec70664707e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_subsample(df):\n",
    "    return pd.DataFrame.sample(df,10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "d0555192-8b3a-4998-ab2c-894a68e2eb15",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.2 s ± 36 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "tmp = pandas_utils.read_filter_by_chunk(pd.read_csv,datadir/\"drug_exposure.csv.gz\",test_subsample,chunksize=1000,nrows=200000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "cc6ac763-9085-4144-88a4-ebf0b56ed179",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "581 ms ± 30.4 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "tmp = pd.read_csv(datadir/\"drug_exposure.csv.gz\",nrows=200000).sample(2000)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:pmsi_db_analysis]",
   "language": "python",
   "name": "conda-env-pmsi_db_analysis-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}