data-analysis/024-pandas-read_html/pandas-read_html.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pandas read_html()\n",
    "\n",
    "This is a notebook for the medium article [Pandas read_html](https://bindichen.medium.com/all-pandas-read-html-you-should-know-for-scraping-data-from-html-tables-a3cbb5ce8274)\n",
    "\n",
    "Please check out article for instructions\n",
    "\n",
    "**License**: [BSD 2-Clause](https://opensource.org/licenses/BSD-2-Clause)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from IPython.display import display_html"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Read an HTML table from a string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<table>\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th>date</th>\n",
       "      <th>name</th>\n",
       "      <th>year</th>\n",
       "      <th>cost</th>\n",
       "      <th>region</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Jenny</td>\n",
       "      <td>1998</td>\n",
       "      <td>0.2</td>\n",
       "      <td>South</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2020-01-02</td>\n",
       "      <td>Alice</td>\n",
       "      <td>1992</td>\n",
       "      <td>-1.34</td>\n",
       "      <td>East</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2020-01-03</td>\n",
       "      <td>Tomas</td>\n",
       "      <td>1982</td>\n",
       "      <td>1.00023</td>\n",
       "      <td>South</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "html_string = \"\"\"\n",
    "<table>\n",
    "  <thead>\n",
    "    <tr>\n",
    "      <th>date</th>\n",
    "      <th>name</th>\n",
    "      <th>year</th>\n",
    "      <th>cost</th>\n",
    "      <th>region</th>\n",
    "    </tr>\n",
    "  </thead>\n",
    "  <tbody>\n",
    "    <tr>\n",
    "      <td>2020-01-01</td>\n",
    "      <td>Jenny</td>\n",
    "      <td>1998</td>\n",
    "      <td>0.2</td>\n",
    "      <td>South</td>\n",
    "    </tr>\n",
    "    <tr>\n",
    "      <td>2020-01-02</td>\n",
    "      <td>Alice</td>\n",
    "      <td>1992</td>\n",
    "      <td>-1.34</td>\n",
    "      <td>East</td>\n",
    "    </tr>\n",
    "    <tr>\n",
    "      <td>2020-01-03</td>\n",
    "      <td>Tomas</td>\n",
    "      <td>1982</td>\n",
    "      <td>1.00023</td>\n",
    "      <td>South</td>\n",
    "    </tr>\n",
    "  </tbody>\n",
    "</table>\n",
    "\"\"\"\n",
    "\n",
    "# Display the HTML representation\n",
    "display_html(html_string, raw=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read html string into a list of DataFrames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[         date   name  year     cost region\n",
       " 0  2020-01-01  Jenny  1998  0.20000  South\n",
       " 1  2020-01-02  Alice  1992 -1.34000   East\n",
       " 2  2020-01-03  Tomas  1982  1.00023  South]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfs = pd.read_html(html_string)\n",
    "# Result is not a Pandas DataFrame\n",
    "dfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "list"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Result is a Python list\n",
    "type(dfs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>name</th>\n",
       "      <th>year</th>\n",
       "      <th>cost</th>\n",
       "      <th>region</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Jenny</td>\n",
       "      <td>1998</td>\n",
       "      <td>0.20000</td>\n",
       "      <td>South</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020-01-02</td>\n",
       "      <td>Alice</td>\n",
       "      <td>1992</td>\n",
       "      <td>-1.34000</td>\n",
       "      <td>East</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2020-01-03</td>\n",
       "      <td>Tomas</td>\n",
       "      <td>1982</td>\n",
       "      <td>1.00023</td>\n",
       "      <td>South</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         date   name  year     cost region\n",
       "0  2020-01-01  Jenny  1998  0.20000  South\n",
       "1  2020-01-02  Alice  1992 -1.34000   East\n",
       "2  2020-01-03  Tomas  1982  1.00023  South"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Use index to get the table\n",
    "dfs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 3 entries, 0 to 2\n",
      "Data columns (total 5 columns):\n",
      " #   Column  Non-Null Count  Dtype  \n",
      "---  ------  --------------  -----  \n",
      " 0   date    3 non-null      object \n",
      " 1   name    3 non-null      object \n",
      " 2   year    3 non-null      int64  \n",
      " 3   cost    3 non-null      float64\n",
      " 4   region  3 non-null      object \n",
      "dtypes: float64(1), int64(1), object(3)\n",
      "memory usage: 248.0+ bytes\n"
     ]
    }
   ],
   "source": [
    "dfs[0].info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Reading tables from a URL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total tables: 31\n"
     ]
    }
   ],
   "source": [
    "URL = 'https://en.wikipedia.org/wiki/London'\n",
    "\n",
    "dfs = pd.read_html(URL)\n",
    "\n",
    "print(f'Total tables: {len(dfs)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">2011 United Kingdom Census[225]</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>Country of birth</th>\n",
       "      <th>Population</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>United Kingdom</td>\n",
       "      <td>5175677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>India</td>\n",
       "      <td>262247</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Poland</td>\n",
       "      <td>158300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Ireland</td>\n",
       "      <td>129807</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Nigeria</td>\n",
       "      <td>114718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Pakistan</td>\n",
       "      <td>112457</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Bangladesh</td>\n",
       "      <td>109948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Jamaica</td>\n",
       "      <td>87467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Sri Lanka</td>\n",
       "      <td>84542</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>France</td>\n",
       "      <td>66654</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  2011 United Kingdom Census[225]           \n",
       "                 Country of birth Population\n",
       "0                  United Kingdom    5175677\n",
       "1                           India     262247\n",
       "2                          Poland     158300\n",
       "3                         Ireland     129807\n",
       "4                         Nigeria     114718\n",
       "5                        Pakistan     112457\n",
       "6                      Bangladesh     109948\n",
       "7                         Jamaica      87467\n",
       "8                       Sri Lanka      84542\n",
       "9                          France      66654"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfs[6]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Reading tables from a file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>name</th>\n",
       "      <th>year</th>\n",
       "      <th>cost</th>\n",
       "      <th>region</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Jenny</td>\n",
       "      <td>1998</td>\n",
       "      <td>0.20000</td>\n",
       "      <td>South</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020-01-02</td>\n",
       "      <td>Alice</td>\n",
       "      <td>1992</td>\n",
       "      <td>-1.34000</td>\n",
       "      <td>East</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2020-01-03</td>\n",
       "      <td>Tomas</td>\n",
       "      <td>1982</td>\n",
       "      <td>1.00023</td>\n",
       "      <td>South</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         date   name  year     cost region\n",
       "0  2020-01-01  Jenny  1998  0.20000  South\n",
       "1  2020-01-02  Alice  1992 -1.34000   East\n",
       "2  2020-01-03  Tomas  1982  1.00023  South"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file_path = 'html_string.txt'\n",
    "with open(file_path, 'r') as f:\n",
    "    dfs = pd.read_html(f.read())\n",
    "\n",
    "dfs[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Parsing date columns using `parse_dates`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfs = pd.read_html(html_string, parse_dates=['date'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>name</th>\n",
       "      <th>year</th>\n",
       "      <th>cost</th>\n",
       "      <th>region</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Jenny</td>\n",
       "      <td>1998</td>\n",
       "      <td>0.20000</td>\n",
       "      <td>South</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020-01-02</td>\n",
       "      <td>Alice</td>\n",
       "      <td>1992</td>\n",
       "      <td>-1.34000</td>\n",
       "      <td>East</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2020-01-03</td>\n",
       "      <td>Tomas</td>\n",
       "      <td>1982</td>\n",
       "      <td>1.00023</td>\n",
       "      <td>South</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        date   name  year     cost region\n",
       "0 2020-01-01  Jenny  1998  0.20000  South\n",
       "1 2020-01-02  Alice  1992 -1.34000   East\n",
       "2 2020-01-03  Tomas  1982  1.00023  South"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 3 entries, 0 to 2\n",
      "Data columns (total 5 columns):\n",
      " #   Column  Non-Null Count  Dtype         \n",
      "---  ------  --------------  -----         \n",
      " 0   date    3 non-null      datetime64[ns]\n",
      " 1   name    3 non-null      object        \n",
      " 2   year    3 non-null      int64         \n",
      " 3   cost    3 non-null      float64       \n",
      " 4   region  3 non-null      object        \n",
      "dtypes: datetime64[ns](1), float64(1), int64(1), object(2)\n",
      "memory usage: 248.0+ bytes\n"
     ]
    }
   ],
   "source": [
    "dfs[0].info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Converting values with `converters`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<table>\n",
       "  <tr>\n",
       "    <th>ID</th>\n",
       "    <th>date</th>\n",
       "    <th>name</th>\n",
       "    <th>year</th>\n",
       "    <th>cost</th>\n",
       "    <th>region</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <td>0001</td>\n",
       "    <td>2020-01-01</td>\n",
       "    <td>Jenny</td>\n",
       "    <td>1998</td>\n",
       "    <td>0.2</td>\n",
       "    <td>South</td>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <td>0002</td>\n",
       "    <td>2020-01-02</td>\n",
       "    <td>Alice</td>\n",
       "    <td>1992</td>\n",
       "    <td>-1.34</td>\n",
       "    <td>East</td>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <td>0003</td>\n",
       "    <td>2020-01-03</td>\n",
       "    <td>Tomas</td>\n",
       "    <td>1982</td>\n",
       "    <td>1.00023</td>\n",
       "    <td>South</td>\n",
       "  </tr>\n",
       "</table>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "html_string = \"\"\"\n",
    "<table>\n",
    "  <tr>\n",
    "    <th>ID</th>\n",
    "    <th>date</th>\n",
    "    <th>name</th>\n",
    "    <th>year</th>\n",
    "    <th>cost</th>\n",
    "    <th>region</th>\n",
    "  </tr>\n",
    "  <tr>\n",
    "    <td>0001</td>\n",
    "    <td>2020-01-01</td>\n",
    "    <td>Jenny</td>\n",
    "    <td>1998</td>\n",
    "    <td>0.2</td>\n",
    "    <td>South</td>\n",
    "  </tr>\n",
    "  <tr>\n",
    "    <td>0002</td>\n",
    "    <td>2020-01-02</td>\n",
    "    <td>Alice</td>\n",
    "    <td>1992</td>\n",
    "    <td>-1.34</td>\n",
    "    <td>East</td>\n",
    "  </tr>\n",
    "  <tr>\n",
    "    <td>0003</td>\n",
    "    <td>2020-01-03</td>\n",
    "    <td>Tomas</td>\n",
    "    <td>1982</td>\n",
    "    <td>1.00023</td>\n",
    "    <td>South</td>\n",
    "  </tr>\n",
    "</table>\n",
    "\"\"\"\n",
    "\n",
    "# Display the HTML representation\n",
    "display_html(html_string, raw=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfs = pd.read_html(html_string, converters={\n",
    "    'ID': str,\n",
    "    'year': int,\n",
    "    'cost': float,\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>date</th>\n",
       "      <th>name</th>\n",
       "      <th>year</th>\n",
       "      <th>cost</th>\n",
       "      <th>region</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0001</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Jenny</td>\n",
       "      <td>1998</td>\n",
       "      <td>0.20000</td>\n",
       "      <td>South</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0002</td>\n",
       "      <td>2020-01-02</td>\n",
       "      <td>Alice</td>\n",
       "      <td>1992</td>\n",
       "      <td>-1.34000</td>\n",
       "      <td>East</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0003</td>\n",
       "      <td>2020-01-03</td>\n",
       "      <td>Tomas</td>\n",
       "      <td>1982</td>\n",
       "      <td>1.00023</td>\n",
       "      <td>South</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     ID        date   name  year     cost region\n",
       "0  0001  2020-01-01  Jenny  1998  0.20000  South\n",
       "1  0002  2020-01-02  Alice  1992 -1.34000   East\n",
       "2  0003  2020-01-03  Tomas  1982  1.00023  South"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 3 entries, 0 to 2\n",
      "Data columns (total 6 columns):\n",
      " #   Column  Non-Null Count  Dtype  \n",
      "---  ------  --------------  -----  \n",
      " 0   ID      3 non-null      object \n",
      " 1   date    3 non-null      object \n",
      " 2   name    3 non-null      object \n",
      " 3   year    3 non-null      int64  \n",
      " 4   cost    3 non-null      float64\n",
      " 5   region  3 non-null      object \n",
      "dtypes: float64(1), int64(1), object(4)\n",
      "memory usage: 272.0+ bytes\n"
     ]
    }
   ],
   "source": [
    "dfs[0].info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. MultiIndex, header, and index column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<table>\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th colspan=\"5\">Year 2020</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date</th>\n",
       "      <th>name</th>\n",
       "      <th>year</th>\n",
       "      <th>cost</th>\n",
       "      <th>region</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Jenny</td>\n",
       "      <td>1998</td>\n",
       "      <td>1.2</td>\n",
       "      <td>South</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2020-01-02</td>\n",
       "      <td>Alice</td>\n",
       "      <td>1992</td>\n",
       "      <td>-1.34</td>\n",
       "      <td>East</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "html_string = \"\"\"\n",
    "<table>\n",
    "  <thead>\n",
    "    <tr>\n",
    "      <th colspan=\"5\">Year 2020</th>\n",
    "    </tr>\n",
    "    <tr>\n",
    "      <th>date</th>\n",
    "      <th>name</th>\n",
    "      <th>year</th>\n",
    "      <th>cost</th>\n",
    "      <th>region</th>\n",
    "    </tr>\n",
    "  </thead>\n",
    "  <tbody>\n",
    "    <tr>\n",
    "      <td>2020-01-01</td>\n",
    "      <td>Jenny</td>\n",
    "      <td>1998</td>\n",
    "      <td>1.2</td>\n",
    "      <td>South</td>\n",
    "    </tr>\n",
    "    <tr>\n",
    "      <td>2020-01-02</td>\n",
    "      <td>Alice</td>\n",
    "      <td>1992</td>\n",
    "      <td>-1.34</td>\n",
    "      <td>East</td>\n",
    "    </tr>\n",
    "  </tbody>\n",
    "</table>\n",
    "\"\"\"\n",
    "\n",
    "# Display the HTML representation\n",
    "display_html(html_string, raw=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"5\" halign=\"left\">Year 2020</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>name</th>\n",
       "      <th>year</th>\n",
       "      <th>cost</th>\n",
       "      <th>region</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Jenny</td>\n",
       "      <td>1998</td>\n",
       "      <td>1.20</td>\n",
       "      <td>South</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020-01-02</td>\n",
       "      <td>Alice</td>\n",
       "      <td>1992</td>\n",
       "      <td>-1.34</td>\n",
       "      <td>East</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Year 2020                          \n",
       "         date   name  year  cost region\n",
       "0  2020-01-01  Jenny  1998  1.20  South\n",
       "1  2020-01-02  Alice  1992 -1.34   East"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# It creates MultiIndex because multiple rows in <thead>\n",
    "dfs = pd.read_html(html_string)\n",
    "dfs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>name</th>\n",
       "      <th>year</th>\n",
       "      <th>cost</th>\n",
       "      <th>region</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Jenny</td>\n",
       "      <td>1998</td>\n",
       "      <td>1.20</td>\n",
       "      <td>South</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020-01-02</td>\n",
       "      <td>Alice</td>\n",
       "      <td>1992</td>\n",
       "      <td>-1.34</td>\n",
       "      <td>East</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         date   name  year  cost region\n",
       "0  2020-01-01  Jenny  1998  1.20  South\n",
       "1  2020-01-02  Alice  1992 -1.34   East"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Specify a header row:\n",
    "dfs = pd.read_html(html_string, header=1)\n",
    "dfs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>year</th>\n",
       "      <th>cost</th>\n",
       "      <th>region</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2020-01-01</th>\n",
       "      <td>Jenny</td>\n",
       "      <td>1998</td>\n",
       "      <td>1.20</td>\n",
       "      <td>South</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-01-02</th>\n",
       "      <td>Alice</td>\n",
       "      <td>1992</td>\n",
       "      <td>-1.34</td>\n",
       "      <td>East</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             name  year  cost region\n",
       "date                                \n",
       "2020-01-01  Jenny  1998  1.20  South\n",
       "2020-01-02  Alice  1992 -1.34   East"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Specify a index column\n",
    "dfs = pd.read_html(html_string, header=1, index_col=0)\n",
    "dfs[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7. Matching a table with `match` argument"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<table id=\"report\">\n",
       "  <caption>2020 report</caption>\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th>date</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Jenny</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2020-01-02</td>\n",
       "      <td>Alice</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "\n",
       "<table>\n",
       "  <caption>Average income</caption>\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th>name</th>\n",
       "      <th>income</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>Tom</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>James</td>\n",
       "      <td>300</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# table with caption\n",
    "html_string = \"\"\"\n",
    "<table id=\"report\">\n",
    "  <caption>2020 report</caption>\n",
    "  <thead>\n",
    "    <tr>\n",
    "      <th>date</th>\n",
    "      <th>name</th>\n",
    "    </tr>\n",
    "  </thead>\n",
    "  <tbody>\n",
    "    <tr>\n",
    "      <td>2020-01-01</td>\n",
    "      <td>Jenny</td>\n",
    "    </tr>\n",
    "    <tr>\n",
    "      <td>2020-01-02</td>\n",
    "      <td>Alice</td>\n",
    "    </tr>\n",
    "  </tbody>\n",
    "</table>\n",
    "\n",
    "<table>\n",
    "  <caption>Average income</caption>\n",
    "  <thead>\n",
    "    <tr>\n",
    "      <th>name</th>\n",
    "      <th>income</th>\n",
    "    </tr>\n",
    "  </thead>\n",
    "  <tbody>\n",
    "    <tr>\n",
    "      <td>Tom</td>\n",
    "      <td>200</td>\n",
    "    </tr>\n",
    "    <tr>\n",
    "      <td>James</td>\n",
    "      <td>300</td>\n",
    "    </tr>\n",
    "  </tbody>\n",
    "</table>\n",
    "\"\"\"\n",
    "# Display the HTML representation\n",
    "display_html(html_string, raw=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    }
   ],
   "source": [
    "# matching caption\n",
    "dfs = pd.read_html(html_string, match='2020 report')\n",
    "print(len(dfs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Jenny</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020-01-02</td>\n",
       "      <td>Alice</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         date   name\n",
       "0  2020-01-01  Jenny\n",
       "1  2020-01-02  Alice"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    }
   ],
   "source": [
    "# matching value\n",
    "dfs = pd.read_html(html_string, match='James')\n",
    "print(len(dfs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>income</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Tom</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>James</td>\n",
       "      <td>300</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    name  income\n",
       "0    Tom     200\n",
       "1  James     300"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfs[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 8. Filter talbe with `attrs` "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfs = pd.read_html(html_string, attrs={'id': 'report'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Jenny</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020-01-02</td>\n",
       "      <td>Alice</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         date   name\n",
       "0  2020-01-01  Jenny\n",
       "1  2020-01-02  Alice"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfs[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 9. Working with missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<table>\n",
       "  <tr>\n",
       "    <th>date</th>\n",
       "    <th>name</th>\n",
       "    <th>year</th>\n",
       "    <th>cost</th>\n",
       "    <th>region</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <td>2020-01-01</td>\n",
       "    <td>Jenny</td>\n",
       "    <td>1998</td>\n",
       "    <td>1.2</td>\n",
       "    <td>South</td>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <td>2020-01-02</td>\n",
       "    <td>Alice</td>\n",
       "    <td>1992</td>\n",
       "    <td></td>\n",
       "    <td>East</td>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <td>2020-01-03</td>\n",
       "    <td>Tomas</td>\n",
       "    <td>1982</td>\n",
       "    <td></td>\n",
       "    <td>South</td>\n",
       "  </tr>\n",
       "</table>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "html_string = \"\"\"\n",
    "<table>\n",
    "  <tr>\n",
    "    <th>date</th>\n",
    "    <th>name</th>\n",
    "    <th>year</th>\n",
    "    <th>cost</th>\n",
    "    <th>region</th>\n",
    "  </tr>\n",
    "  <tr>\n",
    "    <td>2020-01-01</td>\n",
    "    <td>Jenny</td>\n",
    "    <td>1998</td>\n",
    "    <td>1.2</td>\n",
    "    <td>South</td>\n",
    "  </tr>\n",
    "  <tr>\n",
    "    <td>2020-01-02</td>\n",
    "    <td>Alice</td>\n",
    "    <td>1992</td>\n",
    "    <td></td>\n",
    "    <td>East</td>\n",
    "  </tr>\n",
    "  <tr>\n",
    "    <td>2020-01-03</td>\n",
    "    <td>Tomas</td>\n",
    "    <td>1982</td>\n",
    "    <td></td>\n",
    "    <td>South</td>\n",
    "  </tr>\n",
    "</table>\n",
    "\"\"\"\n",
    "\n",
    "# Display the HTML representation\n",
    "display_html(html_string, raw=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>name</th>\n",
       "      <th>year</th>\n",
       "      <th>cost</th>\n",
       "      <th>region</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Jenny</td>\n",
       "      <td>1998</td>\n",
       "      <td>1.2</td>\n",
       "      <td>South</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020-01-02</td>\n",
       "      <td>Alice</td>\n",
       "      <td>1992</td>\n",
       "      <td>NaN</td>\n",
       "      <td>East</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2020-01-03</td>\n",
       "      <td>Tomas</td>\n",
       "      <td>1982</td>\n",
       "      <td>NaN</td>\n",
       "      <td>South</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         date   name  year  cost region\n",
       "0  2020-01-01  Jenny  1998   1.2  South\n",
       "1  2020-01-02  Alice  1992   NaN   East\n",
       "2  2020-01-03  Tomas  1982   NaN  South"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# By default, all empty strings are treated as missing values and read as NaN\n",
    "dfs = pd.read_html(html_string)\n",
    "dfs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>name</th>\n",
       "      <th>year</th>\n",
       "      <th>cost</th>\n",
       "      <th>region</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Jenny</td>\n",
       "      <td>1998</td>\n",
       "      <td>1.2</td>\n",
       "      <td>South</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020-01-02</td>\n",
       "      <td>Alice</td>\n",
       "      <td>1992</td>\n",
       "      <td></td>\n",
       "      <td>East</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2020-01-03</td>\n",
       "      <td>Tomas</td>\n",
       "      <td>1982</td>\n",
       "      <td></td>\n",
       "      <td>South</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         date   name  year cost region\n",
       "0  2020-01-01  Jenny  1998  1.2  South\n",
       "1  2020-01-02  Alice  1992        East\n",
       "2  2020-01-03  Tomas  1982       South"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# To keep empty string\n",
    "dfs = pd.read_html(html_string, keep_default_na=False)\n",
    "dfs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<table>\n",
       "  <tr>\n",
       "    <th>date</th>\n",
       "    <th>name</th>\n",
       "    <th>year</th>\n",
       "    <th>cost</th>\n",
       "    <th>region</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <td>2020-01-01</td>\n",
       "    <td>Jenny</td>\n",
       "    <td>1998</td>\n",
       "    <td>1.2</td>\n",
       "    <td>South</td>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <td>2020-01-02</td>\n",
       "    <td>Alice</td>\n",
       "    <td>1992</td>\n",
       "    <td>?</td>\n",
       "    <td>East</td>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <td>2020-01-03</td>\n",
       "    <td>Tomas</td>\n",
       "    <td>1982</td>\n",
       "    <td>&</td>\n",
       "    <td>South</td>\n",
       "  </tr>\n",
       "</table>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "html_string = \"\"\"\n",
    "<table>\n",
    "  <tr>\n",
    "    <th>date</th>\n",
    "    <th>name</th>\n",
    "    <th>year</th>\n",
    "    <th>cost</th>\n",
    "    <th>region</th>\n",
    "  </tr>\n",
    "  <tr>\n",
    "    <td>2020-01-01</td>\n",
    "    <td>Jenny</td>\n",
    "    <td>1998</td>\n",
    "    <td>1.2</td>\n",
    "    <td>South</td>\n",
    "  </tr>\n",
    "  <tr>\n",
    "    <td>2020-01-02</td>\n",
    "    <td>Alice</td>\n",
    "    <td>1992</td>\n",
    "    <td>?</td>\n",
    "    <td>East</td>\n",
    "  </tr>\n",
    "  <tr>\n",
    "    <td>2020-01-03</td>\n",
    "    <td>Tomas</td>\n",
    "    <td>1982</td>\n",
    "    <td>&</td>\n",
    "    <td>South</td>\n",
    "  </tr>\n",
    "</table>\n",
    "\"\"\"\n",
    "\n",
    "# Display the HTML representation\n",
    "display_html(html_string, raw=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>name</th>\n",
       "      <th>year</th>\n",
       "      <th>cost</th>\n",
       "      <th>region</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Jenny</td>\n",
       "      <td>1998</td>\n",
       "      <td>1.2</td>\n",
       "      <td>South</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020-01-02</td>\n",
       "      <td>Alice</td>\n",
       "      <td>1992</td>\n",
       "      <td>NaN</td>\n",
       "      <td>East</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2020-01-03</td>\n",
       "      <td>Tomas</td>\n",
       "      <td>1982</td>\n",
       "      <td>NaN</td>\n",
       "      <td>South</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         date   name  year  cost region\n",
       "0  2020-01-01  Jenny  1998   1.2  South\n",
       "1  2020-01-02  Alice  1992   NaN   East\n",
       "2  2020-01-03  Tomas  1982   NaN  South"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# other character representations for missing values\n",
    "dfs = pd.read_html(html_string, na_values=['?', '&'])\n",
    "dfs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf-tutorial",
   "language": "python",
   "name": "tf-tutorial"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}