causal_inference_did.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-06T01:55:18.016197Z",
     "start_time": "2021-03-06T01:55:17.062169Z"
    },
    "colab": {},
    "colab_type": "code",
    "id": "Vz5iXrbhuZft"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import statsmodels.api as sm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ZuVYTKfYNQZj"
   },
   "source": [
    "## Import Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-06T01:55:18.039189Z",
     "start_time": "2021-03-06T01:55:18.017190Z"
    },
    "colab": {},
    "colab_type": "code",
    "id": "6vCfX5URvcN7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(394, 15) (394, 8) (394, 7)\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('data/stock_data/data_sz50_cy_lhb_sen.csv', index_col=0)\n",
    "sz = pd.read_csv('data/stock_data/emd_涨跌幅_sz.csv', index_col=0)\n",
    "cy = pd.read_csv('data/stock_data/emd_涨跌幅_cy.csv', index_col=0)\n",
    "\n",
    "print(df.shape, sz.shape, cy.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-06T01:55:18.073190Z",
     "start_time": "2021-03-06T01:55:18.042192Z"
    },
    "colab": {},
    "colab_type": "code",
    "id": "fc-hr_JWy35d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(394, 30)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>收盘_sz</th>\n",
       "      <th>开盘_sz</th>\n",
       "      <th>高_sz</th>\n",
       "      <th>低_sz</th>\n",
       "      <th>交易量_sz</th>\n",
       "      <th>涨跌幅_sz</th>\n",
       "      <th>date</th>\n",
       "      <th>收盘_cy</th>\n",
       "      <th>开盘_cy</th>\n",
       "      <th>高_cy</th>\n",
       "      <th>...</th>\n",
       "      <th>emd_涨跌幅_sz_6</th>\n",
       "      <th>emd_涨跌幅_sz_7</th>\n",
       "      <th>emd_涨跌幅_sz_8</th>\n",
       "      <th>emd_涨跌幅_cy_1</th>\n",
       "      <th>emd_涨跌幅_cy_2</th>\n",
       "      <th>emd_涨跌幅_cy_3</th>\n",
       "      <th>emd_涨跌幅_cy_4</th>\n",
       "      <th>emd_涨跌幅_cy_5</th>\n",
       "      <th>emd_涨跌幅_cy_6</th>\n",
       "      <th>emd_涨跌幅_cy_7</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2,931.88</td>\n",
       "      <td>2,915.97</td>\n",
       "      <td>2,936.85</td>\n",
       "      <td>2,900.87</td>\n",
       "      <td>3.93B</td>\n",
       "      <td>2.09</td>\n",
       "      <td>20180222</td>\n",
       "      <td>1,677.76</td>\n",
       "      <td>1,661.56</td>\n",
       "      <td>1,679.79</td>\n",
       "      <td>...</td>\n",
       "      <td>0.938720</td>\n",
       "      <td>-0.934991</td>\n",
       "      <td>-0.236881</td>\n",
       "      <td>0.298323</td>\n",
       "      <td>-1.972296</td>\n",
       "      <td>1.849848</td>\n",
       "      <td>-1.205973</td>\n",
       "      <td>-1.245964</td>\n",
       "      <td>1.095513</td>\n",
       "      <td>-1.165342</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2,957.17</td>\n",
       "      <td>2,941.42</td>\n",
       "      <td>2,966.75</td>\n",
       "      <td>2,926.36</td>\n",
       "      <td>3.82B</td>\n",
       "      <td>0.86</td>\n",
       "      <td>20180223</td>\n",
       "      <td>1,668.83</td>\n",
       "      <td>1,674.62</td>\n",
       "      <td>1,681.24</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.756677</td>\n",
       "      <td>-0.332565</td>\n",
       "      <td>0.829638</td>\n",
       "      <td>1.369459</td>\n",
       "      <td>-0.260992</td>\n",
       "      <td>-1.732051</td>\n",
       "      <td>-1.172848</td>\n",
       "      <td>1.739993</td>\n",
       "      <td>0.059579</td>\n",
       "      <td>-1.034574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2,973.79</td>\n",
       "      <td>2,973.81</td>\n",
       "      <td>2,986.81</td>\n",
       "      <td>2,933.52</td>\n",
       "      <td>5.02B</td>\n",
       "      <td>0.56</td>\n",
       "      <td>20180226</td>\n",
       "      <td>1,729.15</td>\n",
       "      <td>1,683.15</td>\n",
       "      <td>1,733.71</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.486195</td>\n",
       "      <td>-0.431272</td>\n",
       "      <td>1.621853</td>\n",
       "      <td>-1.343849</td>\n",
       "      <td>1.204914</td>\n",
       "      <td>-1.036658</td>\n",
       "      <td>0.849230</td>\n",
       "      <td>0.795858</td>\n",
       "      <td>-1.067910</td>\n",
       "      <td>1.778090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2,927.10</td>\n",
       "      <td>2,979.77</td>\n",
       "      <td>2,979.77</td>\n",
       "      <td>2,918.01</td>\n",
       "      <td>4.62B</td>\n",
       "      <td>-1.57</td>\n",
       "      <td>20180227</td>\n",
       "      <td>1,743.54</td>\n",
       "      <td>1,729.94</td>\n",
       "      <td>1,757.37</td>\n",
       "      <td>...</td>\n",
       "      <td>0.169594</td>\n",
       "      <td>-0.487137</td>\n",
       "      <td>0.729410</td>\n",
       "      <td>-2.689865</td>\n",
       "      <td>3.331181</td>\n",
       "      <td>1.686321</td>\n",
       "      <td>-3.257950</td>\n",
       "      <td>-1.727923</td>\n",
       "      <td>2.332402</td>\n",
       "      <td>0.222133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2,878.67</td>\n",
       "      <td>2,899.00</td>\n",
       "      <td>2,905.57</td>\n",
       "      <td>2,866.46</td>\n",
       "      <td>3.88B</td>\n",
       "      <td>-1.65</td>\n",
       "      <td>20180228</td>\n",
       "      <td>1,753.63</td>\n",
       "      <td>1,730.15</td>\n",
       "      <td>1,773.73</td>\n",
       "      <td>...</td>\n",
       "      <td>1.116136</td>\n",
       "      <td>1.117061</td>\n",
       "      <td>-1.518490</td>\n",
       "      <td>-0.639171</td>\n",
       "      <td>-1.056843</td>\n",
       "      <td>0.640870</td>\n",
       "      <td>-0.245377</td>\n",
       "      <td>0.056866</td>\n",
       "      <td>-0.259212</td>\n",
       "      <td>0.280557</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      收盘_sz     开盘_sz      高_sz      低_sz 交易量_sz  涨跌幅_sz      date     收盘_cy  \\\n",
       "0  2,931.88  2,915.97  2,936.85  2,900.87  3.93B    2.09  20180222  1,677.76   \n",
       "1  2,957.17  2,941.42  2,966.75  2,926.36  3.82B    0.86  20180223  1,668.83   \n",
       "2  2,973.79  2,973.81  2,986.81  2,933.52  5.02B    0.56  20180226  1,729.15   \n",
       "3  2,927.10  2,979.77  2,979.77  2,918.01  4.62B   -1.57  20180227  1,743.54   \n",
       "4  2,878.67  2,899.00  2,905.57  2,866.46  3.88B   -1.65  20180228  1,753.63   \n",
       "\n",
       "      开盘_cy      高_cy  ... emd_涨跌幅_sz_6 emd_涨跌幅_sz_7  emd_涨跌幅_sz_8  \\\n",
       "0  1,661.56  1,679.79  ...     0.938720    -0.934991     -0.236881   \n",
       "1  1,674.62  1,681.24  ...    -0.756677    -0.332565      0.829638   \n",
       "2  1,683.15  1,733.71  ...    -1.486195    -0.431272      1.621853   \n",
       "3  1,729.94  1,757.37  ...     0.169594    -0.487137      0.729410   \n",
       "4  1,730.15  1,773.73  ...     1.116136     1.117061     -1.518490   \n",
       "\n",
       "   emd_涨跌幅_cy_1  emd_涨跌幅_cy_2  emd_涨跌幅_cy_3  emd_涨跌幅_cy_4  emd_涨跌幅_cy_5  \\\n",
       "0      0.298323     -1.972296      1.849848     -1.205973     -1.245964   \n",
       "1      1.369459     -0.260992     -1.732051     -1.172848      1.739993   \n",
       "2     -1.343849      1.204914     -1.036658      0.849230      0.795858   \n",
       "3     -2.689865      3.331181      1.686321     -3.257950     -1.727923   \n",
       "4     -0.639171     -1.056843      0.640870     -0.245377      0.056866   \n",
       "\n",
       "   emd_涨跌幅_cy_6  emd_涨跌幅_cy_7  \n",
       "0      1.095513     -1.165342  \n",
       "1      0.059579     -1.034574  \n",
       "2     -1.067910      1.778090  \n",
       "3      2.332402      0.222133  \n",
       "4     -0.259212      0.280557  \n",
       "\n",
       "[5 rows x 30 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Merge three dfs into one \n",
    "df = pd.concat([df, sz], axis=1, sort=True)\n",
    "df = pd.concat([df, cy], axis=1, sort=False)\n",
    "\n",
    "print(df.shape)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "aXUAJsUN1HIj"
   },
   "source": [
    "## Correlation Matrices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-06T01:55:18.088193Z",
     "start_time": "2021-03-06T01:55:18.075190Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 219
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 986,
     "status": "ok",
     "timestamp": 1573982864953,
     "user": {
      "displayName": "Soung Low",
      "photoUrl": "",
      "userId": "00570526448316647912"
     },
     "user_tz": -480
    },
    "id": "7Ze62pbL9yiO",
    "outputId": "b1a81ed9-3a6d-4ab0-b404-9d62d4eabbe1"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sen</th>\n",
       "      <th>涨跌幅_sz</th>\n",
       "      <th>涨跌幅_cy</th>\n",
       "      <th>emd_涨跌幅_sz_1</th>\n",
       "      <th>emd_涨跌幅_sz_4</th>\n",
       "      <th>emd_涨跌幅_cy_1</th>\n",
       "      <th>emd_涨跌幅_cy_4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>sen</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.371479</td>\n",
       "      <td>0.551547</td>\n",
       "      <td>0.023434</td>\n",
       "      <td>-0.103434</td>\n",
       "      <td>-0.003524</td>\n",
       "      <td>-0.074396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>涨跌幅_sz</th>\n",
       "      <td>0.371479</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.690440</td>\n",
       "      <td>-0.028648</td>\n",
       "      <td>-0.062449</td>\n",
       "      <td>0.089344</td>\n",
       "      <td>-0.029840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>涨跌幅_cy</th>\n",
       "      <td>0.551547</td>\n",
       "      <td>0.690440</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.048134</td>\n",
       "      <td>-0.154628</td>\n",
       "      <td>0.032343</td>\n",
       "      <td>-0.158057</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>emd_涨跌幅_sz_1</th>\n",
       "      <td>0.023434</td>\n",
       "      <td>-0.028648</td>\n",
       "      <td>-0.048134</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.122608</td>\n",
       "      <td>0.046431</td>\n",
       "      <td>0.058354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>emd_涨跌幅_sz_4</th>\n",
       "      <td>-0.103434</td>\n",
       "      <td>-0.062449</td>\n",
       "      <td>-0.154628</td>\n",
       "      <td>0.122608</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.013843</td>\n",
       "      <td>0.151957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>emd_涨跌幅_cy_1</th>\n",
       "      <td>-0.003524</td>\n",
       "      <td>0.089344</td>\n",
       "      <td>0.032343</td>\n",
       "      <td>0.046431</td>\n",
       "      <td>0.013843</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.184748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>emd_涨跌幅_cy_4</th>\n",
       "      <td>-0.074396</td>\n",
       "      <td>-0.029840</td>\n",
       "      <td>-0.158057</td>\n",
       "      <td>0.058354</td>\n",
       "      <td>0.151957</td>\n",
       "      <td>0.184748</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   sen    涨跌幅_sz    涨跌幅_cy  emd_涨跌幅_sz_1  emd_涨跌幅_sz_4  \\\n",
       "sen           1.000000  0.371479  0.551547      0.023434     -0.103434   \n",
       "涨跌幅_sz        0.371479  1.000000  0.690440     -0.028648     -0.062449   \n",
       "涨跌幅_cy        0.551547  0.690440  1.000000     -0.048134     -0.154628   \n",
       "emd_涨跌幅_sz_1  0.023434 -0.028648 -0.048134      1.000000      0.122608   \n",
       "emd_涨跌幅_sz_4 -0.103434 -0.062449 -0.154628      0.122608      1.000000   \n",
       "emd_涨跌幅_cy_1 -0.003524  0.089344  0.032343      0.046431      0.013843   \n",
       "emd_涨跌幅_cy_4 -0.074396 -0.029840 -0.158057      0.058354      0.151957   \n",
       "\n",
       "              emd_涨跌幅_cy_1  emd_涨跌幅_cy_4  \n",
       "sen              -0.003524     -0.074396  \n",
       "涨跌幅_sz            0.089344     -0.029840  \n",
       "涨跌幅_cy            0.032343     -0.158057  \n",
       "emd_涨跌幅_sz_1      0.046431      0.058354  \n",
       "emd_涨跌幅_sz_4      0.013843      0.151957  \n",
       "emd_涨跌幅_cy_1      1.000000      0.184748  \n",
       "emd_涨跌幅_cy_4      0.184748      1.000000  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Correlation between sz50 and cyb\n",
    "temp = df[['sen','涨跌幅_sz', '涨跌幅_cy', 'emd_涨跌幅_sz_1', 'emd_涨跌幅_sz_4', 'emd_涨跌幅_cy_1', 'emd_涨跌幅_cy_4']]\n",
    "temp.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-06T01:55:18.103191Z",
     "start_time": "2021-03-06T01:55:18.089191Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 317
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1017,
     "status": "ok",
     "timestamp": 1575463780254,
     "user": {
      "displayName": "Soung Low",
      "photoUrl": "",
      "userId": "00570526448316647912"
     },
     "user_tz": -480
    },
    "id": "CRlp3M4iZNRU",
    "outputId": "6f8e1f9f-4f5a-4f55-805a-777f60ab639a"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sen</th>\n",
       "      <th>涨跌幅_sz</th>\n",
       "      <th>emd_涨跌幅_sz_1</th>\n",
       "      <th>emd_涨跌幅_sz_2</th>\n",
       "      <th>emd_涨跌幅_sz_3</th>\n",
       "      <th>emd_涨跌幅_sz_4</th>\n",
       "      <th>emd_涨跌幅_sz_5</th>\n",
       "      <th>emd_涨跌幅_sz_6</th>\n",
       "      <th>emd_涨跌幅_sz_7</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>sen</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.371479</td>\n",
       "      <td>0.023434</td>\n",
       "      <td>-0.003965</td>\n",
       "      <td>-0.018929</td>\n",
       "      <td>-0.103434</td>\n",
       "      <td>0.052558</td>\n",
       "      <td>-0.055899</td>\n",
       "      <td>-0.038483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>涨跌幅_sz</th>\n",
       "      <td>0.371479</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.028648</td>\n",
       "      <td>-0.047204</td>\n",
       "      <td>-0.029874</td>\n",
       "      <td>-0.062449</td>\n",
       "      <td>-0.045887</td>\n",
       "      <td>-0.038680</td>\n",
       "      <td>0.033326</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>emd_涨跌幅_sz_1</th>\n",
       "      <td>0.023434</td>\n",
       "      <td>-0.028648</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.292885</td>\n",
       "      <td>-0.159487</td>\n",
       "      <td>0.122608</td>\n",
       "      <td>-0.121920</td>\n",
       "      <td>-0.018500</td>\n",
       "      <td>0.077992</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>emd_涨跌幅_sz_2</th>\n",
       "      <td>-0.003965</td>\n",
       "      <td>-0.047204</td>\n",
       "      <td>0.292885</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.151685</td>\n",
       "      <td>0.190395</td>\n",
       "      <td>-0.138947</td>\n",
       "      <td>0.046748</td>\n",
       "      <td>0.071578</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>emd_涨跌幅_sz_3</th>\n",
       "      <td>-0.018929</td>\n",
       "      <td>-0.029874</td>\n",
       "      <td>-0.159487</td>\n",
       "      <td>-0.151685</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.030062</td>\n",
       "      <td>0.255393</td>\n",
       "      <td>-0.036489</td>\n",
       "      <td>-0.170040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>emd_涨跌幅_sz_4</th>\n",
       "      <td>-0.103434</td>\n",
       "      <td>-0.062449</td>\n",
       "      <td>0.122608</td>\n",
       "      <td>0.190395</td>\n",
       "      <td>-0.030062</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.163747</td>\n",
       "      <td>0.008874</td>\n",
       "      <td>0.028218</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>emd_涨跌幅_sz_5</th>\n",
       "      <td>0.052558</td>\n",
       "      <td>-0.045887</td>\n",
       "      <td>-0.121920</td>\n",
       "      <td>-0.138947</td>\n",
       "      <td>0.255393</td>\n",
       "      <td>0.163747</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.207003</td>\n",
       "      <td>-0.072847</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>emd_涨跌幅_sz_6</th>\n",
       "      <td>-0.055899</td>\n",
       "      <td>-0.038680</td>\n",
       "      <td>-0.018500</td>\n",
       "      <td>0.046748</td>\n",
       "      <td>-0.036489</td>\n",
       "      <td>0.008874</td>\n",
       "      <td>0.207003</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.124637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>emd_涨跌幅_sz_7</th>\n",
       "      <td>-0.038483</td>\n",
       "      <td>0.033326</td>\n",
       "      <td>0.077992</td>\n",
       "      <td>0.071578</td>\n",
       "      <td>-0.170040</td>\n",
       "      <td>0.028218</td>\n",
       "      <td>-0.072847</td>\n",
       "      <td>0.124637</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   sen    涨跌幅_sz  emd_涨跌幅_sz_1  emd_涨跌幅_sz_2  emd_涨跌幅_sz_3  \\\n",
       "sen           1.000000  0.371479      0.023434     -0.003965     -0.018929   \n",
       "涨跌幅_sz        0.371479  1.000000     -0.028648     -0.047204     -0.029874   \n",
       "emd_涨跌幅_sz_1  0.023434 -0.028648      1.000000      0.292885     -0.159487   \n",
       "emd_涨跌幅_sz_2 -0.003965 -0.047204      0.292885      1.000000     -0.151685   \n",
       "emd_涨跌幅_sz_3 -0.018929 -0.029874     -0.159487     -0.151685      1.000000   \n",
       "emd_涨跌幅_sz_4 -0.103434 -0.062449      0.122608      0.190395     -0.030062   \n",
       "emd_涨跌幅_sz_5  0.052558 -0.045887     -0.121920     -0.138947      0.255393   \n",
       "emd_涨跌幅_sz_6 -0.055899 -0.038680     -0.018500      0.046748     -0.036489   \n",
       "emd_涨跌幅_sz_7 -0.038483  0.033326      0.077992      0.071578     -0.170040   \n",
       "\n",
       "              emd_涨跌幅_sz_4  emd_涨跌幅_sz_5  emd_涨跌幅_sz_6  emd_涨跌幅_sz_7  \n",
       "sen              -0.103434      0.052558     -0.055899     -0.038483  \n",
       "涨跌幅_sz           -0.062449     -0.045887     -0.038680      0.033326  \n",
       "emd_涨跌幅_sz_1      0.122608     -0.121920     -0.018500      0.077992  \n",
       "emd_涨跌幅_sz_2      0.190395     -0.138947      0.046748      0.071578  \n",
       "emd_涨跌幅_sz_3     -0.030062      0.255393     -0.036489     -0.170040  \n",
       "emd_涨跌幅_sz_4      1.000000      0.163747      0.008874      0.028218  \n",
       "emd_涨跌幅_sz_5      0.163747      1.000000      0.207003     -0.072847  \n",
       "emd_涨跌幅_sz_6      0.008874      0.207003      1.000000      0.124637  \n",
       "emd_涨跌幅_sz_7      0.028218     -0.072847      0.124637      1.000000  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Correlation within sz50\n",
    "temp = df[['sen','涨跌幅_sz', 'emd_涨跌幅_sz_1', 'emd_涨跌幅_sz_2', 'emd_涨跌幅_sz_3', \n",
    "           'emd_涨跌幅_sz_4', 'emd_涨跌幅_sz_5', 'emd_涨跌幅_sz_6', 'emd_涨跌幅_sz_7' ]]\n",
    "temp.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-06T01:55:18.116191Z",
     "start_time": "2021-03-06T01:55:18.104190Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 317
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1228,
     "status": "ok",
     "timestamp": 1575463219329,
     "user": {
      "displayName": "Soung Low",
      "photoUrl": "",
      "userId": "00570526448316647912"
     },
     "user_tz": -480
    },
    "id": "os0eh5MF6tID",
    "outputId": "f1870170-ffd3-44da-9212-db0bf67f9b2a"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sen</th>\n",
       "      <th>涨跌幅_cy</th>\n",
       "      <th>emd_涨跌幅_cy_1</th>\n",
       "      <th>emd_涨跌幅_cy_2</th>\n",
       "      <th>emd_涨跌幅_cy_3</th>\n",
       "      <th>emd_涨跌幅_cy_4</th>\n",
       "      <th>emd_涨跌幅_cy_5</th>\n",
       "      <th>emd_涨跌幅_cy_6</th>\n",
       "      <th>emd_涨跌幅_cy_7</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>sen</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.551547</td>\n",
       "      <td>-0.003524</td>\n",
       "      <td>-0.037167</td>\n",
       "      <td>-0.037193</td>\n",
       "      <td>-0.074396</td>\n",
       "      <td>-0.065983</td>\n",
       "      <td>-0.054250</td>\n",
       "      <td>0.002418</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>涨跌幅_cy</th>\n",
       "      <td>0.551547</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.032343</td>\n",
       "      <td>0.001104</td>\n",
       "      <td>0.036410</td>\n",
       "      <td>-0.158057</td>\n",
       "      <td>0.024804</td>\n",
       "      <td>-0.110379</td>\n",
       "      <td>-0.037275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>emd_涨跌幅_cy_1</th>\n",
       "      <td>-0.003524</td>\n",
       "      <td>0.032343</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.040308</td>\n",
       "      <td>0.193766</td>\n",
       "      <td>0.184748</td>\n",
       "      <td>0.137725</td>\n",
       "      <td>-0.097066</td>\n",
       "      <td>-0.136090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>emd_涨跌幅_cy_2</th>\n",
       "      <td>-0.037167</td>\n",
       "      <td>0.001104</td>\n",
       "      <td>0.040308</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.054846</td>\n",
       "      <td>0.007879</td>\n",
       "      <td>0.038990</td>\n",
       "      <td>0.090079</td>\n",
       "      <td>0.034002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>emd_涨跌幅_cy_3</th>\n",
       "      <td>-0.037193</td>\n",
       "      <td>0.036410</td>\n",
       "      <td>0.193766</td>\n",
       "      <td>0.054846</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.192552</td>\n",
       "      <td>0.118606</td>\n",
       "      <td>0.021209</td>\n",
       "      <td>-0.105766</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>emd_涨跌幅_cy_4</th>\n",
       "      <td>-0.074396</td>\n",
       "      <td>-0.158057</td>\n",
       "      <td>0.184748</td>\n",
       "      <td>0.007879</td>\n",
       "      <td>0.192552</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.121116</td>\n",
       "      <td>0.183533</td>\n",
       "      <td>0.172234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>emd_涨跌幅_cy_5</th>\n",
       "      <td>-0.065983</td>\n",
       "      <td>0.024804</td>\n",
       "      <td>0.137725</td>\n",
       "      <td>0.038990</td>\n",
       "      <td>0.118606</td>\n",
       "      <td>0.121116</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.264336</td>\n",
       "      <td>0.058701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>emd_涨跌幅_cy_6</th>\n",
       "      <td>-0.054250</td>\n",
       "      <td>-0.110379</td>\n",
       "      <td>-0.097066</td>\n",
       "      <td>0.090079</td>\n",
       "      <td>0.021209</td>\n",
       "      <td>0.183533</td>\n",
       "      <td>0.264336</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.314011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>emd_涨跌幅_cy_7</th>\n",
       "      <td>0.002418</td>\n",
       "      <td>-0.037275</td>\n",
       "      <td>-0.136090</td>\n",
       "      <td>0.034002</td>\n",
       "      <td>-0.105766</td>\n",
       "      <td>0.172234</td>\n",
       "      <td>0.058701</td>\n",
       "      <td>0.314011</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   sen    涨跌幅_cy  emd_涨跌幅_cy_1  emd_涨跌幅_cy_2  emd_涨跌幅_cy_3  \\\n",
       "sen           1.000000  0.551547     -0.003524     -0.037167     -0.037193   \n",
       "涨跌幅_cy        0.551547  1.000000      0.032343      0.001104      0.036410   \n",
       "emd_涨跌幅_cy_1 -0.003524  0.032343      1.000000      0.040308      0.193766   \n",
       "emd_涨跌幅_cy_2 -0.037167  0.001104      0.040308      1.000000      0.054846   \n",
       "emd_涨跌幅_cy_3 -0.037193  0.036410      0.193766      0.054846      1.000000   \n",
       "emd_涨跌幅_cy_4 -0.074396 -0.158057      0.184748      0.007879      0.192552   \n",
       "emd_涨跌幅_cy_5 -0.065983  0.024804      0.137725      0.038990      0.118606   \n",
       "emd_涨跌幅_cy_6 -0.054250 -0.110379     -0.097066      0.090079      0.021209   \n",
       "emd_涨跌幅_cy_7  0.002418 -0.037275     -0.136090      0.034002     -0.105766   \n",
       "\n",
       "              emd_涨跌幅_cy_4  emd_涨跌幅_cy_5  emd_涨跌幅_cy_6  emd_涨跌幅_cy_7  \n",
       "sen              -0.074396     -0.065983     -0.054250      0.002418  \n",
       "涨跌幅_cy           -0.158057      0.024804     -0.110379     -0.037275  \n",
       "emd_涨跌幅_cy_1      0.184748      0.137725     -0.097066     -0.136090  \n",
       "emd_涨跌幅_cy_2      0.007879      0.038990      0.090079      0.034002  \n",
       "emd_涨跌幅_cy_3      0.192552      0.118606      0.021209     -0.105766  \n",
       "emd_涨跌幅_cy_4      1.000000      0.121116      0.183533      0.172234  \n",
       "emd_涨跌幅_cy_5      0.121116      1.000000      0.264336      0.058701  \n",
       "emd_涨跌幅_cy_6      0.183533      0.264336      1.000000      0.314011  \n",
       "emd_涨跌幅_cy_7      0.172234      0.058701      0.314011      1.000000  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Correlation within cy\n",
    "temp = df[['sen','涨跌幅_cy', 'emd_涨跌幅_cy_1', 'emd_涨跌幅_cy_2', 'emd_涨跌幅_cy_3', \n",
    "           'emd_涨跌幅_cy_4', 'emd_涨跌幅_cy_5', 'emd_涨跌幅_cy_6', 'emd_涨跌幅_cy_7' ]]\n",
    "temp.corr()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "4nWQnDSpNyFl"
   },
   "source": [
    "## DID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-06T01:55:18.128190Z",
     "start_time": "2021-03-06T01:55:18.117191Z"
    },
    "colab": {},
    "colab_type": "code",
    "id": "_G7M3iuTNwjB"
   },
   "outputs": [],
   "source": [
    "# Import date data \n",
    "# t_0: dates in which the sentiment index rises 1sd below than the last day\n",
    "# t_1: dates in which the sentiment index drops 1sd above than the last day\n",
    "\n",
    "day_t_0 = pd.read_csv('data/day_t_0.csv', index_col=0)\n",
    "day_t_1 = pd.read_csv('data/day_t_1.csv', index_col=0)\n",
    "\n",
    "day_t_0 = pd.DataFrame(day_t_0['0'].str.lstrip(\"['\"))\n",
    "day_t_0 = pd.DataFrame(day_t_0['0'].str.rstrip(\"']\"))\n",
    "day_t_0 = pd.DataFrame(set(day_t_0['0'])) # to omit replicates\n",
    "\n",
    "day_t_1 = pd.DataFrame(day_t_1['0'].str.lstrip(\"['\"))\n",
    "day_t_1 = pd.DataFrame(day_t_1['0'].str.rstrip(\"']\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-06T01:55:18.140191Z",
     "start_time": "2021-03-06T01:55:18.130192Z"
    },
    "colab": {},
    "colab_type": "code",
    "id": "GbcDQe0mVYj1"
   },
   "outputs": [],
   "source": [
    "t_0 = []\n",
    "for i in df['date']:\n",
    "    t_0.append(str(i) in day_t_1['0'].values)\n",
    "    \n",
    "df['t'] = t_0\n",
    "df['t'] = df['t'].eq(True).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-06T01:55:18.149192Z",
     "start_time": "2021-03-06T01:55:18.142191Z"
    },
    "colab": {},
    "colab_type": "code",
    "id": "bBOVT-RClpR6"
   },
   "outputs": [],
   "source": [
    "def did(y1_d1, y0_d0, y):\n",
    "    '''Run Difference in difference method using Linear Regression\n",
    "    \n",
    "    Arguments:\n",
    "        y1_d1: treatment group \n",
    "        y0_d0: control group \n",
    "        y: dependent variable\n",
    "        \n",
    "    Return:\n",
    "        Regression result\n",
    "    '''\n",
    "    d0 = df.loc[:, [str(y0_d0), 't']]\n",
    "    d1 = df.loc[:, [str(y1_d1), 't']]\n",
    "    \n",
    "    d0.loc[:,'d'] = 0\n",
    "    d1.loc[:,'d'] = 1\n",
    "    \n",
    "    d0.columns = [str(y), 't', 'd']\n",
    "    d1.columns = [str(y), 't', 'd']\n",
    "\n",
    "    df_reg = d0.append(d1)\n",
    "    df_reg.reset_index(inplace=True, drop=True)\n",
    "    df_reg['t*d'] = df_reg['t']*df_reg['d']\n",
    "\n",
    "    x = df_reg[['t', 'd', 't*d']]\n",
    "    x = sm.add_constant(x)\n",
    "    y = df_reg[str(y)]\n",
    "    model = sm.OLS(y,x).fit()\n",
    "    \n",
    "    return model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-06T01:55:18.178190Z",
     "start_time": "2021-03-06T01:55:18.150191Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 745
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2006,
     "status": "ok",
     "timestamp": 1574156816483,
     "user": {
      "displayName": "Soung Low",
      "photoUrl": "",
      "userId": "00570526448316647912"
     },
     "user_tz": -480
    },
    "id": "t6vUyWn5nd4Z",
    "outputId": "552beff2-0b0b-4e00-a8cb-c87650169d09"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<caption>OLS Regression Results</caption>\n",
       "<tr>\n",
       "  <th>Dep. Variable:</th>        <td>漲跌幅_上證_創業</td>    <th>  R-squared:         </th> <td>   0.098</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Model:</th>                   <td>OLS</td>       <th>  Adj. R-squared:    </th> <td>   0.095</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Method:</th>             <td>Least Squares</td>  <th>  F-statistic:       </th> <td>   28.52</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Date:</th>             <td>Sat, 06 Mar 2021</td> <th>  Prob (F-statistic):</th> <td>1.66e-17</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Time:</th>                 <td>01:55:18</td>     <th>  Log-Likelihood:    </th> <td> -1440.5</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>No. Observations:</th>      <td>   788</td>      <th>  AIC:               </th> <td>   2889.</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Residuals:</th>          <td>   784</td>      <th>  BIC:               </th> <td>   2908.</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Model:</th>              <td>     3</td>      <th>                     </th>     <td> </td>   \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Covariance Type:</th>      <td>nonrobust</td>    <th>                     </th>     <td> </td>   \n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "    <td></td>       <th>coef</th>     <th>std err</th>      <th>t</th>      <th>P>|t|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>const</th> <td>   -0.1520</td> <td>    0.088</td> <td>   -1.721</td> <td> 0.086</td> <td>   -0.325</td> <td>    0.021</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>t</th>     <td>    0.6313</td> <td>    0.174</td> <td>    3.637</td> <td> 0.000</td> <td>    0.291</td> <td>    0.972</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>d</th>     <td>   -0.2173</td> <td>    0.125</td> <td>   -1.739</td> <td> 0.082</td> <td>   -0.462</td> <td>    0.028</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>t*d</th>   <td>    0.8449</td> <td>    0.245</td> <td>    3.442</td> <td> 0.001</td> <td>    0.363</td> <td>    1.327</td>\n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "  <th>Omnibus:</th>       <td>36.804</td> <th>  Durbin-Watson:     </th> <td>   1.928</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Prob(Omnibus):</th> <td> 0.000</td> <th>  Jarque-Bera (JB):  </th> <td> 113.039</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Skew:</th>          <td> 0.030</td> <th>  Prob(JB):          </th> <td>2.84e-25</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Kurtosis:</th>      <td> 4.854</td> <th>  Cond. No.          </th> <td>    6.46</td>\n",
       "</tr>\n",
       "</table><br/><br/>Warnings:<br/>[1] Standard Errors assume that the covariance matrix of the errors is correctly specified."
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary.Summary'>\n",
       "\"\"\"\n",
       "                            OLS Regression Results                            \n",
       "==============================================================================\n",
       "Dep. Variable:              漲跌幅_上證_創業   R-squared:                       0.098\n",
       "Model:                            OLS   Adj. R-squared:                  0.095\n",
       "Method:                 Least Squares   F-statistic:                     28.52\n",
       "Date:                Sat, 06 Mar 2021   Prob (F-statistic):           1.66e-17\n",
       "Time:                        01:55:18   Log-Likelihood:                -1440.5\n",
       "No. Observations:                 788   AIC:                             2889.\n",
       "Df Residuals:                     784   BIC:                             2908.\n",
       "Df Model:                           3                                         \n",
       "Covariance Type:            nonrobust                                         \n",
       "==============================================================================\n",
       "                 coef    std err          t      P>|t|      [0.025      0.975]\n",
       "------------------------------------------------------------------------------\n",
       "const         -0.1520      0.088     -1.721      0.086      -0.325       0.021\n",
       "t              0.6313      0.174      3.637      0.000       0.291       0.972\n",
       "d             -0.2173      0.125     -1.739      0.082      -0.462       0.028\n",
       "t*d            0.8449      0.245      3.442      0.001       0.363       1.327\n",
       "==============================================================================\n",
       "Omnibus:                       36.804   Durbin-Watson:                   1.928\n",
       "Prob(Omnibus):                  0.000   Jarque-Bera (JB):              113.039\n",
       "Skew:                           0.030   Prob(JB):                     2.84e-25\n",
       "Kurtosis:                       4.854   Cond. No.                         6.46\n",
       "==============================================================================\n",
       "\n",
       "Warnings:\n",
       "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
       "\"\"\""
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "did('涨跌幅_cy', '涨跌幅_sz', '漲跌幅_上證_創業')"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "fyp_causal inference.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}