Quant trading test models - Kalman Filter and Halflife.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "##### import the necessary modules and set chart style####\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib as mpl\n",
    "mpl.style.use('bmh')\n",
    "import pandas_datareader.data as web\n",
    "import matplotlib.pylab as plt\n",
    "from datetime import datetime\n",
    "import statsmodels.api as sm\n",
    "from pykalman import KalmanFilter\n",
    "from math import sqrt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Symbol                                               Name  \\\n",
      "0    AAPL                                         Apple Inc.   \n",
      "1    MSFT                              Microsoft Corporation   \n",
      "2    GOOG                                      Alphabet Inc.   \n",
      "3   GOOGL                                      Alphabet Inc.   \n",
      "4      FB                                     Facebook, Inc.   \n",
      "5     TSM  Taiwan Semiconductor Manufacturing Company Lim...   \n",
      "6    INTC                                  Intel Corporation   \n",
      "7    CSCO                                Cisco Systems, Inc.   \n",
      "8    ORCL                                 Oracle Corporation   \n",
      "9     SAP                                             SAP SE   \n",
      "10   ADBE                                         Adobe Inc.   \n",
      "11    CRM                               salesforce.com, inc.   \n",
      "12    TXN                     Texas Instruments Incorporated   \n",
      "13   NVDA                                 NVIDIA Corporation   \n",
      "14    IBM        International Business Machines Corporation   \n",
      "15    ACN                                      Accenture plc   \n",
      "16   AVGO                                      Broadcom Inc.   \n",
      "17   ASML                                  ASML Holding N.V.   \n",
      "18   QCOM                              QUALCOMM Incorporated   \n",
      "19    SNE                                   Sony Corporation   \n",
      "20   INTU                                        Intuit Inc.   \n",
      "21    VMW                                       VMware, Inc.   \n",
      "22   UBER                            Uber Technologies, Inc.   \n",
      "23     MU                            Micron Technology, Inc.   \n",
      "24   AMAT                            Applied Materials, Inc.   \n",
      "25    LHX                        L3Harris Technologies, Inc.   \n",
      "26    NOW                                   ServiceNow, Inc.   \n",
      "27   ATVI                          Activision Blizzard, Inc.   \n",
      "28    ADI                               Analog Devices, Inc.   \n",
      "29   INFY                                    Infosys Limited   \n",
      "..    ...                                                ...   \n",
      "70    IAC                                IAC/InterActiveCorp   \n",
      "71   ANET                              Arista Networks, Inc.   \n",
      "72   ANSS                                        ANSYS, Inc.   \n",
      "73    CDW                                    CDW Corporation   \n",
      "74   CDNS                       Cadence Design Systems, Inc.   \n",
      "75    WDC                        Western Digital Corporation   \n",
      "76   SPLK                                        Splunk Inc.   \n",
      "77   GRMN                                        Garmin Ltd.   \n",
      "78   CHKP             Check Point Software Technologies Ltd.   \n",
      "79   MRVL                      Marvell Technology Group Ltd.   \n",
      "80   MXIM                    Maxim Integrated Products, Inc.   \n",
      "81   SWKS                           Skyworks Solutions, Inc.   \n",
      "82    STX                             Seagate Technology plc   \n",
      "83   AKAM                          Akamai Technologies, Inc.   \n",
      "84   SYMC                               Symantec Corporation   \n",
      "85   PINS                                    Pinterest, Inc.   \n",
      "86   TTWO                Take-Two Interactive Software, Inc.   \n",
      "87   TWLO                                        Twilio Inc.   \n",
      "88   CTXS                               Citrix Systems, Inc.   \n",
      "89   FTNT                                     Fortinet, Inc.   \n",
      "90   NTAP                                       NetApp, Inc.   \n",
      "91     IT                                      Gartner, Inc.   \n",
      "92   LYFT                                         Lyft, Inc.   \n",
      "93    RNG                                  RingCentral, Inc.   \n",
      "94   SSNC                   SS&C Technologies Holdings, Inc.   \n",
      "95     IQ                                        iQIYI, Inc.   \n",
      "96   OKTA                                         Okta, Inc.   \n",
      "97     SE                                        Sea Limited   \n",
      "98   PAGS                             PagSeguro Digital Ltd.   \n",
      "99   WORK                           Slack Technologies, Inc.   \n",
      "\n",
      "    Price (Intraday)  Change % Change   Volume Avg Vol (3 month) Market Cap  \\\n",
      "0             239.96   -0.55   -0.23%  22.684M           28.328M     1.084T   \n",
      "1             136.37   -2.06   -1.49%  27.935M           24.046M     1.034T   \n",
      "2            1242.80   -3.35   -0.27%   1.048M            1.371M   861.202B   \n",
      "3            1241.20   -3.08   -0.25%   1.027M            1.379M   861.202B   \n",
      "4             182.34   -7.42   -3.91%   18.18M           13.998M   520.207B   \n",
      "5              50.64    0.64   +1.28%  10.603M            6.991M    248.59B   \n",
      "6              52.01   -0.12   -0.23%  13.347M           20.609M   229.874B   \n",
      "7              47.55    0.16   +0.34%  12.327M            19.71M   202.545B   \n",
      "8              54.11   -1.02   -1.85%   7.652M           12.547M   177.617B   \n",
      "9             131.80   -0.91   -0.69%   1.088M            724235   163.698B   \n",
      "10            261.09   -5.75   -2.15%   3.413M            2.677M   126.388B   \n",
      "11            142.33   -2.22   -1.54%   4.059M            6.131M   122.463B   \n",
      "12            128.57   -2.36   -1.80%   5.898M            4.621M   120.036B   \n",
      "13            195.61   -0.40   -0.20%   8.434M             9.18M   116.845B   \n",
      "14            133.96    1.38   +1.04%   3.716M            3.827M   118.672B   \n",
      "15            182.72   -4.62   -2.47%   1.721M            1.912M   116.374B   \n",
      "16            287.46   -2.11   -0.73%   1.344M            2.114M   114.027B   \n",
      "17            255.29   -5.14   -1.97%   828422            637730   107.267B   \n",
      "18             78.47   -0.51   -0.65%   4.204M            8.139M    95.393B   \n",
      "19             58.97   -0.21   -0.35%   643305            1.041M    73.174B   \n",
      "20            253.33   -8.87   -3.38%    1.32M            1.252M    65.885B   \n",
      "21            153.02    0.66   +0.43%   600444            1.566M    62.529B   \n",
      "22             32.53    1.12   +3.57%   19.18M            9.865M    55.301B   \n",
      "23             44.66   -0.56   -1.24%  14.008M           22.948M    49.296B   \n",
      "24             51.98   -0.66   -1.25%   4.484M             7.22M    47.415B   \n",
      "25            200.67    1.01   +0.51%   1.416M            1.417M    44.664B   \n",
      "26            228.34  -12.56   -5.21%   6.512M            1.955M    42.814B   \n",
      "27             54.52   -0.99   -1.78%   4.094M            7.424M    41.818B   \n",
      "28            110.88   -0.23   -0.21%   1.664M            2.122M     40.96B   \n",
      "29              9.01   -0.17   -1.82%  47.571M           13.837M    37.243B   \n",
      "..               ...     ...      ...      ...               ...        ...   \n",
      "70            222.01   -3.38   -1.50%   621174            846467    18.715B   \n",
      "71            241.50   -2.34   -0.96%   441576            714712     18.51B   \n",
      "72            214.32   -4.46   -2.04%   523888            466840    18.024B   \n",
      "73            122.41   -1.65   -1.33%   760621            1.439M    17.479B   \n",
      "74             62.94   -3.81   -5.71%   3.608M            1.833M    17.661B   \n",
      "75             58.86    0.26   +0.44%   3.129M            4.251M    17.497B   \n",
      "76            110.46   -0.67   -0.60%   1.434M            1.965M    17.048B   \n",
      "77             87.42    0.36   +0.41%   478954            1.051M    16.619B   \n",
      "78            106.64   -0.75   -0.70%   597357            955221    16.454B   \n",
      "79             24.10    0.08   +0.33%   7.241M            7.933M    16.075B   \n",
      "80             58.61    0.09   +0.15%   991513            1.636M     15.89B   \n",
      "81             88.96   -0.93   -1.03%   1.222M            1.855M    15.275B   \n",
      "82             55.86    0.38   +0.68%   1.445M            2.837M    14.755B   \n",
      "83             89.09   -2.80   -3.05%   1.606M            1.607M    14.599B   \n",
      "84             23.15    0.07   +0.30%   2.007M            6.294M    14.307B   \n",
      "85             25.88   -0.88   -3.29%   8.544M            6.255M    14.045B   \n",
      "86            121.99   -1.11   -0.90%   1.063M            1.469M    13.807B   \n",
      "87            100.65   -4.89   -4.63%    5.97M            3.906M    13.677B   \n",
      "88            104.10    0.91   +0.88%   1.381M            1.983M    13.627B   \n",
      "89             77.04    0.76   +1.00%   737117             1.35M    13.173B   \n",
      "90             55.18    0.61   +1.12%   1.376M            2.781M    13.195B   \n",
      "91            142.98   -2.09   -1.44%   386693            546398    12.888B   \n",
      "92             43.56    2.68   +6.56%   19.56M            6.634M    12.755B   \n",
      "93            153.40   -7.10   -4.42%   931027            1.083M    12.728B   \n",
      "94             48.68   -1.33   -2.66%   1.769M            2.226M    12.353B   \n",
      "95             16.50   -0.03   -0.18%   2.905M            6.112M    11.964B   \n",
      "96             99.29   -1.87   -1.85%   3.106M            2.709M     11.89B   \n",
      "97             26.70   -0.53   -1.95%   3.657M            3.761M    11.881B   \n",
      "98             35.61   -3.34   -8.58%   5.087M              2.3M     11.71B   \n",
      "99             21.03   -1.55   -6.86%  11.238M            7.908M    11.442B   \n",
      "\n",
      "    PE Ratio (TTM)  52 Week Range  \n",
      "0            20.38            NaN  \n",
      "1            26.95            NaN  \n",
      "2            25.09            NaN  \n",
      "3            25.06            NaN  \n",
      "4            30.84            NaN  \n",
      "5            22.71            NaN  \n",
      "6            12.08            NaN  \n",
      "7            18.22            NaN  \n",
      "8            17.71            NaN  \n",
      "9            31.84            NaN  \n",
      "10           46.46            NaN  \n",
      "11          118.21            NaN  \n",
      "12           23.47            NaN  \n",
      "13           44.11            NaN  \n",
      "14           14.07            NaN  \n",
      "15           24.83            NaN  \n",
      "16           40.45            NaN  \n",
      "17           36.68            NaN  \n",
      "18           28.72            NaN  \n",
      "19           12.40            NaN  \n",
      "20           43.01            NaN  \n",
      "21           10.18            NaN  \n",
      "22             NaN            NaN  \n",
      "23            8.11            NaN  \n",
      "24           16.44            NaN  \n",
      "25           25.54            NaN  \n",
      "26        15222.67            NaN  \n",
      "27           24.89            NaN  \n",
      "28           28.14            NaN  \n",
      "29           17.67            NaN  \n",
      "..             ...            ...  \n",
      "70           39.32            NaN  \n",
      "71           27.97            NaN  \n",
      "72           41.92            NaN  \n",
      "73           26.61            NaN  \n",
      "74           41.41            NaN  \n",
      "75             NaN            NaN  \n",
      "76             NaN            NaN  \n",
      "77           22.54            NaN  \n",
      "78           20.84            NaN  \n",
      "79             NaN            NaN  \n",
      "80           19.73            NaN  \n",
      "81           16.93            NaN  \n",
      "82            7.91            NaN  \n",
      "83           34.92            NaN  \n",
      "84          138.62            NaN  \n",
      "85             NaN            NaN  \n",
      "86           45.27            NaN  \n",
      "87             NaN            NaN  \n",
      "88           27.98            NaN  \n",
      "89           36.27            NaN  \n",
      "90           14.11            NaN  \n",
      "91           59.57            NaN  \n",
      "92             NaN            NaN  \n",
      "93             NaN            NaN  \n",
      "94           40.70            NaN  \n",
      "95           15.57            NaN  \n",
      "96             NaN            NaN  \n",
      "97             NaN            NaN  \n",
      "98           51.16            NaN  \n",
      "99             NaN            NaN  \n",
      "\n",
      "[100 rows x 10 columns]\n"
     ]
    }
   ],
   "source": [
    "stock_list = pd.read_html(\"https://finance.yahoo.com/screener/predefined/ms_technology?offset=0&count=100\")[0]\n",
    "print(stock_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     AAPL\n",
       "1     MSFT\n",
       "2     GOOG\n",
       "3    GOOGL\n",
       "Name: Symbol, dtype: object"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stocks = pd.DataFrame(stock_list)\n",
    "stocks[\"Symbol\"]\n",
    "df_list = stocks[\"Symbol\"]\n",
    "df = df_list.astype(str)\n",
    "df[:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "#scrape html from website and store 3rd DataFrame as our stock tickers - this is dictated to us by the structure of the html\n",
    "#stock_list = pd.read_html(\"https://www.marketwatch.com/tools/industry/stocklist.asp?bcind_ind=9535&bcind_period=3mo\")[0]\n",
    "#convert the DataFrame of stocks into a list so we can easily iterate over it\n",
    "#stocks = stock_list[1].dropna()[1:].tolist()\n",
    "#set empty list o hold the stock price DataFrames that we can later concatenate into a master frame\n",
    "#df_list = []\n",
    "#not all stocks will return data so set up an empty list to store the stock tickers that actually successfully returns data\n",
    "#used_stocks = []\n",
    "#iterate over stock tickers in list and download relevant data, storing said data and successfully downloaded tickers along the way\n",
    "#for stock in stocks:\n",
    "  #  try:\n",
    "  #      data = pd.DataFrame(web.DataReader(stock,data_source='iex',start='01/01/2017')['close'])\n",
    "  #      data.columns = [stock]\n",
    "  #      df_list.append(data)\n",
    "  #      used_stocks.append(stock)\n",
    "  #  except:\n",
    "  #      pass\n",
    "#concatenate list of individual tciker price DataFrames into one master DataFrame\n",
    "#df = pd.concat(df_list,axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing data for stock: AAPL\n",
      "Processing data for stock: MSFT\n",
      "Processing data for stock: GOOG\n",
      "Processing data for stock: GOOGL\n",
      "Processing data for stock: FB\n",
      "Processing data for stock: TSM\n",
      "Processing data for stock: INTC\n",
      "Processing data for stock: CSCO\n",
      "Processing data for stock: ORCL\n",
      "Processing data for stock: SAP\n",
      "Processing data for stock: ADBE\n",
      "Processing data for stock: CRM\n",
      "Processing data for stock: TXN\n",
      "Processing data for stock: NVDA\n",
      "Processing data for stock: IBM\n",
      "Processing data for stock: ACN\n",
      "Processing data for stock: AVGO\n",
      "Processing data for stock: ASML\n",
      "Processing data for stock: QCOM\n",
      "Processing data for stock: SNE\n",
      "Processing data for stock: INTU\n",
      "Processing data for stock: VMW\n",
      "Processing data for stock: UBER\n",
      "Processing data for stock: MU\n",
      "Processing data for stock: AMAT\n",
      "Processing data for stock: LHX\n",
      "Processing data for stock: NOW\n",
      "Processing data for stock: ATVI\n",
      "Processing data for stock: ADI\n",
      "Processing data for stock: INFY\n",
      "Processing data for stock: DELL\n",
      "Processing data for stock: BIDU\n",
      "Processing data for stock: NTES\n",
      "Processing data for stock: WDAY\n",
      "Processing data for stock: AMD\n",
      "Processing data for stock: SHOP\n",
      "Processing data for stock: LRCX\n",
      "Processing data for stock: CTSH\n",
      "Processing data for stock: TEL\n",
      "Processing data for stock: ADSK\n",
      "Processing data for stock: NXPI\n",
      "Processing data for stock: TWTR\n",
      "Processing data for stock: ERIC\n",
      "Processing data for stock: APH\n",
      "Processing data for stock: NOK\n",
      "Processing data for stock: MSI\n",
      "Processing data for stock: EA\n",
      "Processing data for stock: KLAC\n",
      "Processing data for stock: HPQ\n",
      "Processing data for stock: SQ\n",
      "Processing data for stock: TEAM\n",
      "Processing data for stock: FTV-PA\n",
      "Processing data for stock: FLT\n",
      "Processing data for stock: FTV\n",
      "Processing data for stock: XLNX\n",
      "Processing data for stock: GLW\n",
      "Processing data for stock: MCHP\n",
      "Processing data for stock: TME\n",
      "Processing data for stock: VRSN\n",
      "Processing data for stock: CERN\n",
      "Processing data for stock: SPOT\n",
      "Processing data for stock: GIB\n",
      "Processing data for stock: VEEV\n",
      "Processing data for stock: HPE\n",
      "Processing data for stock: PANW\n",
      "Processing data for stock: SNPS\n",
      "Processing data for stock: SNAP\n",
      "Processing data for stock: KEYS\n",
      "Processing data for stock: STM\n",
      "Processing data for stock: MTCH\n",
      "Processing data for stock: IAC\n",
      "Processing data for stock: ANET\n",
      "Processing data for stock: ANSS\n",
      "Processing data for stock: CDW\n",
      "Processing data for stock: CDNS\n",
      "Processing data for stock: WDC\n",
      "Processing data for stock: SPLK\n",
      "Processing data for stock: GRMN\n",
      "Processing data for stock: CHKP\n",
      "Processing data for stock: MRVL\n",
      "Processing data for stock: MXIM\n",
      "Processing data for stock: SWKS\n",
      "Processing data for stock: STX\n",
      "Processing data for stock: AKAM\n",
      "Processing data for stock: SYMC\n",
      "Processing data for stock: PINS\n",
      "Processing data for stock: TTWO\n",
      "Processing data for stock: TWLO\n",
      "Processing data for stock: CTXS\n",
      "Processing data for stock: FTNT\n",
      "Processing data for stock: NTAP\n",
      "Processing data for stock: IT\n",
      "Processing data for stock: LYFT\n",
      "Processing data for stock: RNG\n",
      "Processing data for stock: SSNC\n",
      "Processing data for stock: IQ\n",
      "Processing data for stock: OKTA\n",
      "Processing data for stock: SE\n",
      "Processing data for stock: PAGS\n",
      "Processing data for stock: WORK\n",
      "10% done...\n",
      "20% done...\n",
      "30% done...\n",
      "40% done...\n",
      "50% done...\n",
      "60% done...\n",
      "70% done...\n",
      "80% done...\n",
      "90% done...\n"
     ]
    }
   ],
   "source": [
    "from backtester.dataSource.yahoo_data_source import YahooStockDataSource\n",
    "from datetime import datetime\n",
    "\n",
    "startDateStr = '2018/01/01'\n",
    "endDateStr = '2019/10/21'\n",
    "cachedFolderName = 'yahooData/'\n",
    "dataSetId = 'testPairsTrading'\n",
    "instrumentIds = df\n",
    "ds = YahooStockDataSource(cachedFolderName=cachedFolderName,\n",
    "                            dataSetId=dataSetId,\n",
    "                            instrumentIds=instrumentIds,\n",
    "                            startDateStr=startDateStr,\n",
    "                            endDateStr=endDateStr,\n",
    "                            event='history')\n",
    "data = ds.getBookDataByFeature()['adjClose']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\regression\\linear_model.py:1636: RuntimeWarning:\n",
      "\n",
      "invalid value encountered in double_scalars\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\regression\\linear_model.py:1636: RuntimeWarning:\n",
      "\n",
      "divide by zero encountered in double_scalars\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\regression\\linear_model.py:867: RuntimeWarning:\n",
      "\n",
      "divide by zero encountered in log\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\base\\model.py:1294: RuntimeWarning:\n",
      "\n",
      "invalid value encountered in true_divide\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\scipy\\stats\\_distn_infrastructure.py:879: RuntimeWarning:\n",
      "\n",
      "invalid value encountered in greater\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\scipy\\stats\\_distn_infrastructure.py:879: RuntimeWarning:\n",
      "\n",
      "invalid value encountered in less\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\scipy\\stats\\_distn_infrastructure.py:1738: RuntimeWarning:\n",
      "\n",
      "invalid value encountered in greater_equal\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "not enough values to unpack (expected 3, got 2)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-61-41ab328e84a1>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;31m# between each pair of stocks\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0mdata_clean\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdropna\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m \u001b[0mscores\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpvalues\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpairs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfind_cointegrated_pairs\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata_clean\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      5\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mseaborn\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[1;31m#m = [0,0.2,0.4,0.6,0.8,1]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: not enough values to unpack (expected 3, got 2)"
     ]
    }
   ],
   "source": [
    "# Heatmap to show the p-values of the cointegration test\n",
    "# between each pair of stocks\n",
    "data_clean = data.dropna()\n",
    "scores, pvalues, pairs = find_cointegrated_pairs(data_clean)\n",
    "import seaborn\n",
    "#m = [0,0.2,0.4,0.6,0.8,1]\n",
    "#m = np.linspace(0,1,6)\n",
    "m = np.linspace()\n",
    "seaborn.heatmap(pvalues, xticklabels=instrumentIds, \n",
    "                yticklabels=instrumentIds, cmap='RdYlGn_r' \n",
    "                , mask = (pvalues >= 0.05)\n",
    "                )\n",
    "plt.show()\n",
    "print(\"Co-integrated pairs found:   \", pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\regression\\linear_model.py:1636: RuntimeWarning:\n",
      "\n",
      "invalid value encountered in double_scalars\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\regression\\linear_model.py:1636: RuntimeWarning:\n",
      "\n",
      "divide by zero encountered in double_scalars\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\regression\\linear_model.py:867: RuntimeWarning:\n",
      "\n",
      "divide by zero encountered in log\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\base\\model.py:1294: RuntimeWarning:\n",
      "\n",
      "invalid value encountered in true_divide\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\scipy\\stats\\_distn_infrastructure.py:879: RuntimeWarning:\n",
      "\n",
      "invalid value encountered in greater\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\scipy\\stats\\_distn_infrastructure.py:879: RuntimeWarning:\n",
      "\n",
      "invalid value encountered in less\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\scipy\\stats\\_distn_infrastructure.py:1738: RuntimeWarning:\n",
      "\n",
      "invalid value encountered in greater_equal\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n",
      "C:\\Users\\samar\\Anaconda3\\lib\\site-packages\\statsmodels\\tsa\\stattools.py:1432: CollinearityWarning:\n",
      "\n",
      "y0 and y1 are (almost) perfectly colinear.Cointegration test is not reliable in this case.\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([[1.        , 0.        , 1.        , ..., 0.94867693, 0.23760606,\n",
       "         0.99395504],\n",
       "        [1.        , 1.        , 0.08479233, ..., 0.9916792 ,        nan,\n",
       "                nan],\n",
       "        [1.        , 1.        , 1.        , ..., 0.24248368, 0.40192293,\n",
       "         0.20042186],\n",
       "        ...,\n",
       "        [1.        , 1.        , 1.        , ..., 1.        , 0.9899033 ,\n",
       "         0.03206383],\n",
       "        [1.        , 1.        , 1.        , ..., 1.        , 1.        ,\n",
       "         0.86267732],\n",
       "        [1.        , 1.        , 1.        , ..., 1.        , 1.        ,\n",
       "         1.        ]]),\n",
       " [('AAPL', 'MSFT', 0.0),\n",
       "  ('AAPL', 'CRM', 0.0),\n",
       "  ('AAPL', 'QCOM', 0.0),\n",
       "  ('AAPL', 'NTES', 0.0),\n",
       "  ('MSFT', 'XLNX', 4.5442868094471976e-14),\n",
       "  ('MSFT', 'SSNC', 0.0),\n",
       "  ('GOOG', 'SAP', 0.00035011037837011117),\n",
       "  ('GOOG', 'LHX', 0.009053380875921528),\n",
       "  ('GOOG', 'TWTR', 0.011185817191985447),\n",
       "  ('GOOG', 'SNAP', 6.560295752055094e-05),\n",
       "  ('GOOG', 'CHKP', 0.04903498023191316),\n",
       "  ('GOOG', 'AKAM', 0.008245422856339755),\n",
       "  ('GOOG', 'IT', 0.0488322323715459),\n",
       "  ('GOOG', 'SSNC', 0.04413428802113904),\n",
       "  ('GOOGL', 'TXN', 0.007305006745410036),\n",
       "  ('GOOGL', 'AMAT', 0.014996533661180976),\n",
       "  ('GOOGL', 'WDAY', 0.026362855788260506),\n",
       "  ('GOOGL', 'LRCX', 0.015932642601741914),\n",
       "  ('GOOGL', 'KLAC', 0.044494034595234856),\n",
       "  ('GOOGL', 'VEEV', 0.027367555673190002),\n",
       "  ('GOOGL', 'WDC', 0.03873640910700363),\n",
       "  ('GOOGL', 'AKAM', 0.00232887343937125),\n",
       "  ('GOOGL', 'WORK', 0.006455957095150503),\n",
       "  ('FB', 'MSI', 0.0),\n",
       "  ('FB', 'WDC', 0.04502246725414314),\n",
       "  ('TSM', 'CDW', 0.013472232079045077),\n",
       "  ('INTC', 'ANSS', 0.037459704921883503),\n",
       "  ('ORCL', 'LHX', 0.02022418914709882),\n",
       "  ('ORCL', 'TTWO', 0.013422745547741552),\n",
       "  ('ORCL', 'IT', 0.022382526451941966),\n",
       "  ('SAP', 'LHX', 0.00576692263746576),\n",
       "  ('SAP', 'ERIC', 0.02406422112560776),\n",
       "  ('SAP', 'SNAP', 0.0033434732192592713),\n",
       "  ('SAP', 'CHKP', 0.046877946772680895),\n",
       "  ('ADBE', 'IBM', 0.0),\n",
       "  ('ADBE', 'SNE', 0.0),\n",
       "  ('ADBE', 'ADSK', 0.0046328833775658735),\n",
       "  ('ADBE', 'STM', 0.0),\n",
       "  ('ADBE', 'SYMC', 0.0),\n",
       "  ('ADBE', 'LYFT', 0.0),\n",
       "  ('CRM', 'IBM', 0.0),\n",
       "  ('CRM', 'QCOM', 0.0),\n",
       "  ('CRM', 'TWTR', 0.017303369123419055),\n",
       "  ('CRM', 'NOK', 0.0),\n",
       "  ('CRM', 'TEAM', 0.0),\n",
       "  ('CRM', 'ANET', 0.0),\n",
       "  ('TXN', 'AMAT', 0.045576392557483476),\n",
       "  ('TXN', 'WDAY', 0.03128305981226876),\n",
       "  ('TXN', 'NXPI', 0.026258821627410535),\n",
       "  ('TXN', 'KLAC', 0.03031626265062534),\n",
       "  ('TXN', 'VRSN', 0.0311393385126442),\n",
       "  ('TXN', 'CERN', 0.021305282569952896),\n",
       "  ('TXN', 'VEEV', 0.023554248385096375),\n",
       "  ('TXN', 'SE', 0.02441583770532391),\n",
       "  ('TXN', 'WORK', 0.011944493280377009),\n",
       "  ('NVDA', 'ASML', 0.02942596945310142),\n",
       "  ('NVDA', 'TEAM', 0.03605871710029076),\n",
       "  ('NVDA', 'STM', 0.037057662118518575),\n",
       "  ('NVDA', 'GRMN', 0.047132009445925695),\n",
       "  ('NVDA', 'SE', 0.018541957028176775),\n",
       "  ('IBM', 'SHOP', 0.0),\n",
       "  ('IBM', 'TWLO', 0.0),\n",
       "  ('IBM', 'IT', 0.0),\n",
       "  ('AVGO', 'NOW', 0.04996316219402382),\n",
       "  ('AVGO', 'INFY', 0.013198117550629183),\n",
       "  ('AVGO', 'DELL', 0.044303736416764265),\n",
       "  ('AVGO', 'GIB', 0.009942536884889693),\n",
       "  ('ASML', 'NXPI', 0.008833328844962164),\n",
       "  ('ASML', 'SE', 0.04852213808515136),\n",
       "  ('QCOM', 'VEEV', 0.0),\n",
       "  ('QCOM', 'LYFT', 0.0),\n",
       "  ('SNE', 'ATVI', 0.00825321860886929),\n",
       "  ('SNE', 'WDAY', 0.012169981740696664),\n",
       "  ('SNE', 'LRCX', 0.040743274666808424),\n",
       "  ('SNE', 'KLAC', 0.03094772779289775),\n",
       "  ('SNE', 'VEEV', 0.008309853012738684),\n",
       "  ('SNE', 'KEYS', 0.02634441023190985),\n",
       "  ('SNE', 'WDC', 0.02039056478554301),\n",
       "  ('SNE', 'TWLO', 0.017805535065438215),\n",
       "  ('SNE', 'LYFT', 0.010098609816069304),\n",
       "  ('SNE', 'OKTA', 0.03865751620950317),\n",
       "  ('SNE', 'WORK', 0.002954141419767785),\n",
       "  ('INTU', 'TEAM', 0.013247586976288653),\n",
       "  ('INTU', 'IAC', 0.048739246050271245),\n",
       "  ('VMW', 'SQ', 0.01928703512762786),\n",
       "  ('UBER', 'AKAM', 0.03687667953952667),\n",
       "  ('MU', 'ADI', 0.032418518218558665),\n",
       "  ('MU', 'INFY', 0.01187910899205328),\n",
       "  ('LHX', 'TWTR', 0.012763674097352405),\n",
       "  ('LHX', 'ERIC', 0.010164925663052224),\n",
       "  ('LHX', 'FTV-PA', 0.021693067378044027),\n",
       "  ('LHX', 'FTV', 0.02040341616546186),\n",
       "  ('LHX', 'GLW', 0.012724805886578865),\n",
       "  ('LHX', 'SNAP', 0.028986297035399122),\n",
       "  ('LHX', 'CHKP', 0.013855491467334072),\n",
       "  ('LHX', 'IT', 0.031966802900101995),\n",
       "  ('NOW', 'SQ', 0.0335765230429785),\n",
       "  ('NOW', 'FTV', 0.019373003702515153),\n",
       "  ('NOW', 'GLW', 0.009435405024467856),\n",
       "  ('NOW', 'CERN', 0.03436533736692383),\n",
       "  ('NOW', 'CHKP', 0.017645765904117847),\n",
       "  ('NOW', 'AKAM', 0.033763741110032774),\n",
       "  ('ATVI', 'NXPI', 0.039151919309549525),\n",
       "  ('ATVI', 'VRSN', 0.0232405090365819),\n",
       "  ('ATVI', 'SPOT', 0.026374005175656537),\n",
       "  ('ATVI', 'VEEV', 0.007306625105306949),\n",
       "  ('ATVI', 'KEYS', 0.0163462927325681),\n",
       "  ('ATVI', 'TWLO', 0.03203383877300726),\n",
       "  ('ATVI', 'OKTA', 0.04388959926659392),\n",
       "  ('ATVI', 'SE', 0.004496488743391039),\n",
       "  ('ATVI', 'WORK', 0.040395028035752574),\n",
       "  ('ADI', 'DELL', 0.000118982277740874),\n",
       "  ('ADI', 'MSI', 0.04834956742577967),\n",
       "  ('ADI', 'MCHP', 0.04172458116933473),\n",
       "  ('DELL', 'MTCH', 0.04172485982950017),\n",
       "  ('DELL', 'MXIM', 0.010577803700649462),\n",
       "  ('BIDU', 'TWTR', 0.030937687203540048),\n",
       "  ('BIDU', 'GLW', 0.006525191995978428),\n",
       "  ('BIDU', 'SNAP', 0.021938082127747303),\n",
       "  ('BIDU', 'ANET', 0.0367889090420022),\n",
       "  ('BIDU', 'CHKP', 0.005315680621668676),\n",
       "  ('BIDU', 'AKAM', 0.03596886564549986),\n",
       "  ('BIDU', 'TTWO', 0.04483413251232343),\n",
       "  ('BIDU', 'NTAP', 0.014091508747168318),\n",
       "  ('BIDU', 'IT', 0.023364326497874132),\n",
       "  ('BIDU', 'SSNC', 0.003699702557690272),\n",
       "  ('NTES', 'KEYS', 0.04940824645037778),\n",
       "  ('WDAY', 'NXPI', 0.015136066741833116),\n",
       "  ('WDAY', 'CERN', 0.03255735992053567),\n",
       "  ('WDAY', 'SE', 0.04050413299651328),\n",
       "  ('AMD', 'CDW', 0.0),\n",
       "  ('AMD', 'SYMC', 0.0),\n",
       "  ('SHOP', 'MTCH', 0.0009554594865750865),\n",
       "  ('LRCX', 'NXPI', 0.002322213052722497),\n",
       "  ('LRCX', 'VEEV', 0.04317978735978435),\n",
       "  ('LRCX', 'WORK', 0.010006546282458999),\n",
       "  ('CTSH', 'CERN', 0.04428909260992819),\n",
       "  ('TEL', 'EA', 0.02589999287180938),\n",
       "  ('ADSK', 'CHKP', 0.012386865616508027),\n",
       "  ('ADSK', 'AKAM', 0.04516206988309308),\n",
       "  ('NXPI', 'KLAC', 0.007534467583233706),\n",
       "  ('NXPI', 'TEAM', 0.035761197129899995),\n",
       "  ('NXPI', 'XLNX', 0.04340193555229694),\n",
       "  ('NXPI', 'TME', 0.03169520310431026),\n",
       "  ('NXPI', 'VRSN', 0.0007980469371943595),\n",
       "  ('NXPI', 'CERN', 0.0077685792315821435),\n",
       "  ('NXPI', 'SPOT', 0.01745638238583626),\n",
       "  ('NXPI', 'VEEV', 0.00013271576386822648),\n",
       "  ('NXPI', 'CDNS', 0.012887985862741216),\n",
       "  ('NXPI', 'STX', 0.04422751183223617),\n",
       "  ('NXPI', 'TWLO', 0.0012671632323434361),\n",
       "  ('NXPI', 'LYFT', 0.010797453496868921),\n",
       "  ('NXPI', 'OKTA', 0.011193112662866869),\n",
       "  ('NXPI', 'SE', 0.0028064417123737084),\n",
       "  ('NXPI', 'WORK', 0.0004829105947333961),\n",
       "  ('TWTR', 'CHKP', 0.047406073722890504),\n",
       "  ('TWTR', 'AKAM', 0.006134338441582905),\n",
       "  ('ERIC', 'TTWO', 0.049905582805386504),\n",
       "  ('APH', 'IAC', 0.04710381124139773),\n",
       "  ('APH', 'PINS', 8.296769839839297e-05),\n",
       "  ('APH', 'CTXS', 0.011388563122337715),\n",
       "  ('MSI', 'MTCH', 0.018102256467382),\n",
       "  ('MSI', 'MXIM', 0.04645117136824635),\n",
       "  ('MSI', 'NTAP', 0.042086578364112906),\n",
       "  ('EA', 'FTNT', 0.0038987402780628114),\n",
       "  ('KLAC', 'CERN', 0.035509671852403),\n",
       "  ('KLAC', 'WORK', 0.012053376417095649),\n",
       "  ('HPQ', 'CERN', 0.0),\n",
       "  ('HPQ', 'SYMC', 0.0),\n",
       "  ('SQ', 'TME', 0.04576457960559492),\n",
       "  ('SQ', 'CDNS', 0.040323265989670926),\n",
       "  ('TEAM', 'STM', 0.036029813702167415),\n",
       "  ('TEAM', 'GRMN', 0.003531569989684955),\n",
       "  ('TEAM', 'STX', 0.010408674655307084),\n",
       "  ('FTV-PA', 'FTV', 6.281233611024708e-10),\n",
       "  ('FTV-PA', 'SNAP', 0.029816425430244455),\n",
       "  ('FTV-PA', 'CDNS', 0.015880629108613487),\n",
       "  ('FTV-PA', 'AKAM', 0.003021142285010114),\n",
       "  ('FLT', 'FTV', 0.0028757287144443007),\n",
       "  ('FLT', 'GIB', 0.018228011993314516),\n",
       "  ('FLT', 'SNPS', 0.049042209904059365),\n",
       "  ('FLT', 'KEYS', 0.012327608064533712),\n",
       "  ('FLT', 'MTCH', 0.011196290141849594),\n",
       "  ('FLT', 'WDC', 0.0055652246212891535),\n",
       "  ('FLT', 'AKAM', 0.012514734950444186),\n",
       "  ('FLT', 'TTWO', 0.031528122471608806),\n",
       "  ('FLT', 'PAGS', 0.008239815014797366),\n",
       "  ('FTV', 'TME', 0.023055364042155187),\n",
       "  ('FTV', 'SNAP', 0.02802432271180941),\n",
       "  ('FTV', 'AKAM', 0.011051221630065042),\n",
       "  ('XLNX', 'VEEV', 0.0327281537870789),\n",
       "  ('GLW', 'SNAP', 0.029039484411338468),\n",
       "  ('GLW', 'ANET', 0.031138089380054818),\n",
       "  ('MCHP', 'PANW', 0.0333077263811723),\n",
       "  ('MCHP', 'GRMN', 0.019391653977119526),\n",
       "  ('MCHP', 'MRVL', 0.03657942743064822),\n",
       "  ('MCHP', 'FTNT', 0.04370505908333165),\n",
       "  ('MCHP', 'OKTA', 0.01869976629402199),\n",
       "  ('TME', 'CERN', 0.019380794248272413),\n",
       "  ('TME', 'CHKP', 0.0495663652456733),\n",
       "  ('TME', 'AKAM', 0.011737876027217951),\n",
       "  ('VRSN', 'CDW', 0.048919141125286454),\n",
       "  ('VRSN', 'TWLO', 0.0011344714598204649),\n",
       "  ('VRSN', 'WORK', 0.04107551335959061),\n",
       "  ('CERN', 'AKAM', 0.020709283483331016),\n",
       "  ('SPOT', 'GRMN', 0.04547792908208432),\n",
       "  ('SPOT', 'SE', 0.04219512602048986),\n",
       "  ('GIB', 'SNPS', 0.014962700607883104),\n",
       "  ('GIB', 'MXIM', 0.04527230226085976),\n",
       "  ('GIB', 'PINS', 0.04433416725100598),\n",
       "  ('GIB', 'NTAP', 0.044899479784208635),\n",
       "  ('GIB', 'IT', 0.03768158631702158),\n",
       "  ('GIB', 'SSNC', 0.0348269546802594),\n",
       "  ('VEEV', 'CDNS', 0.03326532672742023),\n",
       "  ('VEEV', 'GRMN', 0.04946239017356616),\n",
       "  ('VEEV', 'AKAM', 0.03730999344142121),\n",
       "  ('VEEV', 'TWLO', 0.004790492864039659),\n",
       "  ('VEEV', 'LYFT', 0.006103453750658369),\n",
       "  ('VEEV', 'SE', 0.04317447336419825),\n",
       "  ('VEEV', 'WORK', 0.0006299429699758016),\n",
       "  ('HPE', 'MTCH', 0.04897190460313208),\n",
       "  ('PANW', 'KEYS', 0.04738428742390415),\n",
       "  ('PANW', 'SPLK', 0.009808392467933413),\n",
       "  ('PANW', 'GRMN', 0.049241675440879984),\n",
       "  ('PANW', 'CHKP', 0.038586731855081685),\n",
       "  ('PANW', 'TWLO', 0.03625162349913569),\n",
       "  ('PANW', 'OKTA', 0.04419114976503415),\n",
       "  ('PANW', 'SE', 0.036157526934977216),\n",
       "  ('SNPS', 'ANSS', 0.012988786681962128),\n",
       "  ('SNAP', 'CHKP', 0.001205053538549419),\n",
       "  ('SNAP', 'AKAM', 0.0007977506310112535),\n",
       "  ('SNAP', 'TTWO', 0.021245425700215825),\n",
       "  ('SNAP', 'NTAP', 0.024106881156058753),\n",
       "  ('SNAP', 'IT', 0.0018790164512388439),\n",
       "  ('SNAP', 'SSNC', 0.00283306183142306),\n",
       "  ('SNAP', 'IQ', 0.03351412124184171),\n",
       "  ('STM', 'CDW', 0.020319045341051438),\n",
       "  ('STM', 'GRMN', 0.0336214780788665),\n",
       "  ('MTCH', 'PINS', 0.00021976695336685812),\n",
       "  ('MTCH', 'CTXS', 0.01839956119952083),\n",
       "  ('MTCH', 'SSNC', 0.04682625171794667),\n",
       "  ('MTCH', 'PAGS', 0.0032896935509221727),\n",
       "  ('ANET', 'CHKP', 0.026066501576237685),\n",
       "  ('ANSS', 'CDW', 0.02026421237372876),\n",
       "  ('ANSS', 'GRMN', 0.041109926007634044),\n",
       "  ('ANSS', 'TWLO', 0.04962989876276621),\n",
       "  ('ANSS', 'LYFT', 0.021103708716738508),\n",
       "  ('ANSS', 'SE', 0.005482153595376144),\n",
       "  ('ANSS', 'WORK', 0.020866511500822207),\n",
       "  ('CDW', 'CDNS', 0.04407037511568915),\n",
       "  ('CDW', 'GRMN', 0.025817885211615856),\n",
       "  ('CDW', 'STX', 0.049021449935342035),\n",
       "  ('CDW', 'TWLO', 0.012262336319056216),\n",
       "  ('CDW', 'OKTA', 0.03157574854681414),\n",
       "  ('CDW', 'SE', 0.005820240279185094),\n",
       "  ('CDW', 'WORK', 0.003462720638017827),\n",
       "  ('CDNS', 'AKAM', 0.03510378561248873),\n",
       "  ('CDNS', 'TWLO', 0.022185948609712708),\n",
       "  ('CDNS', 'LYFT', 0.04634569541877567),\n",
       "  ('WDC', 'GRMN', 0.016844003457281216),\n",
       "  ('WDC', 'STX', 0.01399162880268641),\n",
       "  ('WDC', 'AKAM', 0.034476641162593594),\n",
       "  ('WDC', 'SE', 0.024559330120008625),\n",
       "  ('GRMN', 'STX', 0.011880725629297656),\n",
       "  ('CHKP', 'AKAM', 0.034025799216630355),\n",
       "  ('MXIM', 'NTAP', 0.016746933738688727),\n",
       "  ('MXIM', 'SSNC', 0.03733195409966361),\n",
       "  ('STX', 'SE', 0.04606973423002863),\n",
       "  ('AKAM', 'TWLO', 0.011576044358072215),\n",
       "  ('AKAM', 'LYFT', 0.04853722020150469),\n",
       "  ('AKAM', 'SE', 0.0293214494418489),\n",
       "  ('AKAM', 'WORK', 0.006988063239395401),\n",
       "  ('PINS', 'CTXS', 0.009811951825506302),\n",
       "  ('TTWO', 'NTAP', 0.01678116885718195),\n",
       "  ('TTWO', 'SSNC', 0.03807646291475939),\n",
       "  ('TWLO', 'LYFT', 0.041096574084602426),\n",
       "  ('LYFT', 'SE', 0.026010047568521176),\n",
       "  ('RNG', 'IQ', 0.009476295116085609),\n",
       "  ('SE', 'WORK', 0.03206383408489733)])"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#NOTE CRITICAL LEVEL HAS BEEN SET TO 5% FOR COINTEGRATION TEST\n",
    "def find_cointegrated_pairs(dataframe, critial_level = 0.05):\n",
    "    n = dataframe.shape[1] # the length of dateframe\n",
    "    pvalue_matrix = np.ones((n, n)) # initialize the matrix of p\n",
    "    keys = dataframe.columns # get the column names\n",
    "    pairs = [] # initilize the list for cointegration\n",
    "    for i in range(n):\n",
    "        for j in range(i+1, n): # for j bigger than i\n",
    "            stock1 = dataframe[keys[i]] # obtain the price of \"stock1\"\n",
    "            stock2 = dataframe[keys[j]]# obtain the price of \"stock2\"\n",
    "            result = sm.tsa.stattools.coint(stock1, stock2) # get conintegration\n",
    "            pvalue = result[1] # get the pvalue\n",
    "            pvalue_matrix[i, j] = pvalue\n",
    "            if pvalue < critial_level: # if p-value less than the critical level\n",
    "                pairs.append((keys[i], keys[j], pvalue)) # record the contract with that p-value\n",
    "    return pvalue_matrix, pairs\n",
    "pairs = find_cointegrated_pairs(data_clean, critial_level= 0.05)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#set up the split point for our \"training data\" on which to perform the co-integration test (the remaining dat awill be fed to our backtest function)\n",
    "split = int(len(df) * .4)\n",
    "#run our dataframe (up to the split point) of ticker price data through our co-integration function and store results\n",
    "pvalue_matrix,pairs = find_cointegrated_pairs(df[:split])\n",
    "#convert our matrix of stored results into a DataFrame\n",
    "pvalue_matrix_df = pd.DataFrame(pvalue_matrix)\n",
    "#use Seaborn to plot a heatmap of our results matrix\n",
    "fig, ax = plt.subplots(figsize=(15,10))\n",
    "sns.heatmap(pvalue_matrix_df,xticklabels=used_stocks,yticklabels=used_stocks,ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'pairs' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-12-88a2dba280b6>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mfor\u001b[0m \u001b[0mpair\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mpairs\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m     \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Stock {} and stock {} has a co-integration score of {}\"\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpair\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mpair\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mround\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpair\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m4\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'pairs' is not defined"
     ]
    }
   ],
   "source": [
    "for pair in pairs:\n",
    "    print(\"Stock {} and stock {} has a co-integration score of {}\".format(pair[0],pair[1],round(pair[2],4)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def KalmanFilterAverage(x):\n",
    "  # Construct a Kalman filter\n",
    "    kf = KalmanFilter(transition_matrices = [1],\n",
    "    observation_matrices = [1],\n",
    "    initial_state_mean = 0,\n",
    "    initial_state_covariance = 1,\n",
    "    observation_covariance=1,\n",
    "    transition_covariance=.01)\n",
    "  # Use the observed values of the price to get a rolling mean\n",
    "    state_means, _ = kf.filter(x.values)\n",
    "    state_means = pd.Series(state_means.flatten(), index=x.index)\n",
    "    return state_means\n",
    "# Kalman filter regression\n",
    "def KalmanFilterRegression(x,y):\n",
    "    delta = 1e-3\n",
    "    trans_cov = delta / (1 - delta) * np.eye(2) # How much random walk wiggles\n",
    "    obs_mat = np.expand_dims(np.vstack([[x], [np.ones(len(x))]]).T, axis=1)\n",
    "    kf = KalmanFilter(n_dim_obs=1, n_dim_state=2, # y is 1-dimensional, (alpha, beta) is 2-dimensional\n",
    "    initial_state_mean=[0,0],\n",
    "    initial_state_covariance=np.ones((2, 2)),\n",
    "    transition_matrices=np.eye(2),\n",
    "    observation_matrices=obs_mat,\n",
    "    observation_covariance=2,\n",
    "    transition_covariance=trans_cov)\n",
    "    # Use the observations y to get running estimates and errors for the state parameters\n",
    "    state_means, state_covs = kf.filter(y.values)\n",
    "    return state_means\n",
    "def half_life(spread):\n",
    "    spread_lag = spread.shift(1)\n",
    "    spread_lag.iloc[0] = spread_lag.iloc[1]\n",
    "    spread_ret = spread - spread_lag\n",
    "    spread_ret.iloc[0] = spread_ret.iloc[1]\n",
    "    spread_lag2 = sm.add_constant(spread_lag)\n",
    "    model = sm.OLS(spread_ret,spread_lag2)\n",
    "    res = model.fit()\n",
    "    halflife = int(round(-np.log(2) / res.params[1],0))\n",
    "    if halflife <= 0:\n",
    "        halflife = 1\n",
    "    return halflife"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-14-8a074bcd2459>, line 33)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;36m  File \u001b[1;32m\"<ipython-input-14-8a074bcd2459>\"\u001b[1;36m, line \u001b[1;32m33\u001b[0m\n\u001b[1;33m    df1['long exit'] = ((df1.zScore > - exitZscore) & (df1.zScore.shift(1) < - exitZscore)) df1['num units long'] = np.nan df1.loc[df1['long entry'],'num units long'] = 1 df1.loc[df1['long exit'],'num units long'] = 0 df1['num units long'][0] = 0 df1['num units long'] = df1['num units long'].fillna(method='pad') #set up num units short df1['short entry'] = ((df1.zScore > entryZscore) & ( df1.zScore.shift(1) < entryZscore))\u001b[0m\n\u001b[1;37m                                                                                              ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "def backtest(df,s1, s2):\n",
    "    #############################################################\n",
    "    # INPUT:\n",
    "    # DataFrame of prices\n",
    "    # s1: the symbol of contract one\n",
    "    # s2: the symbol of contract two\n",
    "    # x: the price series of contract one\n",
    "    # y: the price series of contract two\n",
    "    # OUTPUT:\n",
    "    # df1['cum rets']: cumulative returns in pandas data frame\n",
    "    # sharpe: Sharpe ratio\n",
    "    # CAGR: Compound Annual Growth Rate\n",
    "    x = df[s1]\n",
    "    y = df[s2]\n",
    "    # run regression (including Kalman Filter) to find hedge ratio and then create spread series\n",
    "    df1 = pd.DataFrame({'y':y,'x':x})\n",
    "    df1.index = pd.to_datetime(df1.index)\n",
    "    state_means = KalmanFilterRegression(KalmanFilterAverage(x),KalmanFilterAverage(y))\n",
    "    df1['hr'] = - state_means[:,0]\n",
    "    df1['spread'] = df1.y + (df1.x * df1.hr)\n",
    "    # calculate half life\n",
    "    halflife = half_life(df1['spread'])\n",
    "    # calculate z-score with window = half life period\n",
    "    meanSpread = df1.spread.rolling(window=halflife).mean()\n",
    "    stdSpread = df1.spread.rolling(window=halflife).std()\n",
    "    df1['zScore'] = (df1.spread-meanSpread)/stdSpread\n",
    "    ##############################################################\n",
    "    # trading logic\n",
    "    entryZscore = 2\n",
    "    exitZscore = 0\n",
    "    #set up num units long\n",
    "    df1['long entry'] = ((df1.zScore < - entryZscore) & ( df1.zScore.shift(1) > - entryZscore))\n",
    "    df1['long exit'] = ((df1.zScore > - exitZscore) & (df1.zScore.shift(1) < - exitZscore)) df1['num units long'] = np.nan df1.loc[df1['long entry'],'num units long'] = 1 df1.loc[df1['long exit'],'num units long'] = 0 df1['num units long'][0] = 0 df1['num units long'] = df1['num units long'].fillna(method='pad') #set up num units short df1['short entry'] = ((df1.zScore > entryZscore) & ( df1.zScore.shift(1) < entryZscore))\n",
    "    df1['short exit'] = ((df1.zScore < exitZscore) & (df1.zScore.shift(1) > exitZscore))\n",
    "    df1.loc[df1['short entry'],'num units short'] = -1\n",
    "    df1.loc[df1['short exit'],'num units short'] = 0\n",
    "    df1['num units short'][0] = 0\n",
    "    df1['num units short'] = df1['num units short'].fillna(method='pad')\n",
    "    df1['numUnits'] = df1['num units long'] + df1['num units short']\n",
    "    df1['spread pct ch'] = (df1['spread'] - df1['spread'].shift(1)) / ((df1['x'] * abs(df1['hr'])) + df1['y'])\n",
    "    df1['port rets'] = df1['spread pct ch'] * df1['numUnits'].shift(1)\n",
    "    df1['cum rets'] = df1['port rets'].cumsum()\n",
    "    df1['cum rets'] = df1['cum rets'] + 1\n",
    "    ##############################################################\n",
    "    try:\n",
    "        sharpe = ((df1['port rets'].mean() / df1['port rets'].std()) * sqrt(252))\n",
    "    except ZeroDivisionError:\n",
    "        sharpe = 0.0\n",
    "    ##############################################################\n",
    "    start_val = 1\n",
    "    end_val = df1['cum rets'].iat[-1]\n",
    "    start_date = df1.iloc[0].name\n",
    "    end_date = df1.iloc[-1].name\n",
    "    days = (end_date - start_date).days\n",
    "    CAGR = round(((float(end_val) / float(start_val)) ** (252.0/days)) - 1,4)\n",
    "    df1[s1+ \" \"+s2] = df1['cum rets']\n",
    "    return df1[s1+\" \"+s2], sharpe, CAGR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'pairs' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-15-6ad1c13552de>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[1;32mfor\u001b[0m \u001b[0mpair\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mpairs\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m     \u001b[0mrets\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msharpe\u001b[0m\u001b[1;33m,\u001b[0m  \u001b[0mCAGR\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbacktest\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0msplit\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mpair\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mpair\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m     \u001b[0mresults\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrets\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m     \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"The pair {} and {} produced a Sharpe Ratio of {} and a CAGR of {}\"\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpair\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mpair\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mround\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msharpe\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mround\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mCAGR\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m4\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'pairs' is not defined"
     ]
    }
   ],
   "source": [
    "results = []\n",
    "for pair in pairs:\n",
    "    rets, sharpe,  CAGR = backtest(df[split:],pair[0],pair[1])\n",
    "    results.append(rets)\n",
    "    print(\"The pair {} and {} produced a Sharpe Ratio of {} and a CAGR of {}\".format(pair[0],pair[1],round(sharpe,2),round(CAGR,4)))\n",
    "    rets.plot(figsize=(20,15),legend=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "No objects to concatenate",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-16-8dc67a09ca05>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m#concatenate together the individual equity curves into a single DataFrame\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mresults_df\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconcat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresults\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdropna\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[1;31m#equally weight each equity curve by dividing each by the number of pairs held in the DataFrame\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mresults_df\u001b[0m \u001b[1;33m/=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresults_df\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;31m#sum up the equally weighted equity curves to get our final equity curve\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\core\\reshape\\concat.py\u001b[0m in \u001b[0;36mconcat\u001b[1;34m(objs, axis, join, join_axes, ignore_index, keys, levels, names, verify_integrity, sort, copy)\u001b[0m\n\u001b[0;32m    223\u001b[0m                        \u001b[0mkeys\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mkeys\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlevels\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlevels\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnames\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mnames\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    224\u001b[0m                        \u001b[0mverify_integrity\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mverify_integrity\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 225\u001b[1;33m                        copy=copy, sort=sort)\n\u001b[0m\u001b[0;32m    226\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mop\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_result\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    227\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\core\\reshape\\concat.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, objs, axis, join, join_axes, keys, levels, names, ignore_index, verify_integrity, copy, sort)\u001b[0m\n\u001b[0;32m    257\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    258\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobjs\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 259\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'No objects to concatenate'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    260\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    261\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mkeys\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: No objects to concatenate"
     ]
    }
   ],
   "source": [
    "#concatenate together the individual equity curves into a single DataFrame\n",
    "results_df = pd.concat(results,axis=1).dropna()\n",
    "#equally weight each equity curve by dividing each by the number of pairs held in the DataFrame\n",
    "results_df /= len(results_df.columns)\n",
    "#sum up the equally weighted equity curves to get our final equity curve\n",
    "final_res = results_df.sum(axis=1)\n",
    "#plot the chart of our final equity curve\n",
    "final_res.plot(figsize=(20,15))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'final_res' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-18-5d094739f7e6>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m#calculate and print our some final stats for our combined equity curve\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0msharpe\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mfinal_res\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpct_change\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m/\u001b[0m \u001b[0mfinal_res\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpct_change\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstd\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m*\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0msqrt\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m252\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[0mstart_val\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mend_val\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfinal_res\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0miloc\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mstart_date\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfinal_res\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'final_res' is not defined"
     ]
    }
   ],
   "source": [
    "#calculate and print our some final stats for our combined equity curve\n",
    "sharpe = (final_res.pct_change().mean() / final_res.pct_change().std()) * (sqrt(252))\n",
    "start_val = 1\n",
    "end_val = final_res.iloc[-1]\n",
    "start_date = final_res.index[0]\n",
    "end_date = final_res.index[-1]\n",
    "days = (end_date - start_date).days\n",
    "CAGR = round(((float(end_val) / float(start_val)) ** (252.0/days)) - 1,4)\n",
    "print(\"Sharpe Ratio is {} and CAGR is {}\".format(round(sharpe,2),round(CAGR,4)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}