MAINT: Update to Python 3.11 (#159)

* Added new CodeType arguments * DateTimeIndex to list * Remove Python 3.7 wheels * List not list * handle get_calendar exception * scm auto version and h5py alert * exchange_calendar warning and install updates * cibuildwheel update * remove windows / Py39 test exclusion
stefan-jansen · Jan 10, 2023 · 5709281 · 5709281
1 parent 2642caa
commit 5709281
Show file tree

Hide file tree

Showing 12 changed files with 110 additions and 57 deletions.
diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
@@ -16,7 +16,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ ubuntu-latest , windows-latest, macos-latest ]
-        python: [ 37, 38, 39, '310' ]
+        python: [ 38, 39, '310' ]
         arch: [ auto64 ]
 
     steps:
@@ -38,11 +38,10 @@ jobs:
 
       - name: Wheels macOS / Linux
         if: runner.os != 'Windows'
-        uses: pypa/cibuildwheel@v2.11.2
+        uses: pypa/cibuildwheel@v2.11.4
         env:
-          CIBW_BEFORE_ALL_LINUX: >
-            ./tools/install_talib.sh
-          CIBW_BEFORE_TEST_MACOS: brew install ta-lib
+          CIBW_BEFORE_ALL_LINUX: ./tools/install_talib.sh
+          CIBW_BEFORE_ALL_MACOS: brew install ta-lib
           CIBW_ARCHS_LINUX: ${{ matrix.arch }}
           CIBW_ARCHS_MACOS: x86_64 arm64
           CIBW_BUILD: "cp${{ matrix.python }}-*"
@@ -56,7 +55,7 @@ jobs:
 
       - name: Wheels Windows
         if: runner.os == 'Windows'
-        uses: pypa/cibuildwheel@v2.11.2
+        uses: pypa/cibuildwheel@v2.11.4
         env:
           CIBW_BUILD: "cp${{ matrix.python }}-win_amd64"
           CIBW_BEFORE_TEST_WINDOWS: >

diff --git a/.github/workflows/ci_tests_full.yml b/.github/workflows/ci_tests_full.yml
@@ -21,7 +21,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4.0.0
+      - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
 
@@ -35,9 +35,6 @@ jobs:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
         python-version: ["3.8", "3.9", "3.10"]
-        exclude:
-          - os: windows-latest
-            python-version: 3.9
 
     steps:
       - name: Checkout Zipline

diff --git a/.github/workflows/ci_tests_quick.yml b/.github/workflows/ci_tests_quick.yml
@@ -23,7 +23,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4.0.0
+      - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
 

diff --git a/README.md b/README.md
@@ -6,10 +6,10 @@
 
 # Backtest your Trading Strategies
 
-| Version Info        | [![Python](https://img.shields.io/pypi/pyversions/zipline-reloaded.svg?cacheSeconds=2592000")](https://pypi.python.org/pypi/zipline-reloaded) [![Anaconda-Server Badge](https://anaconda.org/ml4t/zipline-reloaded/badges/platforms.svg)](https://anaconda.org/ml4t/zipline-reloaded) [![Release](https://img.shields.io/pypi/v/zipline-reloaded.svg?cacheSeconds=2592000)](https://pypi.org/project/zipline-reloaded/) [![Anaconda-Server Badge](https://anaconda.org/ml4t/zipline-reloaded/badges/version.svg)](https://anaconda.org/ml4t/zipline-reloaded)                                                                                                                                                                                                 |
-| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Version Info        | [![Python](https://img.shields.io/pypi/pyversions/zipline-reloaded.svg?cacheSeconds=2592000)](https://pypi.python.org/pypi/zipline-reloaded) [![Anaconda-Server Badge](https://anaconda.org/ml4t/zipline-reloaded/badges/platforms.svg)](https://anaconda.org/ml4t/zipline-reloaded) [![Release](https://img.shields.io/pypi/v/zipline-reloaded.svg?cacheSeconds=2592000)](https://pypi.org/project/zipline-reloaded/) [![Anaconda-Server Badge](https://anaconda.org/ml4t/zipline-reloaded/badges/version.svg)](https://anaconda.org/ml4t/zipline-reloaded)                                                                                                                                                                                                 |
+| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
 | **Test** **Status** | [![CI Tests](https://github.com/stefan-jansen/zipline-reloaded/actions/workflows/ci_tests_full.yml/badge.svg)](https://github.com/stefan-jansen/zipline-reloaded/actions/workflows/unit_tests.yml) [![PyPI](https://github.com/stefan-jansen/zipline-reloaded/actions/workflows/build_wheels.yml/badge.svg)](https://github.com/stefan-jansen/zipline-reloaded/actions/workflows/build_wheels.yml) [![Anaconda](https://github.com/stefan-jansen/zipline-reloaded/actions/workflows/conda_package.yml/badge.svg)](https://github.com/stefan-jansen/zipline-reloaded/actions/workflows/conda_package.yml) [![codecov](https://codecov.io/gh/stefan-jansen/zipline-reloaded/branch/main/graph/badge.svg)](https://codecov.io/gh/stefan-jansen/zipline-reloaded) |
-| **Community**       | [![Discourse](https://img.shields.io/discourse/topics?server=https%3A%2F%2Fexchange.ml4trading.io%2F)](https://exchange.ml4trading.io) [![ML4T](https://img.shields.io/badge/Powered%20by-ML4Trading-blue)](https://ml4trading.io) [![Twitter](https://img.shields.io/twitter/follow/ml4trading.svg?style=social)](https://twitter.com/ml4trading)                                                                                                                                                                                                                                                                                                                                                                                                            |
+| **Community**       | [![Discourse](https://img.shields.io/discourse/topics?server=https%3A%2F%2Fexchange.ml4trading.io%2F)](https://exchange.ml4trading.io) [![ML4T](https://img.shields.io/badge/Powered%20by-ML4Trading-blue)](https://ml4trading.io) [![Twitter](https://img.shields.io/twitter/follow/ml4trading.svg?style=social)](https://twitter.com/ml4trading)                                                                                                                                                                                                                                                                                                                                                                                                           |
 
 Zipline is a Pythonic event-driven system for backtesting, developed and used as the backtesting and live-trading engine by [crowd-sourced investment fund Quantopian](https://www.bizjournals.com/boston/news/2020/11/10/quantopian-shuts-down-cofounders-head-elsewhere.html). Since it closed late 2020, the domain that had hosted these docs expired. The library is used extensively in the book [Machine Larning for Algorithmic Trading](https://ml4trading.io)
 by [Stefan Jansen](https://www.linkedin.com/in/applied-ai/) who is trying to keep the library up to date and available to his readers and the wider Python algotrading community.
@@ -24,18 +24,24 @@ by [Stefan Jansen](https://www.linkedin.com/in/applied-ai/) who is trying to kee
 - **PyData Integration:** Input of historical data and output of performance statistics are based on Pandas DataFrames to integrate nicely into the existing PyData ecosystem.
 - **Statistics and Machine Learning Libraries:** You can use libraries like matplotlib, scipy, statsmodels, and scikit-klearn to support development, analysis, and visualization of state-of-the-art trading systems.
 
+> **Note:** Release 2.4 updates Zipline to use [exchange_calendars](https://github.com/gerrymanoim/exchange_calendars) >= 4.2. This is a major version update and may break existing code (which we have tried to avoid but cannot guarantee). Please review the changes [here](https://github.com/gerrymanoim/exchange_calendars/issues/61).
+
 ## Installation
 
 Zipline supports Python >= 3.8 and is compatible with current versions of the relevant [NumFOCUS](https://numfocus.org/sponsored-projects?_sft_project_category=python-interface) libraries, including [pandas](https://pandas.pydata.org/) and [scikit-learn](https://scikit-learn.org/stable/index.html).
 
-If your system meets the pre-requisites described in the [installation instructions](https://zipline.ml4trading.io/install.html), you can install Zipline using pip by running:
+If your system meets the pre-requisites described in the [installation instructions](https://zipline.ml4trading.io/install.html), you can install Zipline using `pip` by running:
 
 ```bash
 pip install zipline-reloaded
 ```
 
+> **Note:** Installation under Python 3.11 requires building `h5py` [from source](https://docs.h5py.org/en/stable/build.html#source-installation) until [wheels become available](https://github.com/h5py/h5py/issues/2146).
+
 Alternatively, if you are using the [Anaconda](https://www.anaconda.com/products/individual) or [miniconda](https://docs.conda.io/en/latest/miniconda.html) distributions, you can use
 
+> **Note:** We are currently working to transition the conda package to [conda-forge](https://conda-forge.org/docs/index.html).
+
 ```bash
 conda install -c ml4t -c conda-forge -c ranaroussi zipline-reloaded
 ```
@@ -94,7 +100,7 @@ $ zipline ingest -b quandl
 $ zipline run -f dual_moving_average.py --start 2014-1-1 --end 2018-1-1 -o dma.pickle --no-benchmark
 ```
 
-This will download asset pricing data sourced from [Quandl](https://www.quandl.com/databases/WIKIP/documentation?anchor=companies), and stream it through the algorithm over the specified time range. Then, the resulting performance DataFrame is saved as `dma.pickle`, which you can load and analyze from Python.
+This will download asset pricing data sourced from [Quandl](https://www.quandl.com/databases/WIKIP/documentation?anchor=companies) (since [acquisition](https://www.nasdaq.com/about/press-center/nasdaq-acquires-quandl-advance-use-alternative-data) hosted by NASDAQ), and stream it through the algorithm over the specified time range. Then, the resulting performance DataFrame is saved as `dma.pickle`, which you can load and analyze from Python.
 
 You can find other examples in the [zipline/examples](https://github.com/stefan-jansen/zipline-reloaded/tree/main/src/zipline/examples) directory.
 

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,8 +1,8 @@
 [project]
 name = 'zipline-reloaded'
-version = '2.3'
 description = 'A Pythonic backtester for trading algorithms'
 readme = 'README.md'
+dynamic = ["version"]
 
 authors = [
     { name = 'Quantopian Inc' },
@@ -21,12 +21,12 @@ classifiers = [
     'Programming Language :: Python :: 3.8',
     'Programming Language :: Python :: 3.9',
     'Programming Language :: Python :: 3.10',
+    'Programming Language :: Python :: 3.11',
     'Operating System :: OS Independent',
     'Intended Audience :: Science/Research',
     'Topic :: Office/Business :: Financial :: Investment',
     'Topic :: Scientific/Engineering :: Information Analysis',
     'Topic :: System :: Distributed Computing'
-
 ]
 
 license = { file = "LICENSE" }
@@ -38,7 +38,7 @@ dependencies = [
     'bottleneck >=1.0.0',
     'click >=4.0.0',
     'empyrical-reloaded >=0.5.7',
-    'h5py >=2.7.1',
+    'h5py >=2.7.1', # currently requires installation from source for Python 3.11
     'intervaltree >=2.1.0',
     'iso3166 >=2.1.1',
     'iso4217 >=1.6.20180829',
@@ -94,15 +94,15 @@ test = [
     'click <8.1.0',
     'coverage',
     'pytest-rerunfailures',
-    'psycopg2 ==2.9.4',
-    'pytest-postgresql ==3.1.3'
+    # the following are required to run tests using PostgreSQL instead of SQLite
+    #    'psycopg2',
+    #    'pytest-postgresql ==3.1.3'
 ]
 dev = [
     'flake8 >=3.9.1',
     'black',
     'pre-commit >=2.12.1',
     'Cython>=0.29.21,<3',
-
 ]
 docs = [
     'Cython',
@@ -139,9 +139,8 @@ testpaths = 'tests'
 addopts = '-v'
 
 [tool.cibuildwheel]
-skip = ["cp37-macosx_arm64", "cp37-macosx_universal2"]
 test-extras = "test"
-test-command = "pytest -n 8 --reruns 5 {package}/tests"
+test-command = "pytest -n 2 --reruns 5 {package}/tests"
 build-verbosity = 3
 
 [tool.cibuildwheel.macos]
@@ -154,7 +153,7 @@ skip = "*musllinux*"
 
 [tool.black]
 line-length = 88
-target-version = ['py37', 'py38', 'py39', 'py310']
+target-version = ['py38', 'py39', 'py310']
 exclude = '''
 (
     asv_bench/env
@@ -171,7 +170,7 @@ exclude = '''
 [tool.tox]
 legacy_tox_ini = """
 [tox]
-envlist = py{37,38,39,310}-pandas{11,12,13},py{38,39,310}-pandas{14,15}
+envlist = py{38,39,310}-pandas{12,13,14,15}
 isolated_build = True
 skip_missing_interpreters = True
 minversion = 3.23.0
@@ -190,6 +189,7 @@ setenv =
 changedir = tmp
 extras = test
 deps =
+    pandas12: pandas>=1.2.0,<1.3
     pandas13: pandas>=1.3.0,<1.4
     pandas14: pandas>=1.4.0,<1.5
     pandas15: pandas>=1.5.0,<1.6

diff --git a/src/zipline/testing/fixtures.py b/src/zipline/testing/fixtures.py
@@ -478,15 +478,15 @@ class WithTradingCalendars:
     Attributes
     ----------
     TRADING_CALENDAR_STRS : iterable
-        iterable of identifiers of the calendars to use.
+        Iterable of identifiers of the calendars to use.
     TRADING_CALENDAR_FOR_ASSET_TYPE : dict
         A dictionary which maps asset type names to the calendar associated
         with that asset type.
     """
 
     TRADING_CALENDAR_STRS = ("NYSE",)
     TRADING_CALENDAR_FOR_ASSET_TYPE = {Equity: "NYSE", Future: "us_futures"}
-    # For backwards compatibility, exisitng tests and fixtures refer to
+    # For backwards compatibility, existing tests and fixtures refer to
     # `trading_calendar` with the assumption that the value is the NYSE
     # calendar.
     TRADING_CALENDAR_PRIMARY_CAL = "NYSE"

diff --git a/src/zipline/utils/compat.py b/src/zipline/utils/compat.py
@@ -1,10 +1,10 @@
 import functools
 import inspect
-
+from collections import namedtuple  # noqa: compatibility with python 3.11
 from contextlib import contextmanager, ExitStack
 from html import escape as escape_html
-from types import MappingProxyType as mappingproxy
 from math import ceil
+from types import MappingProxyType as mappingproxy
 
 
 def consistent_round(val):
@@ -19,8 +19,11 @@ def consistent_round(val):
 
 
 def getargspec(f):
+    ArgSpec = namedtuple(
+        "ArgSpec", "args varargs keywords defaults"
+    )  # noqa: compatibility with python 3.11
     full_argspec = inspect.getfullargspec(f)
-    return inspect.ArgSpec(
+    return ArgSpec(
         args=full_argspec.args,
         varargs=full_argspec.varargs,
         keywords=full_argspec.varkw,

diff --git a/src/zipline/utils/paths.py b/src/zipline/utils/paths.py
@@ -6,7 +6,7 @@
 """
 import os
 from pathlib import Path
-from typing import Any, Iterable, Mapping, Optional
+from typing import Any, Iterable, Mapping, Optional, List
 
 import pandas as pd
 
@@ -71,7 +71,7 @@ def modified_since(path: str, dt: pd.Timestamp) -> bool:
     Returns
     -------
     was_modified : bool
-        Will be ``False`` if path doesn't exists, or if its last modified date
+        Will be ``False`` if path doesn't exist, or if its last modified date
         is earlier than or equal to `dt`
     """
     return Path(path).exists() and last_modified_time(path) > dt
@@ -103,7 +103,7 @@ def zipline_root(environ: Optional[Mapping[Any, Any]] = None) -> str:
     return root
 
 
-def zipline_path(paths: list[str], environ: Optional[Mapping[Any, Any]] = None) -> str:
+def zipline_path(paths: List[str], environ: Optional[Mapping[Any, Any]] = None) -> str:
     """Get a path relative to the zipline root.
 
     Parameters

diff --git a/src/zipline/utils/preprocess.py b/src/zipline/utils/preprocess.py
@@ -10,16 +10,16 @@
 
 from zipline.utils.compat import getargspec, wraps
 
-if sys.version_info[0:2] >= (3, 8):
+if sys.version_info[0:2] < (3, 7):
+    _code_argorder_head = ("co_argcount", "co_kwonlyargcount")
+else:
     _code_argorder_head = (
         "co_argcount",
         "co_posonlyargcount",
         "co_kwonlyargcount",
     )
-else:
-    _code_argorder_head = ("co_argcount", "co_kwonlyargcount")
 
-_code_argorder = (_code_argorder_head) + (
+_code_argorder_body = (
     "co_nlocals",
     "co_stacksize",
     "co_flags",
@@ -29,12 +29,33 @@
     "co_varnames",
     "co_filename",
     "co_name",
-    "co_firstlineno",
-    "co_lnotab",
+)
+
+_code_argorder_tail = (
     "co_freevars",
     "co_cellvars",
 )
 
+if sys.version_info[0:2] <= (3, 10):
+    _code_argorder = (
+        _code_argorder_head
+        + _code_argorder_body
+        + ("co_firstlineno", "co_lnotab")
+        + _code_argorder_tail
+    )
+
+else:
+    _code_argorder = (
+        _code_argorder_head
+        + _code_argorder_body
+        + (
+            "co_qualname",  # new in 3.11
+            "co_firstlineno",
+            "co_lnotab",
+            "co_exceptiontable",  # new in 3.11
+        )
+        + _code_argorder_tail
+    )
 NO_DEFAULT = object()
 
 
@@ -233,17 +254,16 @@ def {func_name}({signature}):
     # work as intended.
     try:
         # Try to get the pycode object from the underlying function.
-        original_code = func.__code__
+        _ = func.__code__
     except AttributeError:
         try:
             # The underlying callable was not a function, try to grab the
             # `__func__.__code__` which exists on method objects.
-            original_code = func.__func__.__code__
+            _ = func.__func__.__code__
         except AttributeError:
             # The underlying callable does not have a `__code__`. There is
             # nothing for us to correct.
             return new_func
 
-    args["co_firstlineno"] = original_code.co_firstlineno
     new_func.__code__ = CodeType(*map(getitem(args), _code_argorder))
     return new_func
diff --git a/tests/events/test_events.py b/tests/events/test_events.py
@@ -199,10 +199,10 @@ def session_picker(day):
             return ordered_session_list[day]
 
     else:
-        # Other than AfterOpen and BeforeClose, we don't rely on the the nature
+        # Other than AfterOpen and BeforeClose, we don't rely on the nature
         # of the clock, so we don't care.
         def session_picker(day):
-            return random.choice(cal.sessions[:-1])
+            return random.choice(cal.sessions[:-1].tolist())
 
     return [cal.session_minutes(session_picker(cnt)) for cnt in range(500)]