diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e0c3bbd..0f6aeeeb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,58 +9,65 @@ on: jobs: Windows: name: 'Windows (${{ matrix.python }}, ${{ matrix.arch }}${{ matrix.extra_name }})' - timeout-minutes: 20 + timeout-minutes: 30 runs-on: 'windows-latest' strategy: fail-fast: false matrix: - python: ['3.6', '3.7', '3.8', '3.9'] + python: ['3.8', '3.9', '3.10', '3.11', '3.12'] arch: ['x86', 'x64'] steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: '${{ matrix.python }}' + # This allows the matrix to specify just the major.minor version while still + # expanding it to get the latest patch version including alpha releases. + # This avoids the need to update for each new alpha, beta, release candidate, + # and then finally an actual release version. actions/setup-python doesn't + # support this for PyPy presently so we get no help there. + # + # CPython -> 3.9.0-alpha - 3.9.X + # PyPy -> pypy-3.7 + python-version: ${{ fromJSON(format('["{0}", "{1}"]', format('{0}.0-alpha - {0}.X', matrix.python), matrix.python))[startsWith(matrix.python, 'pypy')] }} architecture: '${{ matrix.arch }}' + cache: pip + cache-dependency-path: test-requirements.txt - name: Run tests run: ./ci.sh shell: bash - env: - # Should match 'name:' up above - JOB_NAME: 'Windows (${{ matrix.python }}, ${{ matrix.arch }}${{ matrix.extra_name }})' + - if: always() + uses: codecov/codecov-action@v3 + with: + directory: empty + name: Windows (${{ matrix.python }}, ${{ matrix.arch }}${{ matrix.extra_name }}) + flags: Windows,${{ matrix.python }} Ubuntu: name: 'Ubuntu (${{ matrix.python }}${{ matrix.extra_name }})' - timeout-minutes: 10 + timeout-minutes: 20 runs-on: 'ubuntu-latest' strategy: fail-fast: false matrix: - # No pypy-3.7 because Trio doesn't support it (nightly is fine, it has a fix we need) - python: ['pypy-3.6', '3.6', '3.7', '3.8', '3.9', '3.6-dev', '3.7-dev', '3.8-dev', '3.9-dev'] + python: ['pypy-3.9', '3.7', '3.8', '3.9', '3.10', '3.11', '3.12', 'pypy-3.9-nightly'] check_formatting: ['0'] - check_docs: ['0'] - pypy_nightly_branch: [''] extra_name: [''] include: - - python: '3.8' + - python: '3.11' check_formatting: '1' extra_name: ', check formatting' - # pypy3.7-nightly produces an "OSError: handle is closed" in the - # bowels of multiprocessing after all tests appear to complete successfully - # - python: '3.7' # <- not actually used - # pypy_nightly_branch: 'py3.7' - # extra_name: ', pypy 3.7 nightly' steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 if: "!endsWith(matrix.python, '-dev')" with: - python-version: '${{ matrix.python }}' + python-version: ${{ fromJSON(format('["{0}", "{1}"]', format('{0}.0-alpha - {0}.X', matrix.python), matrix.python))[startsWith(matrix.python, 'pypy')] }} + cache: pip + cache-dependency-path: test-requirements.txt - name: Setup python (dev) uses: deadsnakes/action@v2.0.2 if: endsWith(matrix.python, '-dev') @@ -76,29 +83,36 @@ jobs: - name: Run tests run: ./ci.sh env: - PYPY_NIGHTLY_BRANCH: '${{ matrix.pypy_nightly_branch }}' CHECK_FORMATTING: '${{ matrix.check_formatting }}' - CHECK_DOCS: '${{ matrix.check_docs }}' - # Should match 'name:' up above - JOB_NAME: 'Ubuntu (${{ matrix.python }}${{ matrix.extra_name }})' + - if: always() + uses: codecov/codecov-action@v3 + with: + directory: empty + name: Ubuntu (${{ matrix.python }}${{ matrix.extra_name }}) + flags: Ubuntu,${{ matrix.python }} macOS: name: 'macOS (${{ matrix.python }})' - timeout-minutes: 10 + timeout-minutes: 20 runs-on: 'macos-latest' strategy: fail-fast: false matrix: - python: ['3.6', '3.7', '3.8', '3.9'] + python: ['3.8', '3.9', '3.10', '3.11', '3.12', 'pypy-3.9-nightly'] steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: '${{ matrix.python }}' + python-version: ${{ fromJSON(format('["{0}", "{1}"]', format('{0}.0-alpha - {0}.X', matrix.python), matrix.python))[startsWith(matrix.python, 'pypy')] }} + cache: pip + cache-dependency-path: test-requirements.txt - name: Run tests run: ./ci.sh - env: - # Should match 'name:' up above - JOB_NAME: 'macOS (${{ matrix.python }})' + - if: always() + uses: codecov/codecov-action@v3 + with: + directory: empty + name: macOS (${{ matrix.python }}) + flags: macOS,${{ matrix.python }} diff --git a/.readthedocs.yml b/.readthedocs.yml index 1c381878..056c6917 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,8 +5,12 @@ formats: - htmlzip - epub +build: + os: "ubuntu-22.04" + tools: + python: "3.11" + python: - version: 3.7 install: - requirements: docs-requirements.txt diff --git a/README.rst b/README.rst index 4c8f24c6..863f563f 100644 --- a/README.rst +++ b/README.rst @@ -26,8 +26,8 @@ **trio-asyncio** is a re-implementation of the ``asyncio`` mainloop on top of Trio. -Trio-Asyncio requires at least Python 3.6. It is tested on recent versions of -3.6, 3.7, 3.8, and nightly. +Trio-Asyncio requires at least Python 3.7. It is tested on recent versions of +3.7 through 3.11, plus 3.12-dev. +++++++++++ Rationale diff --git a/ci.sh b/ci.sh index 7e1a7566..2fbc5276 100755 --- a/ci.sh +++ b/ci.sh @@ -26,40 +26,6 @@ function curl-harder() { return 1 } -################################################################ -# Bootstrap python environment, if necessary -################################################################ - -### PyPy nightly (currently on Travis) ### - -if [ "$PYPY_NIGHTLY_BRANCH" != "" ]; then - JOB_NAME="pypy_nightly_${PYPY_NIGHTLY_BRANCH}" - curl-harder -o pypy.tar.bz2 http://buildbot.pypy.org/nightly/${PYPY_NIGHTLY_BRANCH}/pypy-c-jit-latest-linux64.tar.bz2 - if [ ! -s pypy.tar.bz2 ]; then - # We know: - # - curl succeeded (200 response code) - # - nonetheless, pypy.tar.bz2 does not exist, or contains no data - # This isn't going to work, and the failure is not informative of - # anything involving Trio. - ls -l - echo "PyPy3 nightly build failed to download – something is wrong on their end." - echo "Skipping testing against the nightly build for right now." - exit 0 - fi - tar xaf pypy.tar.bz2 - # something like "pypy-c-jit-89963-748aa3022295-linux64" - PYPY_DIR=$(echo pypy-c-jit-*) - PYTHON_EXE=$PYPY_DIR/bin/pypy3 - - if ! ($PYTHON_EXE -m ensurepip \ - && $PYTHON_EXE -m pip install virtualenv \ - && $PYTHON_EXE -m virtualenv testenv); then - echo "pypy nightly is broken; skipping tests" - exit 0 - fi - source testenv/bin/activate -fi - ################################################################ # We have a Python environment! ################################################################ @@ -69,16 +35,7 @@ python -c "import sys, struct, ssl; print('#' * 70); print('python:', sys.versio python -m pip install -U pip setuptools wheel python -m pip --version -python setup.py sdist --formats=zip -python -m pip install dist/*.zip - -if python -c 'import sys; sys.exit(sys.version_info >= (3, 7))'; then - # Python < 3.7, select last ipython with 3.6 support - # macOS requires the suffix for --in-place or you get an undefined label error - sed -i'.bak' 's/ipython==[^ ]*/ipython==7.16.1/' test-requirements.txt - sed -i'.bak' 's/traitlets==[^ ]*/traitlets==4.3.3/' test-requirements.txt - git diff test-requirements.txt -fi +python -m pip install . # See https://github.com/python-trio/trio/issues/334 YAPF_VERSION=0.20.0 @@ -113,6 +70,8 @@ else mkdir empty || true cd empty + cp ../pyproject.toml ../tests/ + # We have to copy .coveragerc into this directory, rather than passing # --cov-config=../.coveragerc to pytest, because codecov.sh will run # 'coverage xml' to generate the report that it uses, and that will only @@ -124,13 +83,5 @@ else PASSED=false fi - # The codecov docs recommend something like 'bash <(curl ...)' to pipe the - # script directly into bash as its being downloaded. But, the codecov - # server is flaky, so we instead save to a temp file with retries, and - # wait until we've successfully fetched the whole script before trying to - # run it. - curl-harder -o codecov.sh https://codecov.io/bash - bash codecov.sh -n "${JOB_NAME}" - $PASSED fi diff --git a/docs/source/conf.py b/docs/source/conf.py index 83019032..c68b4bca 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -104,7 +104,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. diff --git a/docs/source/index.rst b/docs/source/index.rst index 13f96882..e1e26dc8 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -51,7 +51,7 @@ asyncio libraries such as ``home-assistant``. Helpful facts: * Supported environments: Linux, MacOS, or Windows running some kind of Python - 3.6-or-better (either CPython or PyPy3 is fine). \*BSD and illumOS likely + 3.7-or-better (either CPython or PyPy3 is fine). \*BSD and illumOS likely work too, but are untested. * Install: ``python3 -m pip install -U trio-asyncio`` (or on Windows, maybe diff --git a/newsfragments/121.misc.rst b/newsfragments/121.misc.rst new file mode 100644 index 00000000..41677a63 --- /dev/null +++ b/newsfragments/121.misc.rst @@ -0,0 +1,2 @@ +``trio-asyncio`` now requires Trio 0.22 and does not produce deprecation warnings. +Python 3.12 is now supported. Python 3.6 and 3.7 are no longer supported. diff --git a/newsfragments/123.misc.rst b/newsfragments/123.misc.rst new file mode 100644 index 00000000..8f7d27e9 --- /dev/null +++ b/newsfragments/123.misc.rst @@ -0,0 +1,5 @@ +trio-asyncio now indicates its presence to `sniffio` using the +``sniffio.thread_local`` interface that is preferred since sniffio +v1.3.0. This should be less likely than the previous approach to cause +:func:`sniffio.current_async_library` to return incorrect results due +to unintended inheritance of contextvars. diff --git a/pyproject.toml b/pyproject.toml index 37bbc996..cac350a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,20 @@ +[tool.pytest.ini_options] +addopts = ["-p", "no:asyncio"] +filterwarnings = [ + "error", + "ignore:The loop argument is deprecated since Python 3.8:DeprecationWarning", + 'ignore:"@coroutine" decorator is deprecated since Python 3.8:DeprecationWarning', + "default:Tried to add marker .* but that test doesn't exist.:RuntimeWarning", + "ignore:the imp module is deprecated in favour of importlib.*:DeprecationWarning", + "ignore:'AbstractChildWatcher' is deprecated.*:DeprecationWarning" +] +junit_family = "xunit2" +xfail_strict = true + +[tool.flake8] +max-line-length = 99 +extend-ignore = ['D', 'E402', 'E731', 'E127', 'E502', 'E123', 'W503'] + [tool.towncrier] package = "trio_asyncio" title_format = "trio-asyncio {version} ({project_date})" diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 7dcf898b..00000000 --- a/setup.cfg +++ /dev/null @@ -1,10 +0,0 @@ -[aliases] -test=pytest -[flake8] -max-line-length=99 -ignore=E402,E731,E127,E502,E123,W503 -[tool:pytest] -addopts = -p no:asyncio -filterwarnings = - error - ignore:The loop argument is deprecated*:DeprecationWarning diff --git a/setup.py b/setup.py index 0dad1da0..3c810fcb 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ code (asyncio: ~8000) but passes the complete Python 3.6 test suite with no errors. -``trio_asyncio`` requires Python 3.6 or better. +``trio_asyncio`` requires Python 3.7 or better. Author ====== @@ -52,15 +52,6 @@ """ -install_requires = [ - "trio >= 0.15.0", - "outcome", - "sniffio", -] -if sys.version_info < (3, 7): - install_requires.append("contextvars >= 2.1") - install_requires.append("async_generator >= 1.6") - setup( name="trio_asyncio", version=__version__, # noqa: F821 @@ -71,11 +62,16 @@ url="https://github.com/python-trio/trio-asyncio", license="MIT -or- Apache License 2.0", packages=["trio_asyncio"], - install_requires=install_requires, + install_requires=[ + "trio >= 0.22.0", + "outcome", + "sniffio >= 1.3.0", + "exceptiongroup >= 1.0.0; python_version < '3.11'", + ], # This means, just install *everything* you see under trio/, even if it # doesn't look like a source file, so long as it appears in MANIFEST.in: include_package_data=True, - python_requires=">=3.6", # temporary, for RTD + python_requires=">=3.7", keywords=["async", "io", "trio", "asyncio", "trio-asyncio"], setup_requires=['pytest-runner'], tests_require=['pytest >= 5.4', 'pytest-trio >= 0.6', 'outcome'], @@ -91,8 +87,11 @@ "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: System :: Networking", "Framework :: Trio", "Framework :: AsyncIO", diff --git a/tests/conftest.py b/tests/conftest.py index 0c3e77be..9b9b0c3a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,45 +3,6 @@ import trio_asyncio import inspect -# Hacks for <3.7 -if not hasattr(asyncio, 'run'): - - def run(main, *, debug=False): - loop = asyncio.new_event_loop() - loop.set_debug(debug) - return loop.run(main) - - asyncio.run = run - -if not hasattr(asyncio, 'current_task'): - - def current_task(loop=None): - return asyncio.Task.current_task(loop) - - asyncio.current_task = current_task - -if not hasattr(asyncio, 'all_tasks'): - - def all_tasks(loop=None): - return asyncio.Task.all_tasks(loop) - - asyncio.all_tasks = all_tasks - -if not hasattr(asyncio, 'create_task'): - - if hasattr(asyncio.events, 'get_running_loop'): - - def create_task(coro): - loop = asyncio.events.get_running_loop() - return loop.create_task(coro) - else: - - def create_task(coro): - loop = asyncio.events._get_running_loop() - return loop.create_task(coro) - - asyncio.create_task = create_task - @pytest.fixture async def loop(): diff --git a/tests/interop/test_adapter.py b/tests/interop/test_adapter.py index ff10cb5f..e2bcf800 100644 --- a/tests/interop/test_adapter.py +++ b/tests/interop/test_adapter.py @@ -6,10 +6,7 @@ from tests import aiotest import sys import warnings -try: - from contextlib import asynccontextmanager -except ImportError: - from async_generator import asynccontextmanager +from contextlib import asynccontextmanager from trio_asyncio import TrioAsyncioDeprecationWarning @@ -29,37 +26,34 @@ def __init__(self, loop): self.loop = loop async def dly_trio(self): - if sys.version_info >= (3, 7): - assert sniffio.current_async_library() == "trio" + assert sniffio.current_async_library() == "trio" await trio.sleep(0.01) self.flag |= 2 return 8 @trio_as_aio async def dly_trio_adapted(self): - if sys.version_info >= (3, 7): - assert sniffio.current_async_library() == "trio" + assert sniffio.current_async_library() == "trio" await trio.sleep(0.01) self.flag |= 2 return 8 @aio_as_trio async def dly_asyncio_adapted(self): - if sys.version_info >= (3, 7): - assert sniffio.current_async_library() == "asyncio" + assert sniffio.current_async_library() == "asyncio" await asyncio.sleep(0.01) self.flag |= 1 return 4 async def dly_asyncio(self, do_test=True): - if do_test and sys.version_info >= (3, 7): + if do_test: assert sniffio.current_async_library() == "asyncio" await asyncio.sleep(0.01) self.flag |= 1 return 4 async def iter_asyncio(self, do_test=True): - if do_test and sys.version_info >= (3, 7): + if do_test: assert sniffio.current_async_library() == "asyncio" await asyncio.sleep(0.01) yield 1 @@ -69,8 +63,7 @@ async def iter_asyncio(self, do_test=True): self.flag |= 1 async def iter_trio(self): - if sys.version_info >= (3, 7): - assert sniffio.current_async_library() == "trio" + assert sniffio.current_async_library() == "trio" await trio.sleep(0.01) yield 1 await trio.sleep(0.01) @@ -148,8 +141,7 @@ async def test_trio_asyncio_future(self, loop): async def test_trio_asyncio_iter(self, loop): sth = SomeThing(loop) n = 0 - if sys.version_info >= (3, 7): - assert sniffio.current_async_library() == "trio" + assert sniffio.current_async_library() == "trio" async for x in aio_as_trio(sth.iter_asyncio()): n += 1 assert x == n @@ -159,8 +151,7 @@ async def test_trio_asyncio_iter(self, loop): async def run_asyncio_trio_iter(self, loop): sth = SomeThing(loop) n = 0 - if sys.version_info >= (3, 7): - assert sniffio.current_async_library() == "asyncio" + assert sniffio.current_async_library() == "asyncio" async for x in trio_as_aio(sth.iter_trio()): n += 1 assert x == n diff --git a/tests/interop/test_calls.py b/tests/interop/test_calls.py index 2e33aaee..c8cd9d82 100644 --- a/tests/interop/test_calls.py +++ b/tests/interop/test_calls.py @@ -28,8 +28,7 @@ def __init__(self, parent): async def __aenter__(self): assert self.parent.did_it == 0 self.parent.did_it = 1 - if sys.version_info >= (3, 7): - assert sniffio.current_async_library() == "trio" + assert sniffio.current_async_library() == "trio" await trio.sleep(0.01) self.parent.did_it = 2 return self @@ -47,8 +46,7 @@ def __init__(self, parent, loop): async def __aenter__(self): assert self.parent.did_it == 0 self.parent.did_it = 1 - if sys.version_info >= (3, 7): - assert sniffio.current_async_library() == "asyncio" + assert sniffio.current_async_library() == "asyncio" await asyncio.sleep(0.01) self.parent.did_it = 2 return self diff --git a/tests/pytest.ini b/tests/pytest.ini deleted file mode 100644 index a4bbeb50..00000000 --- a/tests/pytest.ini +++ /dev/null @@ -1,6 +0,0 @@ -[pytest] -filterwarnings = - error - ignore:The loop argument is deprecated since Python 3.8:DeprecationWarning - ignore:"@coroutine" decorator is deprecated since Python 3.8:DeprecationWarning - default:Tried to add marker .* but that test doesn't exist.:RuntimeWarning diff --git a/tests/python/conftest.py b/tests/python/conftest.py index 2aec4cb4..afddbffe 100644 --- a/tests/python/conftest.py +++ b/tests/python/conftest.py @@ -1,9 +1,9 @@ import os import sys import warnings -import py import pytest import unittest +from pathlib import Path try: # the global 'test' package installed with Python @@ -32,13 +32,17 @@ def threading_no_cleanup(*original_values): threading_cleanup.__code__ = threading_no_cleanup.__code__ - asyncio_test_dir = py.path.local(test_asyncio.__path__[0]) + asyncio_test_dir = Path(test_asyncio.__path__[0]) - def aio_test_nodeid(fspath): - relpath = fspath.relto(asyncio_test_dir) - if relpath: - return "/Python-{}.{}/test_asyncio/".format(*sys.version_info[:2]) + relpath - return None + def aio_test_nodeid(path): + try: + relpath = path.relative_to(asyncio_test_dir) + except ValueError: + return None + else: + return "/Python-{}.{}/test_asyncio/{}".format( + *sys.version_info[:2], relpath + ) # A pytest.Module that will only collect unittest.TestCase @@ -72,10 +76,10 @@ def pytest_pycollect_makemodule(path, parent): os.path.join(os.path.dirname(__file__), "__init__.py") ) if candidate == expected: - fspath = py.path.local(test_asyncio.__file__) - node = UnittestOnlyPackage.from_parent(parent, fspath=fspath) + path = Path(test_asyncio.__file__) + node = UnittestOnlyPackage.from_parent(parent, path=path) # This keeps all test names from showing as "." - node._nodeid = aio_test_nodeid(fspath) + node._nodeid = aio_test_nodeid(path) return node @@ -110,76 +114,21 @@ def skip(rel_id): "test_base_events.py::BaseEventLoopWithSelectorTests::" "test_log_slow_callbacks" ) - - if sys.version_info >= (3, 7): - xfail( - "test_tasks.py::RunCoroutineThreadsafeTests::" - "test_run_coroutine_threadsafe_task_factory_exception" - ) if sys.version_info >= (3, 8): xfail( "test_tasks.py::RunCoroutineThreadsafeTests::" "test_run_coroutine_threadsafe_task_cancelled" ) - xfail( - "test_tasks.py::RunCoroutineThreadsafeTests::" - "test_run_coroutine_threadsafe_with_timeout" - ) + if sys.version_info < (3, 11): + xfail( + "test_tasks.py::RunCoroutineThreadsafeTests::" + "test_run_coroutine_threadsafe_with_timeout" + ) if sys.platform == "win32": - xfail("test_windows_events.py::ProactorLoopCtrlC::test_ctrl_c") - - # The CPython SSL tests ignored here fail with - # ConnectionResetError on Pythons <= 3.7.x for some unknown x. - # (3.7.1 fails, 3.7.5 and 3.7.6 pass; older 3.6.x also affected) - if sys.platform != "win32": - import selectors - - xfail_per_eventloop = [] - if sys.implementation.name == "pypy": - # pypy uses a different spelling of the certificate - # failure error message which causes this test to spuriously fail - if sys.version_info >= (3, 7): - xfail_per_eventloop += [ - "test_create_server_ssl_match_failed" - ] - else: - if sys.version_info < (3, 8): - xfail_per_eventloop += [ - "test_create_ssl_connection", - "test_create_ssl_unix_connection" - ] - if sys.version_info < (3, 7): - xfail_per_eventloop += [ - "test_legacy_create_ssl_connection", - "test_legacy_create_ssl_unix_connection", - ] - - kinds = ("Select",) - for candidate in ("Kqueue", "Epoll", "Poll"): - if hasattr(selectors, candidate + "Selector"): - kinds += (candidate.replace("Epoll", "EPoll"),) - for kind in kinds: - for test in xfail_per_eventloop: - xfail("test_events.py::{}EventLoopTests::{}".format(kind, test)) - - if sys.implementation.name != "pypy": - if sys.version_info < (3, 7): - stream_suite = "StreamReaderTests" - else: - stream_suite = "StreamTests" - for which in ("open_connection", "open_unix_connection"): - xfail( - "test_streams.py::{}::test_{}_no_loop_ssl" - .format(stream_suite, which) - ) - - if sys.implementation.name == "pypy" and sys.version_info >= (3, 7): - # This fails due to a trivial difference in how pypy handles IPv6 - # addresses - xfail( - "test_base_events.py::BaseEventLoopWithSelectorTests::" - "test_create_connection_ipv6_scope" - ) + # hangs on 3.11+, fails without hanging on 3.8-3.10 + skip("test_windows_events.py::ProactorLoopCtrlC::test_ctrl_c") + + if sys.implementation.name == "pypy": # This test depends on the C implementation of asyncio.Future, and # unlike most such tests it is not configured to be skipped if # the C implementation is not available @@ -187,11 +136,62 @@ def skip(rel_id): "test_futures.py::CFutureInheritanceTests::" "test_inherit_without_calling_super_init" ) - # These tests assume CPython-style immediate finalization of - # objects when they become unreferenced - for test in ( - "test_create_connection_memory_leak", - "test_handshake_timeout", - "test_start_tls_client_reg_proto_1", - ): - xfail("test_sslproto.py::SelectorStartTLSTests::{}".format(test)) + if sys.version_info < (3, 8): + # These tests assume CPython-style immediate finalization of + # objects when they become unreferenced + for test in ( + "test_create_connection_memory_leak", + "test_handshake_timeout", + "test_start_tls_client_reg_proto_1", + ): + xfail("test_sslproto.py::SelectorStartTLSTests::{}".format(test)) + + # This test depends on the name of the loopback interface. On Github Actions + # it fails on macOS always, and on Linux/Windows except on 3.8. + skip( + "test_base_events.py::BaseEventLoopWithSelectorTests::" + "test_create_connection_ipv6_scope" + ) + + if sys.platform == "darwin": + # https://foss.heptapod.net/pypy/pypy/-/issues/3964 causes infinite loops + for nodeid, item in by_id.items(): + if "sendfile" in nodeid: + item.add_marker(pytest.mark.skip) + + if sys.version_info >= (3, 11): + # This tries to use a mock ChildWatcher that does something unlikely. + # We don't support it because we don't actually use the ChildWatcher + # to manage subprocesses. + xfail( + "test_subprocess.py::GenericWatcherTests::" + "test_create_subprocess_fails_with_inactive_watcher" + ) + + # This forks a child process and tries to run a new event loop there, + # but Trio isn't fork-safe -- it hangs nondeterministically. + skip("test_events.py::TestPyGetEventLoop::test_get_event_loop_new_process") + skip("test_events.py::TestCGetEventLoop::test_get_event_loop_new_process") + + if sys.version_info >= (3, 9): + # This tries to create a new loop from within an existing one, + # which we don't support. + xfail("test_locks.py::ConditionTests::test_ambiguous_loops") + + if sys.version_info >= (3, 12): + # This test sets signal handlers from within a coroutine, + # which doesn't work for us because SyncTrioEventLoop runs on + # a non-main thread. + xfail("test_unix_events.py::TestFork::test_fork_signal_handling") + + # This test explicitly uses asyncio.tasks._c_current_task, + # bypassing our monkeypatch. + xfail("test_tasks.py::CCurrentLoopTests::test_current_task_with_implicit_loop") + + # These tests assume asyncio.sleep(0) is sufficient to run all pending tasks + xfail("test_futures2.py::PyFutureTests::test_task_exc_handler_correct_context") + xfail("test_futures2.py::CFutureTests::test_task_exc_handler_correct_context") + + # This test assumes that get_event_loop_policy().get_event_loop() doesn't + # automatically return the running loop + skip("test_subprocess.py::GenericWatcherTests::test_create_subprocess_with_pidfd") diff --git a/tests/test_misc.py b/tests/test_misc.py index dd97f08f..a1e4f81d 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -4,6 +4,9 @@ import trio import sys +if sys.version_info < (3, 11): + from exceptiongroup import ExceptionGroup, BaseExceptionGroup + class Seen: flag = 0 @@ -12,16 +15,16 @@ class Seen: class TestMisc: @pytest.mark.trio async def test_close_no_stop(self): - with pytest.raises(RuntimeError): - async with trio_asyncio.open_loop() as loop: + async with trio_asyncio.open_loop() as loop: + triggered = trio.Event() - def close_no_stop(): + def close_no_stop(): + with pytest.raises(RuntimeError): loop.close() + triggered.set() - loop.call_soon(close_no_stop) - - await trio.sleep(0.1) - await loop.wait_closed() + loop.call_soon(close_no_stop) + await triggered.wait() @pytest.mark.trio async def test_too_many_stops(self): @@ -101,7 +104,7 @@ async def nest(x): with pytest.raises(RuntimeError): trio_asyncio.run_trio_task(nest, 100) - with pytest.raises((AttributeError, RuntimeError)): + with pytest.raises((AttributeError, RuntimeError, TypeError)): with trio_asyncio.open_loop(): nest(1000) @@ -253,8 +256,8 @@ async def trio_task(): except trio.Cancelled: if throw_another: # This will combine with the Cancelled from the - # background sleep_forever task to create a - # MultiError escaping from trio_task + # background sleep_forever task to create an + # ExceptionGroup escaping from trio_task raise ValueError("hi") async with trio.open_nursery() as nursery: @@ -333,14 +336,21 @@ def collect_exceptions(loop, context): await raise_in_aio_loop(expected[0]) with pytest.raises(SystemExit): await raise_in_aio_loop(SystemExit(0)) - with pytest.raises(SystemExit): - await raise_in_aio_loop(trio.MultiError([expected[1], SystemExit()])) - await raise_in_aio_loop(trio.MultiError(expected[2:])) - assert exceptions == expected + with pytest.raises(BaseExceptionGroup) as result: + await raise_in_aio_loop(BaseExceptionGroup("", [expected[1], SystemExit()])) + assert len(result.value.exceptions) == 1 + assert isinstance(result.value.exceptions[0], SystemExit) + await raise_in_aio_loop(ExceptionGroup("", expected[2:])) + + assert len(exceptions) == 3 + assert exceptions[0] is expected[0] + assert isinstance(exceptions[1], ExceptionGroup) + assert exceptions[1].exceptions == (expected[1],) + assert isinstance(exceptions[2], ExceptionGroup) + assert exceptions[2].exceptions == tuple(expected[2:]) @pytest.mark.trio -@pytest.mark.skipif(sys.version_info < (3, 7), reason="needs asyncio contextvars") async def test_contextvars(): import contextvars diff --git a/tests/test_trio_asyncio.py b/tests/test_trio_asyncio.py index ac401e7d..a86643be 100644 --- a/tests/test_trio_asyncio.py +++ b/tests/test_trio_asyncio.py @@ -3,7 +3,6 @@ import types import asyncio import trio -from async_generator import async_generator, yield_ import trio_asyncio @@ -43,12 +42,7 @@ async def test_fixtured_asyncpg_conn(asyncio_fixture_with_fixtured_loop): @pytest.mark.trio async def test_get_running_loop(): async with trio_asyncio.open_loop() as loop: - try: - from asyncio import get_running_loop - except ImportError: - pass # Python 3.6 - else: - assert get_running_loop() == loop + assert asyncio.get_running_loop() == loop @pytest.mark.trio diff --git a/trio_asyncio/_async.py b/trio_asyncio/_async.py index e46d10e9..5e605c12 100644 --- a/trio_asyncio/_async.py +++ b/trio_asyncio/_async.py @@ -1,7 +1,7 @@ import trio import asyncio -from ._base import BaseTrioEventLoop +from ._base import BaseTrioEventLoop, TrioAsyncioExit class TrioEventLoop(BaseTrioEventLoop): @@ -32,17 +32,23 @@ def default_exception_handler(self, context): asynchronous loops. """ - # TODO: add context.get('handle') to the exception + # Call the original default handler so we get the full info in the log + super().default_exception_handler(context) + + # Also raise an exception so it can't go unnoticed exception = context.get('exception') if exception is None: message = context.get('message') if not message: message = 'Unhandled error in event loop' - raise RuntimeError(message) - else: + exception = RuntimeError(message) + + async def propagate_asyncio_error(): raise exception + self._nursery.start_soon(propagate_asyncio_error) + def stop(self, waiter=None): """Halt the main loop. @@ -64,7 +70,7 @@ def stop(self, waiter=None): def stop_me(): waiter.set() - raise StopAsyncIteration + raise TrioAsyncioExit("stopping trio-asyncio loop") if self._stopped.is_set(): waiter.set() diff --git a/trio_asyncio/_base.py b/trio_asyncio/_base.py index ff35c63f..97f9a9e5 100644 --- a/trio_asyncio/_base.py +++ b/trio_asyncio/_base.py @@ -4,7 +4,6 @@ import trio import heapq import signal -import sniffio import asyncio import warnings import concurrent.futures @@ -13,6 +12,7 @@ from ._util import run_aio_future from selectors import _BaseSelectorImpl, EVENT_READ, EVENT_WRITE +from sniffio import thread_local as sniffio_library try: from trio.lowlevel import wait_for_child @@ -37,6 +37,13 @@ def clear(self): pass +# Exception raised internally to stop the main loop. Must subclass +# SystemExit or KeyboardInterrupt in order to make it through various +# asyncio layers. +class TrioAsyncioExit(SystemExit): + pass + + class _TrioSelector(_BaseSelectorImpl): """A selector that hooks into a ``TrioEventLoop``. @@ -209,13 +216,13 @@ async def run_aio_coroutine(self, coro): This is a Trio-flavored async function. """ - self._check_closed() - t = sniffio.current_async_library_cvar.set("asyncio") - fut = asyncio.ensure_future(coro, loop=self) try: - return await run_aio_future(fut) - finally: - sniffio.current_async_library_cvar.reset(t) + self._check_closed() + fut = asyncio.ensure_future(coro, loop=self) + except BaseException: + coro.close() # avoid unawaited coroutine error + raise + return await run_aio_future(fut) def trio_as_future(self, proc, *args): """Start a new Trio task to run ``await proc(*args)`` asynchronously. @@ -499,6 +506,8 @@ async def _reader_loop(self, fd, handle): with handle._scope: try: while True: + if handle._cancelled: + break await _wait_readable(fd) if handle._cancelled: break @@ -550,6 +559,8 @@ async def _writer_loop(self, fd, handle): with handle._scope: try: while True: + if handle._cancelled: + break await _wait_writable(fd) if handle._cancelled: break @@ -626,7 +637,6 @@ async def _main_loop(self, task_status=trio.TASK_STATUS_IGNORED): self._stopped = trio.Event() task_status.started() - sniffio.current_async_library_cvar.set("asyncio") try: # The shield here ensures that if the context surrounding @@ -642,7 +652,7 @@ async def _main_loop(self, task_status=trio.TASK_STATUS_IGNORED): with trio.CancelScope(shield=True): while not self._stopped.is_set(): await self._main_loop_one() - except StopAsyncIteration: + except TrioAsyncioExit: # raised by .stop_me() to interrupt the loop pass finally: @@ -686,16 +696,16 @@ async def _main_loop_one(self, no_wait=False): # Don't go through the expensive nursery dance # if this is a sync function. if isinstance(obj, AsyncHandle): - if hasattr(obj, '_context'): - obj._context.run(self._nursery.start_soon, obj._run, name=obj._callback) - else: - self._nursery.start_soon(obj._run, name=obj._callback) + # AsyncHandle is only used to run Trio tasks, so no need to set the + # sniffio library + obj._context.run(self._nursery.start_soon, obj._run, name=obj._callback) await obj._started.wait() else: - if hasattr(obj, '_context'): - obj._context.run(obj._callback, *obj._args) - else: - obj._callback(*obj._args) + prev_library, sniffio_library.name = sniffio_library.name, "asyncio" + try: + obj._run() + finally: + sniffio_library.name = prev_library async def _main_loop_exit(self): """Finalize the loop. It may not be re-entered.""" @@ -721,7 +731,7 @@ async def _main_loop_exit(self): await self._main_loop_one(no_wait=True) except trio.WouldBlock: break - except StopAsyncIteration: + except TrioAsyncioExit: pass # Kill off unprocessed work diff --git a/trio_asyncio/_handles.py b/trio_asyncio/_handles.py index 4c97471b..b7908b87 100644 --- a/trio_asyncio/_handles.py +++ b/trio_asyncio/_handles.py @@ -1,11 +1,11 @@ import sys import trio +import types import asyncio -import sniffio -try: - from asyncio.format_helpers import _format_callback, _get_function_source -except ImportError: # <3.7 - from asyncio.events import _format_callback, _get_function_source +from asyncio.format_helpers import _format_callback, _get_function_source + +if sys.version_info < (3, 11): + from exceptiongroup import BaseExceptionGroup def _format_callback_source(func, args): @@ -54,6 +54,65 @@ def _raise(self, exc): self._loop.call_exception_handler(context) +# copied from trio._core._multierror, but relying on traceback constructability +# from Python (as introduced in 3.7) instead of ctypes hackery +def concat_tb(head, tail): + # We have to use an iterative algorithm here, because in the worst case + # this might be a RecursionError stack that is by definition too deep to + # process by recursion! + head_tbs = [] + pointer = head + while pointer is not None: + head_tbs.append(pointer) + pointer = pointer.tb_next + current_head = tail + for head_tb in reversed(head_tbs): + current_head = types.TracebackType( + current_head, head_tb.tb_frame, head_tb.tb_lasti, head_tb.tb_lineno + ) + return current_head + + +# copied from trio._core._run with minor modifications: +def collapse_exception_group(excgroup): + """Recursively collapse any single-exception groups into that single contained + exception. + """ + exceptions = list(excgroup.exceptions) + modified = False + for i, exc in enumerate(exceptions): + if isinstance(exc, BaseExceptionGroup): + new_exc = collapse_exception_group(exc) + if new_exc is not exc: + modified = True + exceptions[i] = new_exc + + if len(exceptions) == 1 and getattr(excgroup, "collapse", False): + exceptions[0].__traceback__ = concat_tb( + excgroup.__traceback__, exceptions[0].__traceback__ + ) + return exceptions[0] + elif modified: + return excgroup.derive(exceptions) + else: + return excgroup + + +def collapse_aware_exception_split(exc, etype): + if not isinstance(exc, BaseExceptionGroup): + if isinstance(exc, etype): + return exc, None + else: + return None, exc + + match, rest = exc.split(etype) + if isinstance(match, BaseExceptionGroup): + match = collapse_exception_group(match) + if isinstance(rest, BaseExceptionGroup): + rest = collapse_exception_group(rest) + return match, rest + + class AsyncHandle(ScopedHandle): """A ScopedHandle associated with the execution of a Trio-flavored async function. @@ -79,30 +138,10 @@ def propagate_cancel(f): self.cancel() async def _run(self): - sniffio.current_async_library_cvar.set("trio") self._started.set() if self._cancelled: return - def report_exception(exc): - if not isinstance(exc, Exception): - # Let BaseExceptions such as Cancelled escape without being noted. - return exc - # Otherwise defer to the asyncio exception handler. (In an async loop - # this will still raise the exception out of the loop, terminating it.) - self._raise(exc) - return None - - def remove_cancelled(exc): - if isinstance(exc, trio.Cancelled): - return None - return exc - - def only_cancelled(exc): - if isinstance(exc, trio.Cancelled): - return exc - return None - try: # Run the callback with self._scope: @@ -117,19 +156,24 @@ def only_cancelled(exc): except BaseException as exc: if not self._fut: - # Pass Exceptions through the fallback exception handler since - # they have nowhere better to go. Let BaseExceptions escape so - # that Cancelled and SystemExit work reasonably. - with trio.MultiError.catch(report_exception): - raise + # Pass Exceptions through the fallback exception + # handler since they have nowhere better to go. (In an + # async loop this will still raise the exception out + # of the loop, terminating it.) Let BaseExceptions + # escape so that Cancelled and SystemExit work + # reasonably. + rest, base = collapse_aware_exception_split(exc, Exception) + if rest: + self._raise(rest) + if base: + raise base else: # The result future gets all the non-Cancelled # exceptions. Any Cancelled need to keep propagating # out of this stack frame in order to reach the cancel - # scope for which they're intended. This would be a - # great place for ExceptionGroup.split() if we had it. - cancelled = trio.MultiError.filter(only_cancelled, exc) - rest = trio.MultiError.filter(remove_cancelled, exc) + # scope for which they're intended. Any non-Cancelled + # BaseExceptions keep propagating. + cancelled, rest = collapse_aware_exception_split(exc, trio.Cancelled) if not self._fut.cancelled(): if rest: self._fut.set_exception(rest) @@ -137,6 +181,7 @@ def only_cancelled(exc): self._fut.cancel() if cancelled: raise cancelled + finally: # asyncio says this is needed to break cycles when an exception occurs. # I'm not so sure, but it doesn't seem to do any harm. diff --git a/trio_asyncio/_loop.py b/trio_asyncio/_loop.py index b2dac1af..962f24e1 100644 --- a/trio_asyncio/_loop.py +++ b/trio_asyncio/_loop.py @@ -1,16 +1,13 @@ # This code implements a clone of the asyncio mainloop which hooks into # Trio. +import os import sys import trio import asyncio import threading from contextvars import ContextVar - -try: - from contextlib import asynccontextmanager -except ImportError: - from async_generator import asynccontextmanager +from contextlib import asynccontextmanager from ._async import TrioEventLoop from ._util import run_aio_future @@ -186,6 +183,25 @@ def set_event_loop(self, loop): super().set_event_loop(loop) +# get_event_loop() without a running loop is deprecated in 3.12+. The logic for emitting the +# DeprecationWarning walks the stack looking at module names in order to associate it with +# the first caller outside asyncio. We need to pretend to be asyncio in order for that to work. +if sys.version_info >= (3, 12): + __name__ = "asyncio.fake.trio_asyncio._loop" + +# Make sure we don't try to continue using the Trio loop after a fork() +if hasattr(os, "register_at_fork"): + + def _clear_state_after_fork(): + if _in_trio_context(): + from trio._core._run import GLOBAL_RUN_CONTEXT + + del GLOBAL_RUN_CONTEXT.task + del GLOBAL_RUN_CONTEXT.runner + current_loop.set(None) + + os.register_at_fork(after_in_child=_clear_state_after_fork) + from asyncio import events as _aio_event ##### @@ -205,9 +221,14 @@ def _new_policy_set(new_policy): raise RuntimeError("You can't set the Trio loop policy manually") if _in_trio_context(): raise RuntimeError("You can't change the event loop policy in Trio context") - else: - assert new_policy is None or isinstance(new_policy, asyncio.AbstractEventLoopPolicy) - _faked_policy.policy = new_policy + if (new_policy is not None and not isinstance(new_policy, asyncio.AbstractEventLoopPolicy)): + # Raise the type of error that the CPython test suite expects + raise_type = TypeError if sys.version_info >= (3, 11) else AssertionError + raise raise_type( + "policy must be an instance of AbstractEventLoopPolicy or None, " + f"not '{type(new_policy).__name__}'" + ) + _faked_policy.policy = new_policy _orig_policy_get = _aio_event.get_event_loop_policy @@ -219,39 +240,36 @@ def _new_policy_set(new_policy): ##### -try: - _orig_run_get = _aio_event._get_running_loop +_orig_run_get = _aio_event._get_running_loop -except AttributeError: - pass -else: +def _new_run_get(): + try: + task = trio.lowlevel.current_task() + except RuntimeError: + pass + else: + # Trio context. Note: NOT current_loop.get()! + # See comment in _TrioPolicy.get_event_loop(). + return task.context.get(current_loop) + # Not Trio context + return _orig_run_get() - def _new_run_get(): - try: - task = trio.lowlevel.current_task() - except RuntimeError: - pass - else: - # Trio context. Note: NOT current_loop.get()! - # See comment in _TrioPolicy.get_event_loop(). - return task.context.get(current_loop) - # Not Trio context - return _orig_run_get() - - # Must override the non-underscore-prefixed get_running_loop() too, - # else will use the C-accelerated one which doesn't call the patched - # _get_running_loop() - def _new_run_get_or_throw(): - result = _new_run_get() - if result is None: - raise RuntimeError("no running event loop") - return result - - _aio_event._get_running_loop = _new_run_get - _aio_event.get_running_loop = _new_run_get_or_throw - asyncio._get_running_loop = _new_run_get - asyncio.get_running_loop = _new_run_get_or_throw + +# Must override the non-underscore-prefixed get_running_loop() too, +# else will use the C-accelerated one which doesn't call the patched +# _get_running_loop() +def _new_run_get_or_throw(): + result = _new_run_get() + if result is None: + raise RuntimeError("no running event loop") + return result + + +_aio_event._get_running_loop = _new_run_get +_aio_event.get_running_loop = _new_run_get_or_throw +asyncio._get_running_loop = _new_run_get +asyncio.get_running_loop = _new_run_get_or_throw ##### @@ -281,6 +299,18 @@ def _new_loop_new(): asyncio.set_event_loop = _new_loop_set asyncio.new_event_loop = _new_loop_new +# current_task is implemented in C in 3.12+, which creates a problem because it +# accesses the non-monkeypatched version of _get_running_loop() +from asyncio import current_task as _orig_current_task + + +def _new_current_task(loop=None): + return _orig_current_task(loop or _new_run_get()) + + +asyncio.tasks.current_task = _new_current_task +asyncio.current_task = _new_current_task + ##### @@ -456,10 +486,7 @@ async def wait_for_sync(): # Like asyncio.run(), we don't bother cancelling and waiting # on any additional asyncio tasks that these tasks start as they # unwind. - if sys.version_info >= (3, 7): - aio_tasks = asyncio.all_tasks(loop) - else: - aio_tasks = {t for t in asyncio.Task.all_tasks(loop) if not t.done()} + aio_tasks = asyncio.all_tasks(loop) for task in aio_tasks: tasks_nursery.start_soon(run_aio_future, task) tasks_nursery.cancel_scope.cancel() diff --git a/trio_asyncio/_sync.py b/trio_asyncio/_sync.py index 8650b8f9..32776354 100644 --- a/trio_asyncio/_sync.py +++ b/trio_asyncio/_sync.py @@ -6,7 +6,7 @@ import threading import outcome -from ._base import BaseTrioEventLoop +from ._base import BaseTrioEventLoop, TrioAsyncioExit async def _sync(proc, *args): @@ -49,7 +49,7 @@ def stop(self): def do_stop(): self._stop_pending = False - raise StopAsyncIteration + raise TrioAsyncioExit("stopping trio-asyncio loop") # async def stop_me(): @@ -148,7 +148,7 @@ def is_done(_): while result is None: try: await self._main_loop_one(no_wait=True) - except StopAsyncIteration: + except TrioAsyncioExit: pass except trio.WouldBlock: pass diff --git a/trio_asyncio/_util.py b/trio_asyncio/_util.py index e27a0178..0813c643 100644 --- a/trio_asyncio/_util.py +++ b/trio_asyncio/_util.py @@ -5,7 +5,6 @@ import asyncio import sys import outcome -import sniffio async def run_aio_future(future): @@ -69,7 +68,6 @@ async def run_aio_generator(loop, async_generator): current_read = None async def consume_next(): - t = sniffio.current_async_library_cvar.set("asyncio") try: item = await async_generator.__anext__() result = outcome.Value(value=item) @@ -80,8 +78,6 @@ async def consume_next(): return except Exception as e: result = outcome.Error(error=e) - finally: - sniffio.current_async_library_cvar.reset(t) trio.lowlevel.reschedule(task, result)