From e17bcf169d2b24898fbdf599b39a884d7b1f74fc Mon Sep 17 00:00:00 2001 From: Brian Bouterse Date: Tue, 14 Sep 2021 16:46:50 -0400 Subject: [PATCH] Add xfailing integration tests against proxy.py This patch adds full end-to-end tests for sending requests to HTTP and HTTPS endpoints through an HTTPS proxy. The first case is currently supported and the second one is not. This is why the latter test is marked as expected to fail. The support for TLS-in-TLS in the upstream stdlib asyncio is currently disabled but is available in Python 3.9 via monkey-patching which is demonstrated in the added tests. Refs: * https://bugs.python.org/issue37179 * https://github.com/python/cpython/pull/28073 * https://github.com/aio-libs/aiohttp/pull/5992 Co-Authored-By: Sviatoslav Sydorenko --- CHANGES/6002.misc | 2 + requirements/dev.txt | 3 + requirements/test.txt | 1 + tests/test_proxy_functional.py | 130 +++++++++++++++++++++++++++++++++ 4 files changed, 136 insertions(+) create mode 100644 CHANGES/6002.misc diff --git a/CHANGES/6002.misc b/CHANGES/6002.misc new file mode 100644 index 00000000000..5df927cf65d --- /dev/null +++ b/CHANGES/6002.misc @@ -0,0 +1,2 @@ +Implemented end-to-end testing of sending HTTP and HTTPS requests +via ``proxy.py``. diff --git a/requirements/dev.txt b/requirements/dev.txt index 59ca566d7fe..9364ad351e6 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -163,6 +163,8 @@ pluggy==0.13.1 # pytest pre-commit==2.15.0 # via -r requirements/lint.txt +proxy.py==2.3.1 + # via -r requirements/test.txt py==1.10.0 # via # -r requirements/lint.txt @@ -277,6 +279,7 @@ typing-extensions==3.7.4.3 # -r requirements/lint.txt # async-timeout # mypy + # proxy.py uritemplate==3.0.1 # via gidgethub urllib3==1.26.5 diff --git a/requirements/test.txt b/requirements/test.txt index 8ba2b11d792..d5c4f2d271c 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -6,6 +6,7 @@ cryptography==3.3.1; platform_machine!="i686" and python_version<"3.9" # no 32-b freezegun==1.1.0 mypy==0.910; implementation_name=="cpython" mypy-extensions==0.4.3; implementation_name=="cpython" +proxy.py==2.3.1 pytest==6.2.2 pytest-cov==2.12.1 pytest-mock==3.6.1 diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py index e1c3c0095e7..bde34b2ba8f 100644 --- a/tests/test_proxy_functional.py +++ b/tests/test_proxy_functional.py @@ -5,12 +5,142 @@ from typing import Any from unittest import mock +import proxy import pytest from yarl import URL import aiohttp from aiohttp import web +ASYNCIO_SUPPORTS_TLS_IN_TLS = hasattr( + asyncio.sslproto._SSLProtocolTransport, + "_start_tls_compatible", +) + + +@pytest.fixture +def secure_proxy_url(monkeypatch, tls_certificate_pem_path): + """Return the URL of an instance of a running secure proxy. + + This fixture also spawns that instance and tears it down after the test. + """ + proxypy_args = [ + "--threadless", # use asyncio + "--num-workers", + "1", # the tests only send one query anyway + "--hostname", + "127.0.0.1", # network interface to listen to + "--port", + 0, # ephemeral port, so that kernel allocates a free one + "--cert-file", + tls_certificate_pem_path, # contains both key and cert + "--key-file", + tls_certificate_pem_path, # contains both key and cert + ] + + class PatchedAccetorPool(proxy.core.acceptor.AcceptorPool): + def listen(self): + super().listen() + self.socket_host, self.socket_port = self.socket.getsockname()[:2] + + monkeypatch.setattr(proxy.proxy, "AcceptorPool", PatchedAccetorPool) + + with proxy.Proxy(input_args=proxypy_args) as proxy_instance: + yield URL.build( + scheme="https", + host=proxy_instance.acceptors.socket_host, + port=proxy_instance.acceptors.socket_port, + ) + + +@pytest.fixture +def web_server_endpoint_payload(): + return "Test message" + + +@pytest.fixture(params=("http", "https")) +def web_server_endpoint_type(request): + return request.param + + +@pytest.fixture +async def web_server_endpoint_url( + aiohttp_server, + ssl_ctx, + web_server_endpoint_payload, + web_server_endpoint_type, +): + server_kwargs = ( + { + "ssl": ssl_ctx, + } + if web_server_endpoint_type == "https" + else {} + ) + + async def handler(*args, **kwargs): + return web.Response(text=web_server_endpoint_payload) + + app = web.Application() + app.router.add_route("GET", "/", handler) + server = await aiohttp_server(app, **server_kwargs) + + return URL.build( + scheme=web_server_endpoint_type, + host=server.host, + port=server.port, + ) + + +@pytest.fixture +def _pretend_asyncio_supports_tls_in_tls( + monkeypatch, + web_server_endpoint_type, +): + if web_server_endpoint_type != "https" or ASYNCIO_SUPPORTS_TLS_IN_TLS: + return + + # for https://github.com/python/cpython/pull/28073 + # and https://bugs.python.org/issue37179 + monkeypatch.setattr( + asyncio.sslproto._SSLProtocolTransport, + "_start_tls_compatible", + True, + raising=False, + ) + + +@pytest.mark.parametrize( + "web_server_endpoint_type", + ( + "http", + pytest.param("https", marks=pytest.mark.xfail), + ), +) +@pytest.mark.usefixtures("_pretend_asyncio_supports_tls_in_tls", "loop") +async def test_secure_https_proxy_absolute_path( + client_ssl_ctx, + secure_proxy_url, + web_server_endpoint_url, + web_server_endpoint_payload, +) -> None: + """Test urls can be requested through a secure proxy.""" + conn = aiohttp.TCPConnector() + sess = aiohttp.ClientSession(connector=conn) + + response = await sess.get( + web_server_endpoint_url, + proxy=secure_proxy_url, + ssl=client_ssl_ctx, # used for both proxy and endpoint connections + ) + + assert response.status == 200 + assert await response.text() == web_server_endpoint_payload + + response.close() + await sess.close() + await conn.close() + @pytest.fixture def proxy_test_server(aiohttp_raw_server: Any, loop: Any, monkeypatch: Any):