Add xfailing integration tests against proxy.py

This patch adds full end-to-end tests for sending requests to HTTP and
HTTPS endpoints through an HTTPS proxy. The first case is currently
supported and the second one is not. This is why the latter test is
marked as expected to fail. The support for TLS-in-TLS in the upstream
stdlib asyncio is currently disabled but is available in Python 3.9
via monkey-patching which is demonstrated in the added tests.

Refs:
* https://bugs.python.org/issue37179
* python/cpython#28073
* #5992

Co-Authored-By: Sviatoslav Sydorenko <webknjaz@redhat.com>

CHANGES/6002.misc

@@ -0,0 +1,2 @@
+Implemented end-to-end testing of sending HTTP and HTTPS requests
+via ``proxy.py``.

requirements/dev.txt

@@ -163,6 +163,8 @@ pluggy==0.13.1
     #   pytest
 pre-commit==2.15.0
     # via -r requirements/lint.txt
+proxy.py==2.3.1
+    # via -r requirements/test.txt
 py==1.10.0
     # via
     #   -r requirements/lint.txt
@@ -277,6 +279,7 @@ typing-extensions==3.7.4.3
     #   -r requirements/lint.txt
     #   async-timeout
     #   mypy
+    #   proxy.py
 uritemplate==3.0.1
     # via gidgethub
 urllib3==1.26.5

requirements/test.txt

@@ -6,6 +6,7 @@ cryptography==3.3.1; platform_machine!="i686" and python_version<"3.9" # no 32-b
 freezegun==1.1.0
 mypy==0.910; implementation_name=="cpython"
 mypy-extensions==0.4.3; implementation_name=="cpython"
+proxy.py==2.3.1
 pytest==6.2.2
 pytest-cov==2.12.1
 pytest-mock==3.6.1

tests/test_proxy_functional.py

@@ -5,12 +5,142 @@
 from typing import Any
 from unittest import mock
 
+import proxy
 import pytest
 from yarl import URL
 
 import aiohttp
 from aiohttp import web
 
+ASYNCIO_SUPPORTS_TLS_IN_TLS = hasattr(
+    asyncio.sslproto._SSLProtocolTransport,
+    "_start_tls_compatible",
+)
+
+
+@pytest.fixture
+def secure_proxy_url(monkeypatch, tls_certificate_pem_path):
+    """Return the URL of an instance of a running secure proxy.
+
+    This fixture also spawns that instance and tears it down after the test.
+    """
+    proxypy_args = [
+        "--threadless",  # use asyncio
+        "--num-workers",
+        "1",  # the tests only send one query anyway
+        "--hostname",
+        "127.0.0.1",  # network interface to listen to
+        "--port",
+        0,  # ephemeral port, so that kernel allocates a free one
+        "--cert-file",
+        tls_certificate_pem_path,  # contains both key and cert
+        "--key-file",
+        tls_certificate_pem_path,  # contains both key and cert
+    ]
+
+    class PatchedAccetorPool(proxy.core.acceptor.AcceptorPool):
+        def listen(self):
+            super().listen()
+            self.socket_host, self.socket_port = self.socket.getsockname()[:2]
+
+    monkeypatch.setattr(proxy.proxy, "AcceptorPool", PatchedAccetorPool)
+
+    with proxy.Proxy(input_args=proxypy_args) as proxy_instance:
+        yield URL.build(
+            scheme="https",
+            host=proxy_instance.acceptors.socket_host,
+            port=proxy_instance.acceptors.socket_port,
+        )
+
+
+@pytest.fixture
+def web_server_endpoint_payload():
+    return "Test message"
+
+
+@pytest.fixture(params=("http", "https"))
+def web_server_endpoint_type(request):
+    return request.param
+
+
+@pytest.fixture
+async def web_server_endpoint_url(
+    aiohttp_server,
+    ssl_ctx,
+    web_server_endpoint_payload,
+    web_server_endpoint_type,
+):
+    server_kwargs = (
+        {
+            "ssl": ssl_ctx,
+        }
+        if web_server_endpoint_type == "https"
+        else {}
+    )
+
+    async def handler(*args, **kwargs):
+        return web.Response(text=web_server_endpoint_payload)
+
+    app = web.Application()
+    app.router.add_route("GET", "/", handler)
+    server = await aiohttp_server(app, **server_kwargs)
+
+    return URL.build(
+        scheme=web_server_endpoint_type,
+        host=server.host,
+        port=server.port,
+    )
+
+
+@pytest.fixture
+def _pretend_asyncio_supports_tls_in_tls(
+    monkeypatch,
+    web_server_endpoint_type,
+):
+    if web_server_endpoint_type != "https" or ASYNCIO_SUPPORTS_TLS_IN_TLS:
+        return
+
+    # for https://github.com/python/cpython/pull/28073
+    # and https://bugs.python.org/issue37179
+    monkeypatch.setattr(
+        asyncio.sslproto._SSLProtocolTransport,
+        "_start_tls_compatible",
+        True,
+        raising=False,
+    )
+
+
+@pytest.mark.parametrize(
+    "web_server_endpoint_type",
+    (
+        "http",
+        pytest.param("https", marks=pytest.mark.xfail),
+    ),
+)
+@pytest.mark.usefixtures("_pretend_asyncio_supports_tls_in_tls", "loop")
+async def test_secure_https_proxy_absolute_path(
+    client_ssl_ctx,
+    secure_proxy_url,
+    web_server_endpoint_url,
+    web_server_endpoint_payload,
+) -> None:
+    """Test urls can be requested through a secure proxy."""
+    conn = aiohttp.TCPConnector()
+    sess = aiohttp.ClientSession(connector=conn)
+
+    response = await sess.get(
+        web_server_endpoint_url,
+        proxy=secure_proxy_url,
+        ssl=client_ssl_ctx,  # used for both proxy and endpoint connections
+    )
+
+    assert response.status == 200
+    assert await response.text() == web_server_endpoint_payload
+
+    response.close()
+    await sess.close()
+    await conn.close()
+
 
 @pytest.fixture
 def proxy_test_server(aiohttp_raw_server: Any, loop: Any, monkeypatch: Any):