connection leaking in some network condition

[everAwayne]

python 3.6.9
httpx 0.16.0
httpcore 0.12.0

httpcore_async\http_proxy.py line206 - line244
If any exception(like ReadTimeout) raise between socket establishing and adding connection to pool, connection will leak.

I set read timeout to 1s, connect timeout to 15s.

code here:

import os
os.environ["HTTPX_LOG_LEVEL"] = "TRACE"

import asyncio
import httpx
from httpx._config import Timeout


async def run():
    try:
        async with httpx.AsyncClient(proxies="some proxy address") as client:
            r = await client.get('https://www.baidu.com/', timeout=Timeout(5, connect=15, read=1))
            await asyncio.sleep(40)
            rasie
    except Exception as exc:
        print(type(exc))
        await asyncio.sleep(40)


if __name__ == '__main__':
    loop = asyncio.get_event_loop()
    loop.run_until_complete(run())

log here:

after AsyncClient is closed, socket is still there.

conclusion:
Even if AsyncClient.aexit has executed, but socket remain ESTABLISHED, because exception raise before calling
AsyncConnectionPool._add_to_pool, AsyncConnectionPool._connections is empty.

[cdeler]

Hello @everAwayne ,

Yes, it's probably a bug in httpcore.

I wrote a test case to make it reproducible. In the test case we try to access the domain, which is in our proxy blacklist.

@pytest.mark.trio
async def test_socket_does_not_leak_from_the_connection_has_not_been_added_to_pool(
    proxy_server: URL,
    server: Server,
):
    method = b"GET"
    url = (b"http", b"example.org", 80, b"/")
    headers = [(b"host", b"example.org")]
    proxy_mode = "TUNNEL_ONLY"

    with pytest.warns(ResourceWarning) as recorded_warnings:
        async with httpcore.AsyncHTTPProxy(proxy_server, proxy_mode=proxy_mode) as http:
            for _ in range(100):
                try:
                    _ = await http.arequest(method, url, headers)
                except httpcore.RemoteProtocolError:
                    pass

    assert len(recorded_warnings.list) == 0

To make the test case effective we need to run pproxy with a "blocked domains" list:

# test/utils.py changes
@contextlib.contextmanager
def http_proxy_server(proxy_host: str, proxy_port: int):

    proc = None

    with create_proxy_block_file(["example.org"]) as block_file_name:
        try:
            command = [
                "pproxy",
                "-b",
                block_file_name,
                "-l",
                f"http://{proxy_host}:{proxy_port}/",
            ]
            proc = subprocess.Popen(command)

            _wait_can_connect(proxy_host, proxy_port)

            yield b"http", proxy_host.encode(), proxy_port, b"/"
        finally:
            if proc is not None:
                proc.kill()


@contextlib.contextmanager
def create_proxy_block_file(blocked_domains: List[str]):
    with tempfile.NamedTemporaryFile(delete=True, mode="w+") as file:

        for domain in blocked_domains:
            file.write(domain)
            file.write("\n")

        file.flush()

        yield file.name