diff --git a/Makefile b/Makefile index f99e5fd594..791eef17b0 100644 --- a/Makefile +++ b/Makefile @@ -2,8 +2,7 @@ SHELL := /bin/bash NS ?= abhinavsingh IMAGE_NAME ?= proxy.py -#VERSION ?= v$(shell bash -c "python -m setuptools_scm --version \| awk '{print$3}'") -VERSION ?= v$(shell python -m setuptools_scm --version | awk '{print $$3}' | sed 's/\+/--/') +VERSION ?= v$(shell ./scm-version.sh) LATEST_TAG := $(NS)/$(IMAGE_NAME):latest IMAGE_TAG := $(NS)/$(IMAGE_NAME):$(VERSION) @@ -17,9 +16,9 @@ CA_CERT_FILE_PATH := ca-cert.pem CA_SIGNING_KEY_FILE_PATH := ca-signing-key.pem .PHONY: all https-certificates sign-https-certificates ca-certificates -.PHONY: lib-version lib-clean lib-test lib-package lib-coverage lib-lint lib-pytest +.PHONY: lib-check lib-clean lib-test lib-package lib-coverage lib-lint lib-pytest .PHONY: lib-release-test lib-release lib-profile -.PHONY: lib-dep, lib-flake8, lib-mypy +.PHONY: lib-dep, lib-flake8, lib-mypy, lib-scm-version .PHONY: container container-run container-release .PHONY: devtools dashboard dashboard-clean @@ -67,8 +66,8 @@ ca-certificates: python -m proxy.common.pki remove_passphrase \ --private-key-path $(CA_SIGNING_KEY_FILE_PATH) -lib-version: - python version-check.py +lib-check: + python check.py lib-clean: find . -name '*.pyc' -exec rm -f {} + @@ -89,6 +88,9 @@ lib-dep: -r requirements-release.txt \ -r requirements-tunnel.txt +lib-scm-version: + @echo "version = '$(VERSION)'" > proxy/common/_scm_version.py + lib-lint: python -m tox -e lint @@ -101,9 +103,9 @@ lib-mypy: lib-pytest: python -m tox -e python -- -v -lib-test: lib-clean lib-version lib-lint lib-pytest +lib-test: lib-clean lib-check lib-lint lib-pytest -lib-package: lib-clean lib-version +lib-package: lib-clean lib-check python -m tox -e cleanup-dists,build-dists,metadata-validation lib-release-test: lib-package diff --git a/README.md b/README.md index c56821487a..df5a812f75 100644 --- a/README.md +++ b/README.md @@ -192,6 +192,8 @@ - `http1.1` with pipeline - `http2` - `websockets` +- Support for `HAProxy Protocol` + - See `--enable-proxy-protocol` flag - Static file server support - See `--enable-static-server` and `--static-server-dir` flags - Optimized for large file uploads and downloads @@ -368,7 +370,7 @@ To start `proxy.py` from source code follow these instructions: - Install deps ```console - ❯ make lib-dep + ❯ make lib-dep lib-scm-version ``` - Optionally, run tests @@ -1945,144 +1947,176 @@ for list of tests. ```console ❯ proxy -h -usage: -m [-h] [--enable-events] [--enable-conn-pool] [--threadless] [--threaded] - [--num-workers NUM_WORKERS] [--backlog BACKLOG] [--hostname HOSTNAME] [--port PORT] - [--unix-socket-path UNIX_SOCKET_PATH] [--num-acceptors NUM_ACCEPTORS] [--version] - [--log-level LOG_LEVEL] [--log-file LOG_FILE] [--log-format LOG_FORMAT] - [--open-file-limit OPEN_FILE_LIMIT] [--plugins PLUGINS [PLUGINS ...]] - [--enable-dashboard] [--work-klass WORK_KLASS] [--pid-file PID_FILE] - [--client-recvbuf-size CLIENT_RECVBUF_SIZE] [--key-file KEY_FILE] [--timeout TIMEOUT] - [--server-recvbuf-size SERVER_RECVBUF_SIZE] [--disable-http-proxy] - [--disable-headers DISABLE_HEADERS] [--ca-key-file CA_KEY_FILE] - [--ca-cert-dir CA_CERT_DIR] [--ca-cert-file CA_CERT_FILE] [--ca-file CA_FILE] +usage: -m [-h] [--enable-events] [--enable-conn-pool] [--threadless] + [--threaded] [--num-workers NUM_WORKERS] [--backlog BACKLOG] + [--hostname HOSTNAME] [--port PORT] + [--unix-socket-path UNIX_SOCKET_PATH] + [--num-acceptors NUM_ACCEPTORS] [--version] [--log-level LOG_LEVEL] + [--log-file LOG_FILE] [--log-format LOG_FORMAT] + [--open-file-limit OPEN_FILE_LIMIT] + [--plugins PLUGINS [PLUGINS ...]] [--enable-dashboard] + [--work-klass WORK_KLASS] [--pid-file PID_FILE] + [--enable-proxy-protocol] + [--client-recvbuf-size CLIENT_RECVBUF_SIZE] [--key-file KEY_FILE] + [--timeout TIMEOUT] [--server-recvbuf-size SERVER_RECVBUF_SIZE] + [--disable-http-proxy] [--disable-headers DISABLE_HEADERS] + [--ca-key-file CA_KEY_FILE] [--ca-cert-dir CA_CERT_DIR] + [--ca-cert-file CA_CERT_FILE] [--ca-file CA_FILE] [--ca-signing-key-file CA_SIGNING_KEY_FILE] [--cert-file CERT_FILE] - [--auth-plugin AUTH_PLUGIN] [--basic-auth BASIC_AUTH] [--cache-dir CACHE_DIR] - [--filtered-upstream-hosts FILTERED_UPSTREAM_HOSTS] [--enable-web-server] - [--enable-static-server] [--static-server-dir STATIC_SERVER_DIR] - [--min-compression-length MIN_COMPRESSION_LENGTH] [--pac-file PAC_FILE] - [--pac-file-url-path PAC_FILE_URL_PATH] [--proxy-pool PROXY_POOL] + [--auth-plugin AUTH_PLUGIN] [--basic-auth BASIC_AUTH] + [--cache-dir CACHE_DIR] + [--filtered-upstream-hosts FILTERED_UPSTREAM_HOSTS] + [--enable-web-server] [--enable-static-server] + [--static-server-dir STATIC_SERVER_DIR] + [--min-compression-length MIN_COMPRESSION_LENGTH] + [--pac-file PAC_FILE] [--pac-file-url-path PAC_FILE_URL_PATH] + [--proxy-pool PROXY_POOL] [--filtered-client-ips FILTERED_CLIENT_IPS] [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG] [--cloudflare-dns-mode CLOUDFLARE_DNS_MODE] -proxy.py v2.4.0 +proxy.py v2.3.2 options: -h, --help show this help message and exit - --enable-events Default: False. Enables core to dispatch lifecycle events. Plugins can - be used to subscribe for core events. - --enable-conn-pool Default: False. (WIP) Enable upstream connection pooling. - --threadless Default: True. Enabled by default on Python 3.8+ (mac, linux). When - disabled a new thread is spawned to handle each client connection. - --threaded Default: False. Disabled by default on Python < 3.8 and windows. When - enabled a new thread is spawned to handle each client connection. + --enable-events Default: False. Enables core to dispatch lifecycle + events. Plugins can be used to subscribe for core + events. + --enable-conn-pool Default: False. (WIP) Enable upstream connection + pooling. + --threadless Default: True. Enabled by default on Python 3.8+ (mac, + linux). When disabled a new thread is spawned to + handle each client connection. + --threaded Default: False. Disabled by default on Python < 3.8 + and windows. When enabled a new thread is spawned to + handle each client connection. --num-workers NUM_WORKERS Defaults to number of CPU cores. - --backlog BACKLOG Default: 100. Maximum number of pending connections to proxy server + --backlog BACKLOG Default: 100. Maximum number of pending connections to + proxy server --hostname HOSTNAME Default: ::1. Server IP address. --port PORT Default: 8899. Server port. --unix-socket-path UNIX_SOCKET_PATH - Default: None. Unix socket path to use. When provided --host and --port - flags are ignored + Default: None. Unix socket path to use. When provided + --host and --port flags are ignored --num-acceptors NUM_ACCEPTORS Defaults to number of CPU cores. --version, -v Prints proxy.py version. --log-level LOG_LEVEL - Valid options: DEBUG, INFO (default), WARNING, ERROR, CRITICAL. Both - upper and lowercase values are allowed. You may also simply use the - leading character e.g. --log-level d + Valid options: DEBUG, INFO (default), WARNING, ERROR, + CRITICAL. Both upper and lowercase values are allowed. + You may also simply use the leading character e.g. + --log-level d --log-file LOG_FILE Default: sys.stdout. Log file destination. --log-format LOG_FORMAT Log format for Python logger. --open-file-limit OPEN_FILE_LIMIT - Default: 1024. Maximum number of files (TCP connections) that proxy.py - can open concurrently. + Default: 1024. Maximum number of files (TCP + connections) that proxy.py can open concurrently. --plugins PLUGINS [PLUGINS ...] - Comma separated plugins. You may use --plugins flag multiple times. + Comma separated plugins. You may use --plugins flag + multiple times. --enable-dashboard Default: False. Enables proxy.py dashboard. --work-klass WORK_KLASS - Default: proxy.http.HttpProtocolHandler. Work klass to use for work - execution. + Default: proxy.http.HttpProtocolHandler. Work klass to + use for work execution. --pid-file PID_FILE Default: None. Save "parent" process ID to a file. + --enable-proxy-protocol + Default: False. If used, will enable proxy protocol. + Only version 1 is currently supported. --client-recvbuf-size CLIENT_RECVBUF_SIZE - Default: 1 MB. Maximum amount of data received from the client in a - single recv() operation. Bump this value for faster uploads at the - expense of increased RAM. - --key-file KEY_FILE Default: None. Server key file to enable end-to-end TLS encryption with - clients. If used, must also pass --cert-file. - --timeout TIMEOUT Default: 10.0. Number of seconds after which an inactive connection must - be dropped. Inactivity is defined by no data sent or received by the - client. + Default: 1 MB. Maximum amount of data received from + the client in a single recv() operation. Bump this + value for faster uploads at the expense of increased + RAM. + --key-file KEY_FILE Default: None. Server key file to enable end-to-end + TLS encryption with clients. If used, must also pass + --cert-file. + --timeout TIMEOUT Default: 10.0. Number of seconds after which an + inactive connection must be dropped. Inactivity is + defined by no data sent or received by the client. --server-recvbuf-size SERVER_RECVBUF_SIZE - Default: 1 MB. Maximum amount of data received from the server in a - single recv() operation. Bump this value for faster downloads at the - expense of increased RAM. - --disable-http-proxy Default: False. Whether to disable proxy.HttpProxyPlugin. + Default: 1 MB. Maximum amount of data received from + the server in a single recv() operation. Bump this + value for faster downloads at the expense of increased + RAM. + --disable-http-proxy Default: False. Whether to disable + proxy.HttpProxyPlugin. --disable-headers DISABLE_HEADERS - Default: None. Comma separated list of headers to remove before - dispatching client request to upstream server. + Default: None. Comma separated list of headers to + remove before dispatching client request to upstream + server. --ca-key-file CA_KEY_FILE - Default: None. CA key to use for signing dynamically generated HTTPS - certificates. If used, must also pass --ca-cert-file and --ca-signing- - key-file + Default: None. CA key to use for signing dynamically + generated HTTPS certificates. If used, must also pass + --ca-cert-file and --ca-signing-key-file --ca-cert-dir CA_CERT_DIR - Default: ~/.proxy.py. Directory to store dynamically generated - certificates. Also see --ca-key-file, --ca-cert-file and --ca-signing- - key-file + Default: ~/.proxy.py. Directory to store dynamically + generated certificates. Also see --ca-key-file, --ca- + cert-file and --ca-signing-key-file --ca-cert-file CA_CERT_FILE - Default: None. Signing certificate to use for signing dynamically - generated HTTPS certificates. If used, must also pass --ca-key-file and - --ca-signing-key-file - --ca-file CA_FILE Default: /Users/abhinavsingh/Dev/proxy.py/venv310/lib/python3.10/site- - packages/certifi/cacert.pem. Provide path to custom CA bundle for peer - certificate verification + Default: None. Signing certificate to use for signing + dynamically generated HTTPS certificates. If used, + must also pass --ca-key-file and --ca-signing-key-file + --ca-file CA_FILE Default: /Users/abhinavsingh/Dev/proxy.py/venv310/lib/ + python3.10/site-packages/certifi/cacert.pem. Provide + path to custom CA bundle for peer certificate + verification --ca-signing-key-file CA_SIGNING_KEY_FILE - Default: None. CA signing key to use for dynamic generation of HTTPS - certificates. If used, must also pass --ca-key-file and --ca-cert-file + Default: None. CA signing key to use for dynamic + generation of HTTPS certificates. If used, must also + pass --ca-key-file and --ca-cert-file --cert-file CERT_FILE - Default: None. Server certificate to enable end-to-end TLS encryption - with clients. If used, must also pass --key-file. + Default: None. Server certificate to enable end-to-end + TLS encryption with clients. If used, must also pass + --key-file. --auth-plugin AUTH_PLUGIN - Default: proxy.http.proxy.AuthPlugin. Auth plugin to use instead of - default basic auth plugin. + Default: proxy.http.proxy.AuthPlugin. Auth plugin to + use instead of default basic auth plugin. --basic-auth BASIC_AUTH - Default: No authentication. Specify colon separated user:password to - enable basic authentication. + Default: No authentication. Specify colon separated + user:password to enable basic authentication. --cache-dir CACHE_DIR - Default: A temporary directory. Flag only applicable when cache plugin - is used with on-disk storage. + Default: A temporary directory. Flag only applicable + when cache plugin is used with on-disk storage. --filtered-upstream-hosts FILTERED_UPSTREAM_HOSTS - Default: Blocks Facebook. Comma separated list of IPv4 and IPv6 - addresses. - --enable-web-server Default: False. Whether to enable proxy.HttpWebServerPlugin. + Default: Blocks Facebook. Comma separated list of IPv4 + and IPv6 addresses. + --enable-web-server Default: False. Whether to enable + proxy.HttpWebServerPlugin. --enable-static-server - Default: False. Enable inbuilt static file server. Optionally, also use - --static-server-dir to serve static content from custom directory. By - default, static file server serves out of installed proxy.py python - module folder. + Default: False. Enable inbuilt static file server. + Optionally, also use --static-server-dir to serve + static content from custom directory. By default, + static file server serves out of installed proxy.py + python module folder. --static-server-dir STATIC_SERVER_DIR - Default: "public" folder in directory where proxy.py is placed. This - option is only applicable when static server is also enabled. See - --enable-static-server. + Default: "public" folder in directory where proxy.py + is placed. This option is only applicable when static + server is also enabled. See --enable-static-server. --min-compression-length MIN_COMPRESSION_LENGTH - Default: 20 bytes. Sets the minimum length of a response that will be - compressed (gzipped). - --pac-file PAC_FILE A file (Proxy Auto Configuration) or string to serve when the server - receives a direct file request. Using this option enables - proxy.HttpWebServerPlugin. + Default: 20 bytes. Sets the minimum length of a + response that will be compressed (gzipped). + --pac-file PAC_FILE A file (Proxy Auto Configuration) or string to serve + when the server receives a direct file request. Using + this option enables proxy.HttpWebServerPlugin. --pac-file-url-path PAC_FILE_URL_PATH Default: /. Web server path to serve the PAC file. --proxy-pool PROXY_POOL List of upstream proxies to use in the pool --filtered-client-ips FILTERED_CLIENT_IPS - Default: 127.0.0.1,::1. Comma separated list of IPv4 and IPv6 addresses. + Default: 127.0.0.1,::1. Comma separated list of IPv4 + and IPv6 addresses. --filtered-url-regex-config FILTERED_URL_REGEX_CONFIG - Default: No config. Comma separated list of IPv4 and IPv6 addresses. + Default: No config. Comma separated list of IPv4 and + IPv6 addresses. --cloudflare-dns-mode CLOUDFLARE_DNS_MODE - Default: security. Either "security" (for malware protection) or - "family" (for malware and adult content protection) + Default: security. Either "security" (for malware + protection) or "family" (for malware and adult content + protection) -Proxy.py not working? Report at: https://github.com/abhinavsingh/proxy.py/issues/new +Proxy.py not working? Report at: +https://github.com/abhinavsingh/proxy.py/issues/new ``` # Changelog diff --git a/check.py b/check.py new file mode 100644 index 0000000000..71bb51cf63 --- /dev/null +++ b/check.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import sys +import subprocess + +from pathlib import Path +from proxy.common.version import __version__ as lib_version + +# This script ensures our versions never run out of sync. +# +# 1. TODO: Version is hardcoded in homebrew stable package +# installer file, but it only needs to match with lib +# versions if current git branch is master + +PY_FILE_PREFIX = b'# -*- coding: utf-8 -*-\n' + \ + b'"""\n' + \ + b' proxy.py\n' + \ + b' ~~~~~~~~\n' + \ + b' \xe2\x9a\xa1\xe2\x9a\xa1\xe2\x9a\xa1 Fast, Lightweight, Pluggable, TLS interception capable' + \ + b' proxy server focused on\n' + \ + b' Network monitoring, controls & Application development, testing, debugging.\n' + \ + b'\n' + \ + b' :copyright: (c) 2013-present by Abhinav Singh and contributors.\n' + \ + b' :license: BSD, see LICENSE for more details.\n' + +REPO_ROOT = Path(__file__).parent +ALL_PY_FILES = ( + list(REPO_ROOT.glob('*.py')) + + list((REPO_ROOT / 'proxy').rglob('*.py')) + + list((REPO_ROOT / 'examples').rglob('*.py')) + + list((REPO_ROOT / 'tests').rglob('*.py')) +) + +# Ensure all python files start with licensing information +for py_file in ALL_PY_FILES: + if py_file.is_file() and py_file.name != '_scm_version.py': + with open(py_file, 'rb') as f: + code = f.read(len(PY_FILE_PREFIX)) + if code != PY_FILE_PREFIX: + print( + 'Expected license not found in {0}'.format( + str(py_file), + ), + ) + sys.exit(1) + +# Update README.md flags section to match current library --help output +# lib_help = subprocess.check_output( +# ['python', '-m', 'proxy', '-h'] +# ) +# with open('README.md', 'rb+') as f: +# c = f.read() +# pre_flags, post_flags = c.split(b'# Flags') +# help_text, post_changelog = post_flags.split(b'# Changelog') +# f.seek(0) +# f.write(pre_flags + b'# Flags\n\n```console\n\xe2\x9d\xaf proxy -h\n' + lib_help + b'```' + +# b'\n# Changelog' + post_changelog) + +# Version is also hardcoded in README.md flags section +readme_version_cmd = 'cat README.md | grep "proxy.py v" | tail -2 | head -1 | cut -d " " -f 2 | cut -c2-' +readme_version_output = subprocess.check_output( + ['bash', '-c', readme_version_cmd], +) +# Doesn't contain "v" prefix +readme_version = readme_version_output.decode().strip() + +if readme_version != lib_version[1:].split('-')[0]: + print( + 'Version mismatch found. {0} (readme) vs {1} (lib).'.format( + readme_version, lib_version, + ), + ) + sys.exit(1) diff --git a/helper/homebrew/develop/proxy.rb b/helper/homebrew/develop/proxy.rb index 37cff9a783..e4b30649c5 100644 --- a/helper/homebrew/develop/proxy.rb +++ b/helper/homebrew/develop/proxy.rb @@ -7,7 +7,7 @@ class Proxy < Formula url "https://github.com/abhinavsingh/proxy.py/archive/develop.zip" version "develop" - depends_on "python" + depends_on "python@3.10" def install virtualenv_install_with_resources diff --git a/helper/homebrew/stable/proxy.rb b/helper/homebrew/stable/proxy.rb index 7d0e3a9fc5..c91be85cc9 100644 --- a/helper/homebrew/stable/proxy.rb +++ b/helper/homebrew/stable/proxy.rb @@ -8,7 +8,7 @@ class Proxy < Formula sha256 "715687cebd451285d266f29d6509a64becc93da21f61ba9b4414e7dc4ecaaeed" version "2.3.1" - depends_on "python" + depends_on "python@3.10" def install virtualenv_install_with_resources diff --git a/proxy/common/_version.py b/proxy/common/_version.py index b1e36e6c57..2cc67b546a 100644 --- a/proxy/common/_version.py +++ b/proxy/common/_version.py @@ -1,5 +1,15 @@ -"""Version definition.""" +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. + + Version definition. +""" try: # pylint: disable=unused-import from ._scm_version import version as __version__ # noqa: WPS433, WPS436 diff --git a/proxy/common/constants.py b/proxy/common/constants.py index 98b4d62f4f..0451bf83c9 100644 --- a/proxy/common/constants.py +++ b/proxy/common/constants.py @@ -107,6 +107,7 @@ def _env_threadless_compliant() -> bool: DEFAULT_HTTPS_PORT = 443 DEFAULT_MAX_SEND_SIZE = 16 * 1024 DEFAULT_WORK_KLASS = 'proxy.http.HttpProtocolHandler' +DEFAULT_ENABLE_PROXY_PROTOCOL = False DEFAULT_DEVTOOLS_DOC_URL = 'http://proxy' DEFAULT_DEVTOOLS_FRAME_ID = secrets.token_hex(8) diff --git a/proxy/common/flag.py b/proxy/common/flag.py index 9a54ce76f4..5c3de444ef 100644 --- a/proxy/common/flag.py +++ b/proxy/common/flag.py @@ -352,7 +352,7 @@ def get_default_plugins( args: argparse.Namespace, ) -> List[str]: """Prepare list of plugins to load based upon - --enable-*, --disable-* and --basic-auth flags. + --enable-* and --disable-* flags. """ default_plugins: List[str] = [] if hasattr(args, 'enable_dashboard') and args.enable_dashboard: diff --git a/proxy/core/base/tcp_tunnel.py b/proxy/core/base/tcp_tunnel.py index 3a3b69041f..af46557c3a 100644 --- a/proxy/core/base/tcp_tunnel.py +++ b/proxy/core/base/tcp_tunnel.py @@ -38,7 +38,10 @@ class BaseTcpTunnelHandler(BaseTcpServerHandler): def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) - self.request = HttpParser(httpParserTypes.REQUEST_PARSER) + self.request = HttpParser( + httpParserTypes.REQUEST_PARSER, + enable_proxy_protocol=self.flags.enable_proxy_protocol, + ) self.upstream: Optional[TcpServerConnection] = None @abstractmethod diff --git a/proxy/dashboard/dashboard.py b/proxy/dashboard/dashboard.py index 7d2593539d..2a82fa828b 100644 --- a/proxy/dashboard/dashboard.py +++ b/proxy/dashboard/dashboard.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ proxy.py ~~~~~~~~ diff --git a/proxy/http/handler.py b/proxy/http/handler.py index 76a14b6c58..84cab4dabc 100644 --- a/proxy/http/handler.py +++ b/proxy/http/handler.py @@ -75,8 +75,10 @@ def __init__(self, *args: Any, **kwargs: Any): super().__init__(*args, **kwargs) self.start_time: float = time.time() self.last_activity: float = self.start_time - self.request: HttpParser = HttpParser(httpParserTypes.REQUEST_PARSER) - self.response: HttpParser = HttpParser(httpParserTypes.RESPONSE_PARSER) + self.request: HttpParser = HttpParser( + httpParserTypes.REQUEST_PARSER, + enable_proxy_protocol=self.flags.enable_proxy_protocol, + ) self.selector: Optional[selectors.DefaultSelector] = None if not is_threadless(self.flags.threadless, self.flags.threaded): self.selector = selectors.DefaultSelector() diff --git a/proxy/http/parser/__init__.py b/proxy/http/parser/__init__.py index dbde64a636..62e819e4f7 100644 --- a/proxy/http/parser/__init__.py +++ b/proxy/http/parser/__init__.py @@ -8,11 +8,13 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from .parser import HttpParser, httpParserTypes, httpParserStates +from .parser import HttpParser from .chunk import ChunkParser, chunkParserStates from .codes import httpStatusCodes -from .url import Url from .methods import httpMethods +from .types import httpParserStates, httpParserTypes +from .url import Url +from .protocol import ProxyProtocol, PROXY_PROTOCOL_V2_SIGNATURE __all__ = [ 'HttpParser', @@ -21,6 +23,8 @@ 'ChunkParser', 'chunkParserStates', 'httpStatusCodes', - 'Url', 'httpMethods', + 'Url', + 'ProxyProtocol', + 'PROXY_PROTOCOL_V2_SIGNATURE', ] diff --git a/proxy/http/parser/parser.py b/proxy/http/parser/parser.py index c2c785c086..912b0f4bed 100644 --- a/proxy/http/parser/parser.py +++ b/proxy/http/parser/parser.py @@ -8,35 +8,29 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from typing import TypeVar, NamedTuple, Optional, Dict, Type, Tuple, List +from typing import TypeVar, Optional, Dict, Type, Tuple, List -from ...common.constants import DEFAULT_DISABLE_HEADERS, COLON, HTTP_1_0, SLASH, CRLF -from ...common.constants import WHITESPACE, HTTP_1_1, DEFAULT_HTTP_PORT +from ...common.constants import DEFAULT_DISABLE_HEADERS, COLON, DEFAULT_ENABLE_PROXY_PROTOCOL +from ...common.constants import HTTP_1_1, HTTP_1_0, SLASH, CRLF +from ...common.constants import WHITESPACE, DEFAULT_HTTP_PORT from ...common.utils import build_http_request, build_http_response, find_http_line, text_ +from ...common.flag import flags from .url import Url from .methods import httpMethods +from .protocol import ProxyProtocol from .chunk import ChunkParser, chunkParserStates +from .types import httpParserTypes, httpParserStates -HttpParserStates = NamedTuple( - 'HttpParserStates', [ - ('INITIALIZED', int), - ('LINE_RCVD', int), - ('RCVING_HEADERS', int), - ('HEADERS_COMPLETE', int), - ('RCVING_BODY', int), - ('COMPLETE', int), - ], -) -httpParserStates = HttpParserStates(1, 2, 3, 4, 5, 6) -HttpParserTypes = NamedTuple( - 'HttpParserTypes', [ - ('REQUEST_PARSER', int), - ('RESPONSE_PARSER', int), - ], +flags.add_argument( + '--enable-proxy-protocol', + action='store_true', + default=DEFAULT_ENABLE_PROXY_PROTOCOL, + help='Default: ' + str(DEFAULT_ENABLE_PROXY_PROTOCOL) + '. ' + + 'If used, will enable proxy protocol. ' + + 'Only version 1 is currently supported.', ) -httpParserTypes = HttpParserTypes(1, 2) T = TypeVar('T', bound='HttpParser') @@ -55,9 +49,16 @@ class HttpParser: update parser to work accordingly. """ - def __init__(self, parser_type: int) -> None: - self.type: int = parser_type + def __init__( + self, parser_type: int, + enable_proxy_protocol: int = DEFAULT_ENABLE_PROXY_PROTOCOL, + ) -> None: self.state: int = httpParserStates.INITIALIZED + self.type: int = parser_type + self.protocol: Optional[ProxyProtocol] = None + if enable_proxy_protocol: + assert self.type == httpParserTypes.REQUEST_PARSER + self.protocol = ProxyProtocol() self.host: Optional[bytes] = None self.port: Optional[int] = None self.path: Optional[bytes] = None @@ -80,8 +81,15 @@ def __init__(self, parser_type: int) -> None: self._url: Optional[Url] = None @classmethod - def request(cls: Type[T], raw: bytes) -> T: - parser = cls(httpParserTypes.REQUEST_PARSER) + def request( + cls: Type[T], + raw: bytes, + enable_proxy_protocol: int = DEFAULT_ENABLE_PROXY_PROTOCOL, + ) -> T: + parser = cls( + httpParserTypes.REQUEST_PARSER, + enable_proxy_protocol=enable_proxy_protocol, + ) parser.parse(raw) return parser @@ -165,8 +173,7 @@ def body_expected(self) -> bool: def parse(self, raw: bytes) -> None: """Parses Http request out of raw bytes. - Check for `HttpParser.state` after `parse` has successfully returned. - """ + Check for `HttpParser.state` after `parse` has successfully returned.""" self.total_size += len(raw) raw = self.buffer + raw self.buffer, more = b'', len(raw) > 0 @@ -267,7 +274,8 @@ def _process_line_and_headers(self, raw: bytes) -> Tuple[bool, bytes]: if self.state == httpParserStates.INITIALIZED: self._process_line(line) - self.state = httpParserStates.LINE_RCVD + if self.state == httpParserStates.INITIALIZED: + return len(raw) > 0, raw elif self.state in (httpParserStates.LINE_RCVD, httpParserStates.RCVING_HEADERS): if self.state == httpParserStates.LINE_RCVD: # LINE_RCVD state is equivalent to RCVING_HEADERS @@ -291,15 +299,23 @@ def _process_line_and_headers(self, raw: bytes) -> Tuple[bool, bytes]: return len(raw) > 0, raw def _process_line(self, raw: bytes) -> None: - line = raw.split(WHITESPACE) if self.type == httpParserTypes.REQUEST_PARSER: - self.method = line[0].upper() - self.set_url(line[1]) - self.version = line[2] + if self.protocol is not None and self.protocol.version is None: + # We expect to receive entire proxy protocol v1 line + # in one network read and don't expect partial packets + self.protocol.parse(raw) + else: + line = raw.split(WHITESPACE) + self.method = line[0].upper() + self.set_url(line[1]) + self.version = line[2] + self.state = httpParserStates.LINE_RCVD else: + line = raw.split(WHITESPACE) self.version = line[0] self.code = line[1] self.reason = WHITESPACE.join(line[2:]) + self.state = httpParserStates.LINE_RCVD def _process_header(self, raw: bytes) -> None: parts = raw.split(COLON) diff --git a/proxy/http/parser/protocol.py b/proxy/http/parser/protocol.py new file mode 100644 index 0000000000..0c3623dd27 --- /dev/null +++ b/proxy/http/parser/protocol.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +from typing import Optional, Tuple +from ...common.constants import WHITESPACE + +PROXY_PROTOCOL_V2_SIGNATURE = b'\x0D\x0A\x0D\x0A\x00\x0D\x0A\x51\x55\x49\x54\x0A' + + +class ProxyProtocol: + """Reference https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt""" + + def __init__(self) -> None: + self.version: Optional[int] = None + self.family: Optional[bytes] = None + self.source: Optional[Tuple[bytes, int]] = None + self.destination: Optional[Tuple[bytes, int]] = None + + def parse(self, raw: bytes) -> None: + if raw.startswith(b'PROXY'): + self.version = 1 + # Per spec, v1 line cannot exceed this limit + assert len(raw) <= 57 + line = raw.split(WHITESPACE) + assert line[0] == b'PROXY' and line[1] in ( + b'TCP4', b'TCP6', b'UNKNOWN', + ) + self.family = line[1] + if len(line) == 6: + self.source = (line[2], int(line[4])) + self.destination = (line[3], int(line[5])) + else: + assert self.family == b'UNKNOWN' + elif raw.startswith(PROXY_PROTOCOL_V2_SIGNATURE): + self.version = 2 + raise NotImplementedError() + else: + raise ValueError('Neither a v1 or v2 proxy protocol packet') diff --git a/proxy/http/parser/types.py b/proxy/http/parser/types.py new file mode 100644 index 0000000000..582b22921b --- /dev/null +++ b/proxy/http/parser/types.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +from typing import NamedTuple + + +HttpParserStates = NamedTuple( + 'HttpParserStates', [ + ('INITIALIZED', int), + ('LINE_RCVD', int), + ('RCVING_HEADERS', int), + ('HEADERS_COMPLETE', int), + ('RCVING_BODY', int), + ('COMPLETE', int), + ], +) +httpParserStates = HttpParserStates(1, 2, 3, 4, 5, 6) + +HttpParserTypes = NamedTuple( + 'HttpParserTypes', [ + ('REQUEST_PARSER', int), + ('RESPONSE_PARSER', int), + ], +) +httpParserTypes = HttpParserTypes(1, 2) diff --git a/proxy/http/proxy/server.py b/proxy/http/proxy/server.py index 1916337a1e..562b94af11 100644 --- a/proxy/http/proxy/server.py +++ b/proxy/http/proxy/server.py @@ -25,12 +25,12 @@ from ..parser import HttpParser, httpParserStates, httpParserTypes, httpStatusCodes, httpMethods from ...common.types import Readables, Writables -from ...common.constants import DEFAULT_CA_CERT_DIR, DEFAULT_CA_CERT_FILE, DEFAULT_CA_FILE, PLUGIN_PROXY_AUTH +from ...common.constants import DEFAULT_CA_CERT_DIR, DEFAULT_CA_CERT_FILE, DEFAULT_CA_FILE from ...common.constants import DEFAULT_CA_KEY_FILE, DEFAULT_CA_SIGNING_KEY_FILE from ...common.constants import COMMA, DEFAULT_SERVER_RECVBUF_SIZE, DEFAULT_CERT_FILE from ...common.constants import PROXY_AGENT_HEADER_VALUE, DEFAULT_DISABLE_HEADERS from ...common.constants import DEFAULT_HTTP_ACCESS_LOG_FORMAT, DEFAULT_HTTPS_ACCESS_LOG_FORMAT -from ...common.constants import DEFAULT_DISABLE_HTTP_PROXY +from ...common.constants import DEFAULT_DISABLE_HTTP_PROXY, PLUGIN_PROXY_AUTH from ...common.utils import build_http_response, text_ from ...common.pki import gen_public_key, gen_csr, sign_csr @@ -348,6 +348,28 @@ def on_client_connection_close(self) -> None: 'response_code': text_(self.response.code), 'response_reason': text_(self.response.reason), } + if self.flags.enable_proxy_protocol: + assert self.request.protocol and self.request.protocol.family + context.update({ + 'protocol': { + 'family': text_(self.request.protocol.family), + }, + }) + if self.request.protocol.source: + context.update({ + 'protocol': { + 'source_ip': text_(self.request.protocol.source[0]), + 'source_port': self.request.protocol.source[1], + }, + }) + if self.request.protocol.destination: + context.update({ + 'protocol': { + 'destination_ip': text_(self.request.protocol.destination[0]), + 'destination_port': self.request.protocol.destination[1], + }, + }) + log_handled = False for plugin in self.plugins.values(): ctx = plugin.on_access_log(context) @@ -453,6 +475,9 @@ def on_client_data(self, raw: memoryview) -> Optional[memoryview]: return None if self.pipeline_request is None: + # For pipeline requests, we never + # want to use --enable-proxy-protocol flag + # as proxy protocol header will not be present self.pipeline_request = HttpParser( httpParserTypes.REQUEST_PARSER, ) diff --git a/scm-version.sh b/scm-version.sh new file mode 100755 index 0000000000..509579136c --- /dev/null +++ b/scm-version.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Guessed Version 2.3.2.dev146+gad54132.d20211114 +python -m setuptools_scm --version | \ + # 2.3.2.dev146+gad54132.d20211114 + awk '{print $3}' | \ + # 2.3.2.dev146-gad54132.d20211114 + sed 's/\+/-/' | \ + # 2.3.2.dev146-gad54132-d20211114 + sed -E 's/(.*)\./\1-/' | \ + # 2.3.2-dev146-gad54132-d20211114 + sed -E 's/(.*)\./\1-/' | \ + # 2.3.2-dev146-gad54132.d20211114 + sed -E 's/(.*)-/\1\./' | \ + # 2.3.2-dev146.gad54132.d20211114 + sed -E 's/(.*)-/\1\./' diff --git a/tests/http/test_http_parser.py b/tests/http/test_http_parser.py index 5cde14939a..d7c47f73ee 100644 --- a/tests/http/test_http_parser.py +++ b/tests/http/test_http_parser.py @@ -717,3 +717,27 @@ def test_response_factory(self) -> None: self.assertEqual(r.code, b'200') self.assertEqual(r.reason, b'OK') self.assertEqual(r.header(b'key'), b'value') + + def test_proxy_protocol(self) -> None: + r = HttpParser.request( + b'PROXY TCP4 192.168.0.1 192.168.0.11 56324 443' + CRLF + + b'GET / HTTP/1.1' + CRLF + + b'Host: 192.168.0.11' + CRLF + CRLF, + enable_proxy_protocol=True, + ) + self.assertTrue(r.protocol is not None) + assert r.protocol and r.protocol.version and \ + r.protocol.family and \ + r.protocol.source and \ + r.protocol.destination + self.assertEqual(r.protocol.version, 1) + self.assertEqual(r.protocol.family, b'TCP4') + self.assertEqual(r.protocol.source, (b'192.168.0.1', 56324)) + self.assertEqual(r.protocol.destination, (b'192.168.0.11', 443)) + + def test_proxy_protocol_not_for_response_parser(self) -> None: + with self.assertRaises(AssertionError): + HttpParser( + httpParserTypes.RESPONSE_PARSER, + enable_proxy_protocol=True, + ) diff --git a/tests/http/test_proxy_protocol.py b/tests/http/test_proxy_protocol.py new file mode 100644 index 0000000000..b6701abfb7 --- /dev/null +++ b/tests/http/test_proxy_protocol.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import unittest + +from proxy.http.parser import ProxyProtocol, PROXY_PROTOCOL_V2_SIGNATURE + + +class TestProxyProtocol(unittest.TestCase): + + def setUp(self) -> None: + self.protocol = ProxyProtocol() + + def test_v1(self) -> None: + self.protocol.parse(b'PROXY TCP6 ::1 ::1 64665 8899') + self.assertEqual(self.protocol.version, 1) + self.assertEqual(self.protocol.family, b'TCP6') + self.assertEqual(self.protocol.source, (b'::1', 64665)) + self.assertEqual(self.protocol.destination, (b'::1', 8899)) + + def test_v1_example_from_spec(self) -> None: + self.protocol.parse(b'PROXY TCP4 192.168.0.1 192.168.0.11 56324 443') + self.assertEqual(self.protocol.version, 1) + self.assertEqual(self.protocol.family, b'TCP4') + self.assertEqual(self.protocol.source, (b'192.168.0.1', 56324)) + self.assertEqual(self.protocol.destination, (b'192.168.0.11', 443)) + + def test_v1_worst_case_ipv4_from_spec(self) -> None: + self.protocol.parse( + b'PROXY TCP4 255.255.255.255 255.255.255.255 65535 65535', + ) + self.assertEqual(self.protocol.version, 1) + self.assertEqual(self.protocol.family, b'TCP4') + self.assertEqual(self.protocol.source, (b'255.255.255.255', 65535)) + self.assertEqual( + self.protocol.destination, + (b'255.255.255.255', 65535), + ) + + def test_v1_worst_case_ipv6_from_spec(self) -> None: + self.protocol.parse( + b'PROXY TCP6 ffff:f...f:ffff ffff:f...f:ffff 65535 65535', + ) + self.assertEqual(self.protocol.version, 1) + self.assertEqual(self.protocol.family, b'TCP6') + self.assertEqual(self.protocol.source, (b'ffff:f...f:ffff', 65535)) + self.assertEqual( + self.protocol.destination, + (b'ffff:f...f:ffff', 65535), + ) + + def test_v1_worst_case_unknown_from_spec(self) -> None: + self.protocol.parse( + b'PROXY UNKNOWN ffff:f...f:ffff ffff:f...f:ffff 65535 65535', + ) + self.assertEqual(self.protocol.version, 1) + self.assertEqual(self.protocol.family, b'UNKNOWN') + self.assertEqual(self.protocol.source, (b'ffff:f...f:ffff', 65535)) + self.assertEqual( + self.protocol.destination, + (b'ffff:f...f:ffff', 65535), + ) + + def test_v1_unknown_with_no_src_dst(self) -> None: + self.protocol.parse(b'PROXY UNKNOWN') + self.assertEqual(self.protocol.version, 1) + self.assertEqual(self.protocol.family, b'UNKNOWN') + self.assertEqual(self.protocol.source, None) + self.assertEqual(self.protocol.destination, None) + + def test_v2_not_implemented(self) -> None: + with self.assertRaises(NotImplementedError): + self.protocol.parse(PROXY_PROTOCOL_V2_SIGNATURE) + self.assertEqual(self.protocol.version, 2) + + def test_unknown_value_error(self) -> None: + with self.assertRaises(ValueError): + self.protocol.parse(PROXY_PROTOCOL_V2_SIGNATURE[:10]) + self.assertEqual(self.protocol.version, None) diff --git a/version-check.py b/version-check.py deleted file mode 100644 index a957a40e02..0000000000 --- a/version-check.py +++ /dev/null @@ -1,34 +0,0 @@ -# -*- coding: utf-8 -*- -""" - proxy.py - ~~~~~~~~ - ⚡⚡⚡ Fast, Lightweight, Programmable, TLS interception capable - proxy server for Application debugging, testing and development. - - :copyright: (c) 2013-present by Abhinav Singh and contributors. - :license: BSD, see LICENSE for more details. -""" -import sys -import subprocess -from proxy.common.version import __version__ as lib_version - -# This script ensures our versions never run out of sync. -# -# 1. TODO: Version is hardcoded in homebrew stable package -# installer file, but it only needs to match with lib -# versions if current git branch is master - -# Version is also hardcoded in README.md flags section -readme_version_cmd = 'cat README.md | grep "proxy.py v" | tail -2 | head -1 | cut -d " " -f 2 | cut -c2-' -readme_version_output = subprocess.check_output( - ['bash', '-c', readme_version_cmd], -) -readme_version = readme_version_output.decode().strip() - -if readme_version != lib_version: - print( - 'Version mismatch found. {0} (readme) vs {1} (lib).'.format( - readme_version, lib_version, - ), - ) - sys.exit(1)