--enable-proxy-protocol : HAProxy Protocol v1 (#735)

* Introduce `--haproxy-protocol` flag * Complete proxy protocol v1 implementation, enable using `--enable-proxy-protocol` flag * link checks * Advertise support for haproxy protocol in readme * Add make target `lib-scm-version` * `make lib-version` is now `make lib-check` * Dont enforce -dev part of version within README * Add provision to update readme flags using check * Wrap help text within console * Add closing ticks * Remove verbose logging and update homebrew formulae (may be fixed?)
abhinavsingh · Nov 14, 2021 · d72ee22 · d72ee22
1 parent 8d3fe87
commit d72ee22
Show file tree

Hide file tree

Showing 20 changed files with 524 additions and 176 deletions.
diff --git a/Makefile b/Makefile
@@ -2,8 +2,7 @@ SHELL := /bin/bash
 
 NS ?= abhinavsingh
 IMAGE_NAME ?= proxy.py
-#VERSION ?= v$(shell bash -c "python -m setuptools_scm --version \| awk '{print$3}'")
-VERSION ?= v$(shell python -m setuptools_scm --version | awk '{print $$3}' | sed 's/\+/--/')
+VERSION ?= v$(shell ./scm-version.sh)
 LATEST_TAG := $(NS)/$(IMAGE_NAME):latest
 IMAGE_TAG := $(NS)/$(IMAGE_NAME):$(VERSION)
 
@@ -17,9 +16,9 @@ CA_CERT_FILE_PATH := ca-cert.pem
 CA_SIGNING_KEY_FILE_PATH := ca-signing-key.pem
 
 .PHONY: all https-certificates sign-https-certificates ca-certificates
-.PHONY: lib-version lib-clean lib-test lib-package lib-coverage lib-lint lib-pytest
+.PHONY: lib-check lib-clean lib-test lib-package lib-coverage lib-lint lib-pytest
 .PHONY: lib-release-test lib-release lib-profile
-.PHONY: lib-dep, lib-flake8, lib-mypy
+.PHONY: lib-dep, lib-flake8, lib-mypy, lib-scm-version
 .PHONY: container container-run container-release
 .PHONY: devtools dashboard dashboard-clean
 
@@ -67,8 +66,8 @@ ca-certificates:
 	python -m proxy.common.pki remove_passphrase \
 		--private-key-path $(CA_SIGNING_KEY_FILE_PATH)
 
-lib-version:
-	python version-check.py
+lib-check:
+	python check.py
 
 lib-clean:
 	find . -name '*.pyc' -exec rm -f {} +
@@ -89,6 +88,9 @@ lib-dep:
 		-r requirements-release.txt \
 		-r requirements-tunnel.txt
 
+lib-scm-version:
+	@echo "version = '$(VERSION)'" > proxy/common/_scm_version.py
+
 lib-lint:
 	python -m tox -e lint
 
@@ -101,9 +103,9 @@ lib-mypy:
 lib-pytest:
 	python -m tox -e python -- -v
 
-lib-test: lib-clean lib-version lib-lint lib-pytest
+lib-test: lib-clean lib-check lib-lint lib-pytest
 
-lib-package: lib-clean lib-version
+lib-package: lib-clean lib-check
 	python -m tox -e cleanup-dists,build-dists,metadata-validation
 
 lib-release-test: lib-package

diff --git a/README.md b/README.md
diff --git a/check.py b/check.py
@@ -0,0 +1,81 @@
+# -*- coding: utf-8 -*-
+"""
+    proxy.py
+    ~~~~~~~~
+    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
+    Network monitoring, controls & Application development, testing, debugging.
+
+    :copyright: (c) 2013-present by Abhinav Singh and contributors.
+    :license: BSD, see LICENSE for more details.
+"""
+import sys
+import subprocess
+
+from pathlib import Path
+from proxy.common.version import __version__ as lib_version
+
+# This script ensures our versions never run out of sync.
+#
+# 1. TODO: Version is hardcoded in homebrew stable package
+#    installer file, but it only needs to match with lib
+#    versions if current git branch is master
+
+PY_FILE_PREFIX = b'# -*- coding: utf-8 -*-\n' + \
+    b'"""\n' + \
+    b'    proxy.py\n' + \
+    b'    ~~~~~~~~\n' + \
+    b'    \xe2\x9a\xa1\xe2\x9a\xa1\xe2\x9a\xa1 Fast, Lightweight, Pluggable, TLS interception capable' + \
+    b' proxy server focused on\n' + \
+    b'    Network monitoring, controls & Application development, testing, debugging.\n' + \
+    b'\n' + \
+    b'    :copyright: (c) 2013-present by Abhinav Singh and contributors.\n' + \
+    b'    :license: BSD, see LICENSE for more details.\n'
+
+REPO_ROOT = Path(__file__).parent
+ALL_PY_FILES = (
+    list(REPO_ROOT.glob('*.py')) +
+    list((REPO_ROOT / 'proxy').rglob('*.py')) +
+    list((REPO_ROOT / 'examples').rglob('*.py')) +
+    list((REPO_ROOT / 'tests').rglob('*.py'))
+)
+
+# Ensure all python files start with licensing information
+for py_file in ALL_PY_FILES:
+    if py_file.is_file() and py_file.name != '_scm_version.py':
+        with open(py_file, 'rb') as f:
+            code = f.read(len(PY_FILE_PREFIX))
+            if code != PY_FILE_PREFIX:
+                print(
+                    'Expected license not found in {0}'.format(
+                        str(py_file),
+                    ),
+                )
+                sys.exit(1)
+
+# Update README.md flags section to match current library --help output
+# lib_help = subprocess.check_output(
+#     ['python', '-m', 'proxy', '-h']
+# )
+# with open('README.md', 'rb+') as f:
+#     c = f.read()
+#     pre_flags, post_flags = c.split(b'# Flags')
+#     help_text, post_changelog = post_flags.split(b'# Changelog')
+#     f.seek(0)
+#     f.write(pre_flags + b'# Flags\n\n```console\n\xe2\x9d\xaf proxy -h\n' + lib_help + b'```' +
+#             b'\n# Changelog' + post_changelog)
+
+# Version is also hardcoded in README.md flags section
+readme_version_cmd = 'cat README.md | grep "proxy.py v" | tail -2 | head -1 | cut -d " " -f 2 | cut -c2-'
+readme_version_output = subprocess.check_output(
+    ['bash', '-c', readme_version_cmd],
+)
+# Doesn't contain "v" prefix
+readme_version = readme_version_output.decode().strip()
+
+if readme_version != lib_version[1:].split('-')[0]:
+    print(
+        'Version mismatch found. {0} (readme) vs {1} (lib).'.format(
+            readme_version, lib_version,
+        ),
+    )
+    sys.exit(1)
diff --git a/helper/homebrew/develop/proxy.rb b/helper/homebrew/develop/proxy.rb
@@ -7,7 +7,7 @@ class Proxy < Formula
   url "https://github.com/abhinavsingh/proxy.py/archive/develop.zip"
   version "develop"
 
-  depends_on "python"
+  depends_on "python@3.10"
 
   def install
     virtualenv_install_with_resources

diff --git a/helper/homebrew/stable/proxy.rb b/helper/homebrew/stable/proxy.rb
@@ -8,7 +8,7 @@ class Proxy < Formula
   sha256 "715687cebd451285d266f29d6509a64becc93da21f61ba9b4414e7dc4ecaaeed"
   version "2.3.1"
 
-  depends_on "python"
+  depends_on "python@3.10"
 
   def install
     virtualenv_install_with_resources

diff --git a/proxy/common/_version.py b/proxy/common/_version.py
@@ -1,5 +1,15 @@
-"""Version definition."""
+# -*- coding: utf-8 -*-
+"""
+    proxy.py
+    ~~~~~~~~
+    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
+    Network monitoring, controls & Application development, testing, debugging.
 
+    :copyright: (c) 2013-present by Abhinav Singh and contributors.
+    :license: BSD, see LICENSE for more details.
+
+    Version definition.
+"""
 try:
     # pylint: disable=unused-import
     from ._scm_version import version as __version__  # noqa: WPS433, WPS436

diff --git a/proxy/common/constants.py b/proxy/common/constants.py
@@ -107,6 +107,7 @@ def _env_threadless_compliant() -> bool:
 DEFAULT_HTTPS_PORT = 443
 DEFAULT_MAX_SEND_SIZE = 16 * 1024
 DEFAULT_WORK_KLASS = 'proxy.http.HttpProtocolHandler'
+DEFAULT_ENABLE_PROXY_PROTOCOL = False
 
 DEFAULT_DEVTOOLS_DOC_URL = 'http://proxy'
 DEFAULT_DEVTOOLS_FRAME_ID = secrets.token_hex(8)

diff --git a/proxy/common/flag.py b/proxy/common/flag.py
@@ -352,7 +352,7 @@ def get_default_plugins(
             args: argparse.Namespace,
     ) -> List[str]:
         """Prepare list of plugins to load based upon
-        --enable-*, --disable-* and --basic-auth flags.
+        --enable-* and --disable-* flags.
         """
         default_plugins: List[str] = []
         if hasattr(args, 'enable_dashboard') and args.enable_dashboard:

diff --git a/proxy/core/base/tcp_tunnel.py b/proxy/core/base/tcp_tunnel.py
@@ -38,7 +38,10 @@ class BaseTcpTunnelHandler(BaseTcpServerHandler):
 
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
-        self.request = HttpParser(httpParserTypes.REQUEST_PARSER)
+        self.request = HttpParser(
+            httpParserTypes.REQUEST_PARSER,
+            enable_proxy_protocol=self.flags.enable_proxy_protocol,
+        )
         self.upstream: Optional[TcpServerConnection] = None
 
     @abstractmethod

diff --git a/proxy/dashboard/dashboard.py b/proxy/dashboard/dashboard.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 """
     proxy.py
     ~~~~~~~~

diff --git a/proxy/http/handler.py b/proxy/http/handler.py
@@ -75,8 +75,10 @@ def __init__(self, *args: Any, **kwargs: Any):
         super().__init__(*args, **kwargs)
         self.start_time: float = time.time()
         self.last_activity: float = self.start_time
-        self.request: HttpParser = HttpParser(httpParserTypes.REQUEST_PARSER)
-        self.response: HttpParser = HttpParser(httpParserTypes.RESPONSE_PARSER)
+        self.request: HttpParser = HttpParser(
+            httpParserTypes.REQUEST_PARSER,
+            enable_proxy_protocol=self.flags.enable_proxy_protocol,
+        )
         self.selector: Optional[selectors.DefaultSelector] = None
         if not is_threadless(self.flags.threadless, self.flags.threaded):
             self.selector = selectors.DefaultSelector()

diff --git a/proxy/http/parser/__init__.py b/proxy/http/parser/__init__.py
@@ -8,11 +8,13 @@
     :copyright: (c) 2013-present by Abhinav Singh and contributors.
     :license: BSD, see LICENSE for more details.
 """
-from .parser import HttpParser, httpParserTypes, httpParserStates
+from .parser import HttpParser
 from .chunk import ChunkParser, chunkParserStates
 from .codes import httpStatusCodes
-from .url import Url
 from .methods import httpMethods
+from .types import httpParserStates, httpParserTypes
+from .url import Url
+from .protocol import ProxyProtocol, PROXY_PROTOCOL_V2_SIGNATURE
 
 __all__ = [
     'HttpParser',
@@ -21,6 +23,8 @@
     'ChunkParser',
     'chunkParserStates',
     'httpStatusCodes',
-    'Url',
     'httpMethods',
+    'Url',
+    'ProxyProtocol',
+    'PROXY_PROTOCOL_V2_SIGNATURE',
 ]
diff --git a/proxy/http/parser/parser.py b/proxy/http/parser/parser.py
@@ -8,35 +8,29 @@
     :copyright: (c) 2013-present by Abhinav Singh and contributors.
     :license: BSD, see LICENSE for more details.
 """
-from typing import TypeVar, NamedTuple, Optional, Dict, Type, Tuple, List
+from typing import TypeVar, Optional, Dict, Type, Tuple, List
 
-from ...common.constants import DEFAULT_DISABLE_HEADERS, COLON, HTTP_1_0, SLASH, CRLF
-from ...common.constants import WHITESPACE, HTTP_1_1, DEFAULT_HTTP_PORT
+from ...common.constants import DEFAULT_DISABLE_HEADERS, COLON, DEFAULT_ENABLE_PROXY_PROTOCOL
+from ...common.constants import HTTP_1_1, HTTP_1_0, SLASH, CRLF
+from ...common.constants import WHITESPACE, DEFAULT_HTTP_PORT
 from ...common.utils import build_http_request, build_http_response, find_http_line, text_
+from ...common.flag import flags
 
 from .url import Url
 from .methods import httpMethods
+from .protocol import ProxyProtocol
 from .chunk import ChunkParser, chunkParserStates
+from .types import httpParserTypes, httpParserStates
 
-HttpParserStates = NamedTuple(
-    'HttpParserStates', [
-        ('INITIALIZED', int),
-        ('LINE_RCVD', int),
-        ('RCVING_HEADERS', int),
-        ('HEADERS_COMPLETE', int),
-        ('RCVING_BODY', int),
-        ('COMPLETE', int),
-    ],
-)
-httpParserStates = HttpParserStates(1, 2, 3, 4, 5, 6)
 
-HttpParserTypes = NamedTuple(
-    'HttpParserTypes', [
-        ('REQUEST_PARSER', int),
-        ('RESPONSE_PARSER', int),
-    ],
+flags.add_argument(
+    '--enable-proxy-protocol',
+    action='store_true',
+    default=DEFAULT_ENABLE_PROXY_PROTOCOL,
+    help='Default: ' + str(DEFAULT_ENABLE_PROXY_PROTOCOL) + '.  ' +
+    'If used, will enable proxy protocol.  ' +
+    'Only version 1 is currently supported.',
 )
-httpParserTypes = HttpParserTypes(1, 2)
 
 
 T = TypeVar('T', bound='HttpParser')
@@ -55,9 +49,16 @@ class HttpParser:
     update parser to work accordingly.
     """
 
-    def __init__(self, parser_type: int) -> None:
-        self.type: int = parser_type
+    def __init__(
+            self, parser_type: int,
+            enable_proxy_protocol: int = DEFAULT_ENABLE_PROXY_PROTOCOL,
+    ) -> None:
         self.state: int = httpParserStates.INITIALIZED
+        self.type: int = parser_type
+        self.protocol: Optional[ProxyProtocol] = None
+        if enable_proxy_protocol:
+            assert self.type == httpParserTypes.REQUEST_PARSER
+            self.protocol = ProxyProtocol()
         self.host: Optional[bytes] = None
         self.port: Optional[int] = None
         self.path: Optional[bytes] = None
@@ -80,8 +81,15 @@ def __init__(self, parser_type: int) -> None:
         self._url: Optional[Url] = None
 
     @classmethod
-    def request(cls: Type[T], raw: bytes) -> T:
-        parser = cls(httpParserTypes.REQUEST_PARSER)
+    def request(
+            cls: Type[T],
+            raw: bytes,
+            enable_proxy_protocol: int = DEFAULT_ENABLE_PROXY_PROTOCOL,
+    ) -> T:
+        parser = cls(
+            httpParserTypes.REQUEST_PARSER,
+            enable_proxy_protocol=enable_proxy_protocol,
+        )
         parser.parse(raw)
         return parser
 
@@ -165,8 +173,7 @@ def body_expected(self) -> bool:
     def parse(self, raw: bytes) -> None:
         """Parses Http request out of raw bytes.
 
-        Check for `HttpParser.state` after `parse` has successfully returned.
-        """
+        Check for `HttpParser.state` after `parse` has successfully returned."""
         self.total_size += len(raw)
         raw = self.buffer + raw
         self.buffer, more = b'', len(raw) > 0
@@ -267,7 +274,8 @@ def _process_line_and_headers(self, raw: bytes) -> Tuple[bool, bytes]:
 
         if self.state == httpParserStates.INITIALIZED:
             self._process_line(line)
-            self.state = httpParserStates.LINE_RCVD
+            if self.state == httpParserStates.INITIALIZED:
+                return len(raw) > 0, raw
         elif self.state in (httpParserStates.LINE_RCVD, httpParserStates.RCVING_HEADERS):
             if self.state == httpParserStates.LINE_RCVD:
                 # LINE_RCVD state is equivalent to RCVING_HEADERS
@@ -291,15 +299,23 @@ def _process_line_and_headers(self, raw: bytes) -> Tuple[bool, bytes]:
         return len(raw) > 0, raw
 
     def _process_line(self, raw: bytes) -> None:
-        line = raw.split(WHITESPACE)
         if self.type == httpParserTypes.REQUEST_PARSER:
-            self.method = line[0].upper()
-            self.set_url(line[1])
-            self.version = line[2]
+            if self.protocol is not None and self.protocol.version is None:
+                # We expect to receive entire proxy protocol v1 line
+                # in one network read and don't expect partial packets
+                self.protocol.parse(raw)
+            else:
+                line = raw.split(WHITESPACE)
+                self.method = line[0].upper()
+                self.set_url(line[1])
+                self.version = line[2]
+                self.state = httpParserStates.LINE_RCVD
         else:
+            line = raw.split(WHITESPACE)
             self.version = line[0]
             self.code = line[1]
             self.reason = WHITESPACE.join(line[2:])
+            self.state = httpParserStates.LINE_RCVD
 
     def _process_header(self, raw: bytes) -> None:
         parts = raw.split(COLON)