diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000000..67f3f9b8cb --- /dev/null +++ b/.coveragerc @@ -0,0 +1,25 @@ +[paths] +source = + proxy + */proxy + .tox/*/lib/python*/site-packages/proxy + +[report] +skip_covered = true +show_missing = true +exclude_lines = + \#\s*pragma: no cover + ^\s*raise AssertionError\b + ^\s*raise NotImplementedError\b + ^\s*return NotImplemented\b + ^\s*raise$ + ^if __name__ == ['"]__main__['"]:$ + +[run] +branch = true +cover_pylib = false +parallel = true +relative_files = true +source = + proxy + tests diff --git a/.darglint b/.darglint new file mode 100644 index 0000000000..6e89243a69 --- /dev/null +++ b/.darglint @@ -0,0 +1,3 @@ +[darglint] +docstring_style=sphinx +strictness=long diff --git a/.deepsource.toml b/.deepsource.toml new file mode 100644 index 0000000000..2fd3572f97 --- /dev/null +++ b/.deepsource.toml @@ -0,0 +1,29 @@ +version = 1 + +test_patterns = [ + "tests/**", + "**/test_*.py" +] + +exclude_patterns = [ + "helper/**", + "menubar/**" +] + +[[analyzers]] +name = "python" +enabled = true + + [analyzers.meta] + runtime_version = "3.x.x" + +[[analyzers]] +name = "javascript" +enabled = true + + [analyzers.meta] + environment = [ + "nodejs", + "browser" + ] + dialect = "typescript" diff --git a/.dockerignore b/.dockerignore index 32e6c2a0cd..3bd432ac85 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,11 +1,5 @@ # Ignore everything ** -# Except proxy -!proxy -!requirements.txt -!setup.py +!dist/*.whl !README.md - -# Ignore __pycache__ directory -proxy/__pycache__ diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000000..1bffdb4268 --- /dev/null +++ b/.flake8 @@ -0,0 +1,248 @@ +[flake8] + +# Don't even try to analyze these: +extend-exclude = + # No need to traverse egg info dir + *.egg-info, + # GitHub configs + .github, + # Cache files of MyPy + .mypy_cache, + # Cache files of pytest + .pytest_cache, + # Temp dir of pytest-testmon + .tmontmp, + # Occasional virtualenv dir + .venv, + # VS Code + .vscode, + # Temporary build dir + build, + # This contains sdists and wheels of proxy.py that we don't want to check + dist, + # Occasional virtualenv dir + env, + # Metadata of `pip wheel` cmd is autogenerated + pip-wheel-metadata, + +# IMPORTANT: avoid using ignore option, always use extend-ignore instead +# Completely and unconditionally ignore the following errors: +extend-ignore = + I # flake8-isort is drunk + we have isort integrated into pre-commit + B009 # FIXME: `getattr()` called with a constant arg + C812 # FIXME: missing trailing comma + C819 # FIXME: inline trailing comma + D101 + D102 + D103 + D105 + D106 + D107 + D203 + D205 + D208 + D209 + D212 + D213 + D300 + D400 + D401 + D402 + D403 + D404 + D405 + D407 + D412 + D413 + D415 + DAR101 # FIXME: undocumented docstring param + DAR201 # FIXME: no "returns" in docstring + DAR301 # FIXME: no "yields" in docstring + DAR401 # FIXME: no "raises" in docstring + E800 # FIXME: commented out code + N400 # FIXME: escaping with backslash at EOL + N801 # FIXME: class name should use ProudCamelCase + N802 # FIXME: func name should be lowercase + N816 # FIXME: mixed case var name + P101 # FIXME: format string with unindexed params + PT009 # FIXME: pytest encourages use of `assert` + PT018 # FIXME: use multiple `assert`s instead of one complex + Q000 # FIXME: inconsistent double quotes use when single ones are needed + Q001 # FIXME: use double quotes in multiline strings + Q002 # FIXME: use double quote docstrings + Q003 # FIXME: avoid escaping in-string quotes + RST201 # FIXME: missing trailing blank line in docstring + RST203 # FIXME: no trailing blank line in docstring + RST301 # FIXME: unexpected indent in docstring + S101 # FIXME: assertions are thrown away in optimized mode, needs audit + S104 # FIXME: bind-all interface listen + S105 # FIXME: hardcoded password? + S303 # FIXME: insecure hash func + S311 # FIXME: `random` needs auditing + S404 # FIXME: `subprocess` use needs auditing + S603 # FIXME: audit untrusted `subprocess.Popen` input + S607 # FIXME: running subprocess with a non-absolute executable path + WPS100 # FIXME: unhelpful module name + WPS102 # FIXME: incorrect module name pattern + WPS110 # FIXME: unhelpful var name + WPS111 # FIXME: too short var name + WPS114 # FIXME: underscored numbers in var name + WPS115 # FIXME: uppercase class attr + WPS118 # FIXME: long func name + WPS120 # FIXME: regular name w/ trailing underscore + WPS121 # FIXME: unused var used 0_O + WPS122 # FIXME: unused var definition 0_O + WPS201 # FIXME: too many imports + WPS202 # FIXME: too many mod members + WPS203 # FIXME: too many mod imported names + WPS204 # FIXME: too much copy-paste + WPS210 # FIXME: too many local vars + WPS211 # FIXME: too many "__init__()" args + WPS212 # FIXME: too many "return"s + WPS213 # FIXME: too many expressions + WPS214 # FIXME: too many methods + WPS216 # FIXME: too many decorators + WPS219 # FIXME: deep object access is unreadable + WPS220 # FIXME: deep code nesting + WPS221 # FIXME: too big inline complexity / tested instructions + WPS222 # FIXME: too much logic in condition + WPS223 # FIXME: the code is too branchy + WPS225 # FIXME: too many "except"s + WPS226 # FIXME: magic string constant used too much, put it in a var + WPS229 # FIXME: try/except should wrap exactly one instruction + WPS230 # FIXME: too many public instance attrs + WPS231 # FIXME: insane complexity/code nesting in a function + WPS232 # FIXME: module is too complex + WPS234 # FIXME: annotation is too complex + WPS235 # FIXME: too many imported names from a single module + WPS237 # FIXME: too complex f-string + WPS300 # local folder imports are needed + WPS301 # FIXME: dotted import + WPS305 # this project is Python 3 only and so f-strings are allowed + WPS306 # this project is Python 3 so it doesn't need an explicit class base + WPS313 # FIXME: parens after keyword + WPS317 # enforces weird indents + WPS318 # enforces weird indents + WPS319 # FIXME: asymmetric closing bracket + WPS320 # FIXME: multiline func type annotation + WPS322 # FIXME: inline multiline str + WPS323 # false-positive: %-formatting in logging + WPS324 # FIXME: inconsistent "return" in func + WPS326 # doesn't allow implicit string concat + WPS328 # FIXME: useless `while` node + WPS336 # FIXME: explicit string concat + WPS337 # FIXME: multiline conditions + WPS338 # FIXME: unordered class methods + WPS339 # FIXME: meaningless leading zeros in number + WPS349 # FIXME: redundant slice + WPS360 # FIXME: unnecessary r-string + WPS361 # FIXME: inconsistent comprehension structure + WPS403 # FIXME: `# noqa` overuse + WPS407 # FIXME: mutable mod const + WPS408 # FIXME: duplicate logical condition + WPS410 # allow `__all__` + WPS412 # FIXME: logic in `__init__` + WPS414 # FIXME: consusing unpacking target + WPS420 # FIXME: pointless keyword like `pass` + WPS421 # FIXME: call to `print()` + WPS425 # FIXME: bool non-keyword arg + WPS427 # FIXME: unreachable code + WPS428 # FIXME: pointless statement + WPS430 # FIXME: nested func + WPS431 # FIXME: nested class + WPS432 # FIXME: magic number w/o assigned context/name + WPS433 # FIXME: nested import + WPS437 # FIXME: protected attr access + WPS440 # FIXME: block vars overlap + WPS441 # FIXME: control var use after block + WPS442 # FIXME: outer scope var shadowing + WPS453 # FIXME: executable file w/o shebang + WPS454 # FIXME: don't raise a broad exception, use a specific one + WPS457 # FIXME: infinite `while` + WPS458 # FIXME: import collision + WPS460 # FIXME: single element unpacking + WPS464 # FIXME: empty comment + WPS501 # FIXME: "finally" in "try" w/o "except" + WPS504 # FIXME: invert a negated condition + WPS505 # FIXME: nested "try" in "try" + WPS507 # FIXME: useless `len()` + WPS508 # FIXME: misused `not` in if-clause + WPS509 # FIXME: incorrect ternary nesting + WPS510 # FIXME: if-clause with `in` operator w/ wrong set semantics + WPS513 # FIXME: implicit `elif` + WPS515 # FIXME: implicit `open()` w/o a CM + WPS518 # FIXME: implicit `enumerate()` pattern + WPS519 # FIXME: implicit `sum()` pattern + WPS528 # FIXME: implicit `dict.items()` pattern + WPS529 # FIXME: implicit `dict.get()` pattern + WPS531 # FIXME: simplifiable returning `if` in func + WPS602 # FIXME: `@staticmethod` is usually a code smell, use module funcs + WPS604 # FIXME: incorrect class body node + WPS605 # FIXME: method w/o args + WPS609 # FIXME: direct call to magic method + WPS612 # FIXME: useless `__init__()` override + WPS613 # FIXME: unmatching super method access + WPS615 # FIXME: unpythonic setter/getter + +# https://wemake-python-stylegui.de/en/latest/pages/usage/formatter.html +format = wemake + +# Let's not overcomplicate the code: +#max-complexity = 10 +# FIXME: this is a lot! +max-complexity = 19 + +# Accessibility/large fonts and PEP8 friendly: +#max-line-length = 79 +# Accessibility/large fonts and PEP8 unfriendly: +#max-line-length = 100 +# Even more Accessibility/large fonts and PEP8 unfriendlier: +max-line-length = 127 + +# Allow certain violations in certain files: +per-file-ignores = + + # E800 reports a lot of false-positives for legit + # tool-related comments; + # WPS412 logic of an extension is in __init__.py file; + # FIXME: WPS201 too many imports + # FIXME: WPS402 too many `noqa`s + #proxy/__init__.py: E800, WPS201, WPS402, WPS412 + + # There are multiple `assert`s (S101) + # and subprocesses (import โ€“ S404; call โ€“ S603) in tests; + # also, using fixtures looks like shadowing the outer scope (WPS442); + # and finally it's impossible to have <= members in tests (WPS202): + tests/**.py: S101, S404, S603, WPS202, WPS442 + +# wemake-python-styleguide +show-source = true + +# flake8-pytest-style +# PT001: +pytest-fixture-no-parentheses = true +# PT006: +pytest-parametrize-names-type = tuple +# PT007: +pytest-parametrize-values-type = tuple +pytest-parametrize-values-row-type = tuple +# PT023: +pytest-mark-no-parentheses = true + +# flake8-rst-docstrings +rst-directives = + spelling +rst-roles = + # Built-in Sphinx roles: + class, + data, + exc, + meth, + term, + py:class, + py:data, + py:exc, + py:meth, + py:term, + # Sphinx's internal role: + event, diff --git a/.git_archival.txt b/.git_archival.txt new file mode 100644 index 0000000000..95cb3eea4e --- /dev/null +++ b/.git_archival.txt @@ -0,0 +1 @@ +ref-names: $Format:%D$ diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..ec8c33334f --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +# Force LF line endings for text files +* text=auto eol=lf + +# Needed for setuptools-scm-git-archive +.git_archival.txt export-subst diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index 40025c3ce4..b814208f01 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1 +1,3 @@ +--- open_collective: proxypy +... diff --git a/.github/buildkitd.toml b/.github/buildkitd.toml new file mode 100644 index 0000000000..59b12d17bd --- /dev/null +++ b/.github/buildkitd.toml @@ -0,0 +1,4 @@ +[worker.oci] + max-parallelism = 4 +[registry."docker.io"] + mirrors = ["mirror.gcr.io"] diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..ba086b02a6 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,48 @@ +--- +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: +# Maintain dependencies for GitHub Actions +- package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + # Allow up to 1 open pull requests + open-pull-requests-limit: 1 + # Add reviewers + reviewers: + - abhinavsingh + +# Maintain dependencies for npm +- package-ecosystem: "npm" + directory: "/dashboard" + schedule: + interval: "daily" + # Allow up to 1 open pull requests + open-pull-requests-limit: 1 + reviewers: + - abhinavsingh + # Prefix all commit messages with "npm" + commit-message: + prefix: "npm" + +# Maintain dependencies for pip +- package-ecosystem: "pip" + directory: "/" + schedule: + interval: "daily" + # Allow up to 1 open pull requests + open-pull-requests-limit: 1 + reviewers: + - abhinavsingh + # Include a list of updated dependencies + # with a prefix determined by the dependency group + commit-message: + prefix: "pip prod" + prefix-development: "pip dev" + include: "scope" +... diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index 2dd16165d2..0000000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,71 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -name: "CodeQL" - -on: - push: - branches: [develop, master] - pull_request: - # The branches below must be a subset of the branches above - branches: [develop] - schedule: - - cron: '0 14 * * 1' - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - # Override automatic language detection by changing the below list - # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] - language: ['python', 'javascript'] - # Learn more... - # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - with: - # We must fetch at least the immediate parents so that if this is - # a pull request then we can checkout the head. - fetch-depth: 2 - - # If this run was triggered by a pull request event, then checkout - # the head of the pull request instead of the merge commit. - - run: git checkout HEAD^2 - if: ${{ github.event_name == 'pull_request' }} - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - # - name: Autobuild - # uses: github/codeql-action/autobuild@v1 - - # โ„น๏ธ Command-line programs to run using the OS shell. - # ๐Ÿ“š https://git.io/JvXDl - - # โœ๏ธ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 diff --git a/.github/workflows/test-brew.yml b/.github/workflows/test-brew.yml deleted file mode 100644 index e6afe49d2b..0000000000 --- a/.github/workflows/test-brew.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Proxy.py Brew - -on: [push, pull_request] - -jobs: - build: - runs-on: ${{ matrix.os }}-latest - name: Brew - Python ${{ matrix.python }} on ${{ matrix.os }} - strategy: - matrix: - os: [macOS] - python: [3.8] - max-parallel: 1 - fail-fast: false - steps: - - uses: actions/checkout@v1 - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python }} - - name: Brew - run: | - brew install ./helper/homebrew/develop/proxy.rb - - name: Verify - run: | - proxy -h diff --git a/.github/workflows/test-dashboard.yml b/.github/workflows/test-dashboard.yml deleted file mode 100644 index ae1bb107e1..0000000000 --- a/.github/workflows/test-dashboard.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Proxy.py Dashboard - -on: [push, pull_request] - -jobs: - build: - runs-on: ${{ matrix.os }}-latest - name: Dashboard - Node ${{ matrix.node }} on ${{ matrix.os }} - strategy: - matrix: - os: [macOS, ubuntu, windows] - node: [10.x, 11.x, 12.x] - max-parallel: 4 - fail-fast: false - steps: - - uses: actions/checkout@v1 - - name: Setup Node - uses: actions/setup-node@v1 - with: - node-version: ${{ matrix.node }} - - name: Install Dependencies - run: | - cd dashboard - npm install - cd .. - - name: Build Dashboard - run: | - cd dashboard - npm run build - cd .. diff --git a/.github/workflows/test-docker.yml b/.github/workflows/test-docker.yml deleted file mode 100644 index e623186544..0000000000 --- a/.github/workflows/test-docker.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Proxy.py Docker - -on: [push, pull_request] - -jobs: - build: - runs-on: ${{ matrix.os }}-latest - name: Docker - Python ${{ matrix.python }} on ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu] - python: [3.8] - max-parallel: 1 - fail-fast: false - steps: - - uses: actions/checkout@v1 - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install -r requirements-testing.txt - - name: Build - run: | - make container diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index 9068485381..5d416fd3be 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -1,41 +1,846 @@ -name: Proxy.py Library +--- +# yamllint disable rule:line-length +name: lib -on: [push, pull_request] +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - develop + pull_request: + workflow_dispatch: + inputs: + release-version: + # github.event_name == 'workflow_dispatch' + # && github.event.inputs.release-version + description: >- + Target PEP440-compliant version to release. + Please, don't prepend `v`. + required: true + release-commitish: + # github.event_name == 'workflow_dispatch' + # && github.event.inputs.release-commitish + default: '' + description: >- + The commit to be released to PyPI and tagged + in Git as `release-version`. Normally, you + should keep this empty. + YOLO: + default: false + description: >- + Flag whether test results should block the + release (true/false). Only use this under + extraordinary circumstances to ignore the + test failures and cut the release regardless. + +concurrency: + group: >- + ${{ + github.workflow + }}-${{ + github.event.pull_request.number || github.sha + }} + cancel-in-progress: true jobs: + pre-setup: + name: โš™๏ธ Pre-set global build settings + runs-on: ubuntu-latest + defaults: + run: + shell: python + outputs: + dist-version: >- + ${{ + steps.request-check.outputs.release-requested == 'true' + && github.event.inputs.release-version + || steps.scm-version.outputs.dist-version + }} + is-untagged-devel: >- + ${{ steps.untagged-check.outputs.is-untagged-devel || false }} + release-requested: >- + ${{ + steps.request-check.outputs.release-requested || false + }} + cache-key-files: >- + ${{ steps.calc-cache-key-files.outputs.files-hash-key }} + git-tag: ${{ steps.git-tag.outputs.tag }} + sdist-artifact-name: ${{ steps.artifact-name.outputs.sdist }} + wheel-artifact-name: ${{ steps.artifact-name.outputs.wheel }} + steps: + - name: Switch to using Python 3.9 by default + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: >- + Mark the build as untagged '${{ + github.event.repository.default_branch + }}' branch build + id: untagged-check + if: >- + github.event_name == 'push' && + github.ref == format( + 'refs/heads/{0}', github.event.repository.default_branch + ) + run: >- + print('::set-output name=is-untagged-devel::true') + - name: Mark the build as "release request" + id: request-check + if: github.event_name == 'workflow_dispatch' + run: >- + print('::set-output name=release-requested::true') + - name: Check out src from Git + if: >- + steps.request-check.outputs.release-requested != 'true' + uses: actions/checkout@v2 + with: + fetch-depth: 0 + ref: ${{ github.event.inputs.release-commitish }} + - name: >- + Calculate Python interpreter version hash value + for use in the cache key + if: >- + steps.request-check.outputs.release-requested != 'true' + id: calc-cache-key-py + run: | + from hashlib import sha512 + from sys import version + hash = sha512(version.encode()).hexdigest() + print(f'::set-output name=py-hash-key::{hash}') + - name: >- + Calculate dependency files' combined hash value + for use in the cache key + if: >- + steps.request-check.outputs.release-requested != 'true' + id: calc-cache-key-files + run: | + print( + "::set-output name=files-hash-key::${{ + hashFiles( + 'setup.cfg', 'tox.ini', 'pyproject.toml', + '.pre-commit-config.yaml', 'pytest.ini' + ) + }}", + ) + - name: Get pip cache dir + id: pip-cache-dir + if: >- + steps.request-check.outputs.release-requested != 'true' + run: >- + echo "::set-output name=dir::$(python -m pip cache dir)" + shell: bash + - name: Set up pip cache + if: >- + steps.request-check.outputs.release-requested != 'true' + uses: actions/cache@v2.1.7 + with: + path: ${{ steps.pip-cache-dir.outputs.dir }} + key: >- + ${{ runner.os }}-pip-${{ + steps.calc-cache-key-py.outputs.py-hash-key }}-${{ + steps.calc-cache-key-files.outputs.files-hash-key }} + restore-keys: | + ${{ runner.os }}-pip-${{ + steps.calc-cache-key-py.outputs.py-hash-key + }}- + ${{ runner.os }}-pip- + ${{ runner.os }}- + - name: Drop Git tags from HEAD for non-release requests + if: >- + steps.request-check.outputs.release-requested != 'true' + run: >- + git tag --points-at HEAD + | + xargs git tag --delete + shell: bash + - name: Set up versioning prerequisites + if: >- + steps.request-check.outputs.release-requested != 'true' + run: >- + python -m + pip install + --user + setuptools-scm + shell: bash + - name: Set the current dist version from Git + if: steps.request-check.outputs.release-requested != 'true' + id: scm-version + run: | + import setuptools_scm + ver = setuptools_scm.get_version( + ${{ + steps.untagged-check.outputs.is-untagged-devel == 'true' + && 'local_scheme="no-local-version"' || '' + }} + ) + print('::set-output name=dist-version::{ver}'.format(ver=ver)) + - name: Set the target Git tag + id: git-tag + run: >- + print('::set-output name=tag::v${{ + steps.request-check.outputs.release-requested == 'true' + && github.event.inputs.release-version + || steps.scm-version.outputs.dist-version + }}') + - name: Set the expected dist artifact names + id: artifact-name + run: | + print('::set-output name=sdist::proxy.py-${{ + steps.request-check.outputs.release-requested == 'true' + && github.event.inputs.release-version + || steps.scm-version.outputs.dist-version + }}.tar.gz') + print('::set-output name=wheel::proxy.py-${{ + steps.request-check.outputs.release-requested == 'true' + && github.event.inputs.release-version + || steps.scm-version.outputs.dist-version + }}-py3-none-any.whl') + build: + name: ๐Ÿ‘ท dists ${{ needs.pre-setup.outputs.git-tag }} + needs: + - pre-setup # transitive, for accessing settings + + runs-on: Ubuntu-latest + + env: + PY_COLORS: 1 + TOX_PARALLEL_NO_SPINNER: 1 + TOXENV: cleanup-dists,build-dists + + steps: + - name: Switch to using Python v3.10 + uses: actions/setup-python@v2 + with: + python-version: '3.10' + - name: >- + Calculate Python interpreter version hash value + for use in the cache key + id: calc-cache-key-py + run: | + from hashlib import sha512 + from sys import version + + hash = sha512(version.encode()).hexdigest() + print(f'::set-output name=py-hash-key::{hash}') + shell: python + - name: Get pip cache dir + id: pip-cache + run: >- + echo "::set-output name=dir::$(pip cache dir)" + - name: Set up pip cache + uses: actions/cache@v2.1.7 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: >- + ${{ runner.os }}-pip-${{ + steps.calc-cache-key-py.outputs.py-hash-key }}-${{ + hashFiles('tox.ini') }} + restore-keys: | + ${{ runner.os }}-pip-${{ + steps.calc-cache-key-py.outputs.py-hash-key + }}- + ${{ runner.os }}-pip- + - name: Install tox + run: >- + python -m + pip install + --user + tox + + - name: Grab the source from Git + uses: actions/checkout@v2 + with: + fetch-depth: 0 + ref: ${{ github.event.inputs.release-commitish }} + + - name: Pre-populate the tox env + run: >- + python -m + tox + --parallel auto + --parallel-live + --skip-missing-interpreters false + --notest + + - name: Setup git user as [bot] + if: >- + fromJSON(needs.pre-setup.outputs.is-untagged-devel) + || fromJSON(needs.pre-setup.outputs.release-requested) + uses: fregante/setup-git-user@v1.0.1 + - name: >- + Tag the release in the local Git repo + as ${{ needs.pre-setup.outputs.git-tag }} + for setuptools-scm to set the desired version + if: >- + fromJSON(needs.pre-setup.outputs.is-untagged-devel) + || fromJSON(needs.pre-setup.outputs.release-requested) + run: >- + git tag + -m '${{ needs.pre-setup.outputs.git-tag }}' + '${{ needs.pre-setup.outputs.git-tag }}' + -- + ${{ github.event.inputs.release-commitish }} + + - name: Build dists + run: >- + python -m + tox + --parallel auto + --parallel-live + --skip-missing-interpreters false + --skip-pkg-install + - name: Verify that the artifacts with expected names got created + run: >- + ls -1 + 'dist/${{ needs.pre-setup.outputs.sdist-artifact-name }}' + 'dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}' + - name: Store the distribution packages + uses: actions/upload-artifact@v2 + with: + name: python-package-distributions + # NOTE: Exact expected file names are specified here + # NOTE: as a safety measure โ€” if anything weird ends + # NOTE: up being in this dir or not all dists will be + # NOTE: produced, this will fail the workflow. + path: | + dist/${{ needs.pre-setup.outputs.sdist-artifact-name }} + dist/${{ needs.pre-setup.outputs.wheel-artifact-name }} + retention-days: 30 # Defaults to 90 + + lint: + name: ๐Ÿงน ${{ matrix.toxenv }} + needs: + - build + - pre-setup # transitive, for accessing settings + + runs-on: Ubuntu-latest + strategy: + matrix: + toxenv: + - lint + - metadata-validation + - build-docs + - doctest-docs + - linkcheck-docs + - spellcheck-docs + fail-fast: false + + env: + PY_COLORS: 1 + TOX_PARALLEL_NO_SPINNER: 1 + TOXENV: ${{ matrix.toxenv }} + + steps: + - name: Switch to using Python v3.10 + uses: actions/setup-python@v2 + with: + python-version: '3.10' + - name: >- + Calculate Python interpreter version hash value + for use in the cache key + id: calc-cache-key-py + run: | + from hashlib import sha512 + from sys import version + + hash = sha512(version.encode()).hexdigest() + print(f'::set-output name=py-hash-key::{hash}') + shell: python + - name: Get pip cache dir + id: pip-cache + run: >- + echo "::set-output name=dir::$(pip cache dir)" + - name: Set up pip cache + uses: actions/cache@v2.1.7 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: >- + ${{ runner.os }}-pip-${{ + steps.calc-cache-key-py.outputs.py-hash-key }}-${{ + hashFiles('tox.ini') }} + restore-keys: | + ${{ runner.os }}-pip-${{ + steps.calc-cache-key-py.outputs.py-hash-key + }}- + ${{ runner.os }}-pip- + - name: Install tox + run: >- + python -m + pip install + --user + tox + + - name: Grab the source from Git + uses: actions/checkout@v2 + with: + ref: ${{ github.event.inputs.release-commitish }} + + - name: Make the env clean of non-test files + if: matrix.toxenv == 'metadata-validation' + run: | + shopt -s extglob + rm -rf !tox.ini + shell: bash + - name: Download all the dists + if: matrix.toxenv == 'metadata-validation' + uses: actions/download-artifact@v2 + with: + name: python-package-distributions + path: dist/ + + - name: >- + Pre-populate tox envs: `${{ env.TOXENV }}` + run: >- + python -m + tox + --parallel auto + --parallel-live + --skip-missing-interpreters false + --notest + - name: >- + Run tox envs: `${{ env.TOXENV }}` + run: >- + python -m + tox + --parallel auto + --parallel-live + --skip-missing-interpreters false + --skip-pkg-install + + test: + name: ๐Ÿงช ๐Ÿ${{ matrix.python }} @ ${{ matrix.os }} + needs: + - build + - pre-setup # transitive, for accessing settings + runs-on: ${{ matrix.os }}-latest - name: Library - Python ${{ matrix.python }} on ${{ matrix.os }} strategy: + fail-fast: false + # max-parallel: 4 matrix: - os: [macOS, ubuntu, windows] - python: [3.6, 3.7, 3.8] - max-parallel: 4 + os: + - macOS + - Ubuntu + - Windows + python: + # NOTE: The latest and the lowest supported Pythons are prioritized + # NOTE: to improve the responsiveness. It's nice to see the most + # NOTE: important results first. + - '3.10' + - 3.6 + - 3.9 + - 3.8 + - 3.7 + + continue-on-error: >- + ${{ + ( + needs.pre-setup.outputs.release-requested == 'true' && + !toJSON(github.event.inputs.YOLO) + ) && true || false + }} + + env: + PY_COLORS: 1 + TOX_PARALLEL_NO_SPINNER: 1 + TOXENV: python + + steps: + - name: Switch to using Python v${{ matrix.python }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + - name: >- + Calculate Python interpreter version hash value + for use in the cache key + id: calc-cache-key-py + run: | + from hashlib import sha512 + from sys import version + + hash = sha512(version.encode()).hexdigest() + print(f'::set-output name=py-hash-key::{hash}') + shell: python + - name: Get pip cache dir + id: pip-cache + run: >- + echo "::set-output name=dir::$(pip cache dir)" + - name: Set up pip cache + uses: actions/cache@v2.1.7 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: >- + ${{ runner.os }}-pip-${{ + steps.calc-cache-key-py.outputs.py-hash-key }}-${{ + hashFiles('tox.ini', 'requirements.txt', 'requirements-testing.txt') + }} + restore-keys: | + ${{ runner.os }}-pip-${{ + steps.calc-cache-key-py.outputs.py-hash-key + }}- + ${{ runner.os }}-pip- + - name: Install tox + run: >- + python -m + pip install + --user + tox + + - name: Grab the source from Git + uses: actions/checkout@v2 + with: + ref: ${{ github.event.inputs.release-commitish }} + + - name: Download all the dists + uses: actions/download-artifact@v2 + with: + name: python-package-distributions + path: dist/ + + - name: Pre-populate the testing env + run: >- + python -m + tox + --parallel auto + --parallel-live + --skip-missing-interpreters false + --installpkg 'dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}' + --notest + shell: bash + - name: Run the testing + run: >- + python -m + tox + --parallel auto + --parallel-live + --skip-missing-interpreters false + --skip-pkg-install + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v2 + with: + flags: pytest, GHA, Python ${{ matrix.python }}, ${{ runner.os }} + verbose: true + + analyze: + runs-on: ubuntu-latest + name: ๐Ÿ›ก๏ธ Analyze + # schedule: + # - cron: '0 14 * * 1' + strategy: fail-fast: false + matrix: + # Override automatic language detection by changing the below list + # Supported options are + # ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] + language: ['python', 'javascript'] + # Learn more... + # https://docs.github.com/en/github + # /finding-security-vulnerabilities-and-errors-in-your-code + # /configuring-code-scanning#overriding-automatic-language-detection + steps: - - uses: actions/checkout@v1 - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install -r requirements-testing.txt - - name: Quality Check - run: | - flake8 --ignore=W504 --max-line-length=127 --max-complexity=19 proxy/ tests/ setup.py - mypy --strict --ignore-missing-imports proxy/ tests/ setup.py - - name: Run Tests - run: pytest --cov=proxy tests/ - - name: Upload coverage to Codecov - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - run: codecov - - name: Integration testing - if: matrix.os != 'windows' - run: | - python setup.py install - proxy --hostname 127.0.0.1 --enable-web-server --pid-file proxy.pid --log-file proxy.log & - ./tests/integration/main.sh + - name: Checkout repository + uses: actions/checkout@v2 + with: + # We must fetch at least the immediate parents so that if this is + # a pull request then we can checkout the head. + fetch-depth: 2 + + # If this run was triggered by a pull request event, then checkout + # the head of the pull request instead of the merge commit. + - run: git checkout HEAD^2 + if: ${{ github.event_name == 'pull_request' }} + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a + # config file. By default, queries listed here will override any + # specified in a config file. Prefix the list here with "+" to use + # these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually + # (see below) + # - name: Autobuild + # uses: github/codeql-action/autobuild@v1 + + # โ„น๏ธ Command-line programs to run using the OS shell. + # ๐Ÿ“š https://git.io/JvXDl + + # โœ๏ธ If the Autobuild fails above, remove it and uncomment the following + # three lines and modify them (or add more) to build your code if your + # project uses a compiled language + + # - run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 + + brew: + runs-on: ${{ matrix.os }}-latest + name: ๐Ÿบ ๐Ÿ${{ matrix.python }} @ ${{ matrix.os }} + strategy: + matrix: + os: [macOS] + python: ['3.10'] + # max-parallel: 1 + fail-fast: false + steps: + - uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + - name: Brew + run: | + brew install ./helper/homebrew/develop/proxy.rb + - name: Verify + run: | + proxy -h + + dashboard: + runs-on: ${{ matrix.os }}-latest + name: ๐Ÿ“Š Node ${{ matrix.node }} @ ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu, windows, macOS] + node: ['10.x', '11.x', '12.x'] + # max-parallel: 4 + fail-fast: false + steps: + - uses: actions/checkout@v2 + - name: Setup Node + uses: actions/setup-node@v2 + with: + node-version: ${{ matrix.node }} + - name: Install Dependencies + run: | + cd dashboard + npm install + cd .. + - name: Build Dashboard + run: | + cd dashboard + npm run build + cd .. + + docker: + # TODO: To build our docker container, we must wait for check, + # so that we can use the same distribution available. + runs-on: ${{ matrix.os }}-latest + needs: + - build + - pre-setup # transitive, for accessing settings + name: ๐Ÿณ ๐Ÿ${{ matrix.python }} @ ${{ matrix.targetplatform }} + strategy: + matrix: + os: + - Ubuntu + python: + - '3.10' + targetplatform: + - 'linux/386' + - 'linux/amd64' + - 'linux/arm/v6' + - 'linux/arm/v7' + - 'linux/arm64/v8' + - 'linux/ppc64le' + - 'linux/s390x' + # max-parallel: 1 + fail-fast: false + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + with: + buildkitd-flags: --debug + config: .github/buildkitd.toml + install: true + - name: Download all the dists + uses: actions/download-artifact@v2 + with: + name: python-package-distributions + path: dist/ + - name: Enable Multiarch # This slows down arm build by 4-5x + run: | + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + - name: Create builder + run: | + docker buildx create --name proxypybuilder + docker buildx use proxypybuilder + docker buildx inspect + docker buildx ls + - name: Set PROXYPY_CONTAINER_VERSION + run: | + echo "PROXYPY_CONTAINER_VERSION=$(echo '${{ needs.pre-setup.outputs.dist-version }}' | tr + .)" > $GITHUB_ENV + - name: Build container + run: | + make container-buildx \ + -e PROXYPY_PKG_PATH='dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}' \ + -e BUILDX_TARGET_PLATFORM='${{ matrix.targetplatform }}' \ + -e PROXYPY_CONTAINER_VERSION='${{ env.PROXYPY_CONTAINER_VERSION }}' + + check: # This job does nothing and is only used for the branch protection + needs: + - analyze + - test + - lint + - docker + - dashboard + - brew + + runs-on: ubuntu-latest + + steps: + - name: Report success of the test matrix + run: >- + print("All's good") + shell: python + + publish-pypi: + name: Publish ๐Ÿ๐Ÿ“ฆ ${{ needs.pre-setup.outputs.git-tag }} to PyPI + needs: + - check + - pre-setup # transitive, for accessing settings + if: >- + fromJSON(needs.pre-setup.outputs.release-requested) + runs-on: Ubuntu-latest + + environment: + name: release + url: >- + https://pypi.org/project/proxy.py/${{ + needs.pre-setup.outputs.dist-version + }} + + steps: + - name: Download all the dists + uses: actions/download-artifact@v2 + with: + name: python-package-distributions + path: dist/ + - name: >- + Publish ๐Ÿ๐Ÿ“ฆ v${{ needs.pre-setup.outputs.git-tag }} to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_TOKEN }} + + publish-testpypi: + name: Publish ๐Ÿ๐Ÿ“ฆ to TestPyPI + needs: + - check + - pre-setup # transitive, for accessing settings + if: >- + fromJSON(needs.pre-setup.outputs.is-untagged-devel) + || fromJSON(needs.pre-setup.outputs.release-requested) + runs-on: Ubuntu-latest + + environment: + name: release-testpypi + url: >- + https://test.pypi.org/project/proxy.py/${{ + needs.pre-setup.outputs.dist-version + }} + + steps: + - name: Download all the dists + uses: actions/download-artifact@v2 + with: + name: python-package-distributions + path: dist/ + - name: >- + Publish ๐Ÿ๐Ÿ“ฆ v${{ needs.pre-setup.outputs.git-tag }} to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.TESTPYPI_API_TOKEN }} + repository_url: https://test.pypi.org/legacy/ + + post-release-repo-update: + name: >- + Publish post-release Git tag + for ${{ needs.pre-setup.outputs.git-tag }} + needs: + - publish-pypi + - pre-setup # transitive, for accessing settings + runs-on: Ubuntu-latest + + steps: + - name: Fetch the src snapshot + uses: actions/checkout@v2 + with: + fetch-depth: 1 + ref: ${{ github.event.inputs.release-commitish }} + - name: Setup git user as [bot] + uses: fregante/setup-git-user@v1.0.1 + + - name: >- + Tag the release in the local Git repo + as v${{ needs.pre-setup.outputs.git-tag }} + run: >- + git tag + -m '${{ needs.pre-setup.outputs.git-tag }}' + '${{ needs.pre-setup.outputs.git-tag }}' + -- + ${{ github.event.inputs.release-commitish }} + - name: >- + Push ${{ needs.pre-setup.outputs.git-tag }} tag corresponding + to the just published release back to GitHub + run: >- + git push --atomic origin '${{ needs.pre-setup.outputs.git-tag }}' + + publish-github-release: + name: >- + Publish a GitHub Release for + ${{ needs.pre-setup.outputs.git-tag }} + needs: + - post-release-repo-update + - pre-setup # transitive, for accessing settings + runs-on: Ubuntu-latest + + permissions: + contents: write + discussions: write + + steps: + - name: Download all the dists + uses: actions/download-artifact@v2 + with: + name: python-package-distributions + path: dist/ + + - name: >- + Publish a GitHub Release for + ${{ needs.pre-setup.outputs.git-tag }} + uses: ncipollo/release-action@v1.8.10 + with: + allowUpdates: false + artifactErrorsFailBuild: false + artifacts: | + dist/${{ needs.pre-setup.outputs.sdist-artifact-name }} + dist/${{ needs.pre-setup.outputs.wheel-artifact-name }} + artifactContentType: raw # Because whl and tgz are of different types + # body/bodyFile: # FIXME: Use once Towncrier is integrated. + commit: ${{ github.event.inputs.release-commitish }} + discussionCategory: Announcements + draft: false + name: ${{ needs.pre-setup.outputs.git-tag }} + # omitBody: false + omitBodyDuringUpdate: true + omitName: false + omitNameDuringUpdate: true + omitPrereleaseDuringUpdate: true + prerelease: false + removeArtifacts: false + replacesArtifacts: false + tag: ${{ needs.pre-setup.outputs.git-tag }} + token: ${{ secrets.GITHUB_TOKEN }} +... diff --git a/.gitignore b/.gitignore index 343de585d2..217ce78fd2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,17 +2,20 @@ .coverage .coverage.* .idea -.vscode .project .pydevproject .settings .mypy_cache .hypothesis .tox +.python-version coverage.xml proxy.py.iml +.vscode/* +!.vscode/settings.json + *.pyc *.egg-info *.csr @@ -25,4 +28,6 @@ cover htmlcov dist build + proxy/public +profile.svg diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..db86982576 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,193 @@ +--- +repos: +- repo: https://github.com/asottile/add-trailing-comma.git + rev: v2.0.1 + hooks: + - id: add-trailing-comma + args: + - --py36-plus + +# - repo: https://github.com/timothycrosley/isort.git +# rev: 5.4.2 +# hooks: +# - id: isort +# args: +# - --honor-noqa + +- repo: https://github.com/Lucas-C/pre-commit-hooks.git + rev: v1.1.7 + hooks: + - id: remove-tabs + exclude: | + (?x) + ^ + helper/proxy\.pac| + Makefile| + proxy/common/pki\.py| + README\.md| + .+\.(plist|pbxproj) + $ + +- repo: https://github.com/pre-commit/pre-commit-hooks.git + rev: v4.0.1 + hooks: + # Side-effects: + - id: trailing-whitespace + exclude: | + (?x) + ^ + \.github/workflows/codeql-analysis\.yml| + dashboard/src/core/plugins/inspect_traffic\.json + $ + - id: check-merge-conflict + - id: double-quote-string-fixer + exclude: | + (?x) + ^ + ( + tests/( + http/exceptions/test_http_proxy_auth_failed| + plugin/test_http_proxy_plugins + )| + proxy/( + common/constants| + plugin/(cache/store/disk|filter_by_url_regex|proxy_pool) + ) + )\.py + $ + - id: end-of-file-fixer + exclude: | + (?x) + ^ + dashboard/( + src/core/plugins/inspect_traffic\.json| + static/bootstrap-4\.3\.1\.min\.(cs|j)s + )| + menubar/proxy\.py/( + Assets\.xcassets/( + AppIcon\.appiconset/| + StatusBarButtonImage\.imageset/| + )| + Preview\sContent/Preview\sAssets\.xcassets/ + )Contents\.json| + requirements-release\.txt + $ + - id: requirements-txt-fixer + exclude: >- + ^(docs/requirements|requirements(|-(release|testing|tunnel)))\.txt$ + # Non-modifying checks: + - id: name-tests-test + args: + - --django + exclude: >- + ^tests/plugin/utils\.py$ + files: >- + ^tests/[^_].*\.py$ + - id: check-added-large-files + - id: check-byte-order-marker + - id: check-case-conflict + # disabled due to pre-commit/pre-commit-hooks#159 + # - id: check-docstring-first + - id: check-json + - id: check-symlinks + - id: check-yaml + - id: detect-private-key + + # Heavy checks: + - id: check-ast + - id: debug-statements + +- repo: https://github.com/PyCQA/pydocstyle.git + rev: 6.1.1 + hooks: + - id: pydocstyle + additional_dependencies: + - toml + args: + - |- + --ignore= + D101, + D102, + D103, + D105, + D106, + D107, + D203, + D205, + D208, + D209, + D212, + D213, + D300, + D400, + D401, + D402, + D403, + D404, + D405, + D407, + D412, + D413, + D415, + +- repo: https://github.com/codespell-project/codespell + rev: v2.1.0 + hooks: + - id: codespell + exclude: >- + ^.+\.min\.js$ + +- repo: https://github.com/adrienverge/yamllint.git + rev: v1.26.2 + hooks: + - id: yamllint + args: + - --strict + types: [file, yaml] + +- repo: https://github.com/PyCQA/flake8.git + rev: 3.9.2 + hooks: + - id: flake8 + language_version: python3 + additional_dependencies: + - flake8-2020 >= 1.6.0 + - flake8-docstrings >= 1.5.0 + - flake8-pytest-style >= 1.2.2 + - wemake-python-styleguide ~= 0.15.0 + +- repo: https://github.com/pre-commit/mirrors-mypy.git + rev: v0.910 + hooks: + - id: mypy + additional_dependencies: + - paramiko == 2.8.0 + - types-paramiko == 2.7.3 + - types-setuptools == 57.4.2 + args: + # FIXME: get rid of missing imports ignore + - --ignore-missing-imports + - --install-types + - --namespace-packages + - --non-interactive + - --pretty + - --show-column-numbers + - --show-error-codes + - --show-error-context + - --strict + - --strict-optional + - examples/ + - proxy/ + - tests/ + pass_filenames: false + +- repo: local + hooks: + - id: pylint + language: system + name: PyLint + files: \.py$ + entry: python -m pylint + args: [] + stages: + - manual diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000000..1203c8dec9 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,610 @@ +[MASTER] + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-allow-list= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +extension-pkg-whitelist= + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= + +# Specify a score threshold to be exceeded before program exits with error. +fail-under=10.0 + +# Files or directories to be skipped. They should be base names, not paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the ignore-list. The +# regex matches against paths. +ignore-paths= + +# Files or directories matching the regex patterns are skipped. The regex +# matches against base names, not paths. +ignore-patterns= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use. +#jobs=1 +jobs=0 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Min Python version to use for version dependend checks. Will default to the +# version used to run pylint. +py-version=3.9 + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. +confidence= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + use-symbolic-message-instead, + # FIXME: Everything below has been violated at the time of the + # FIXME: initial addition. All of these rules must be fixed and + # FIXME: removed/re-enabled over time. + arguments-differ, + attribute-defined-outside-init, + broad-except, + consider-using-dict-items, + consider-using-enumerate, + consider-using-f-string, + consider-using-ternary, + consider-using-with, + cyclic-import, + deprecated-method, + duplicate-code, + fixme, + global-variable-not-assigned, + invalid-name, + logging-format-interpolation, + logging-not-lazy, + missing-class-docstring, + missing-function-docstring, + no-else-raise, + no-self-use, + no-value-for-parameter, + protected-access, + raise-missing-from, + raising-format-tuple, + redefined-outer-name, + super-init-not-called, + superfluous-parens, + too-few-public-methods, + too-many-arguments, + too-many-branches, + too-many-instance-attributes, + too-many-locals, + too-many-nested-blocks, + too-many-public-methods, + too-many-return-statements, + too-many-statements, + unnecessary-pass, + unreachable, + unused-argument, + useless-return, + useless-super-delegation, + wrong-import-order, + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[REPORTS] + +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'error', 'warning', 'refactor', and 'convention' +# which contain the number of messages in each category, as well as 'statement' +# which is the total number of statements analyzed. This score is used by the +# global evaluation report (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +#msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +#output-format=text +output-format=colorized + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit,argparse.parse_error + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. +#argument-rgx= + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. +#class-attribute-rgx= + +# Naming style matching correct class constant names. +class-const-naming-style=UPPER_CASE + +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. +#class-const-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. +#class-rgx= + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + _ + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. +#variable-rgx= + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +# Accessibility/large fonts and PEP8 friendly: +#max-line-length = 79 +# Accessibility/large fonts and PEP8 unfriendly: +max-line-length = 127 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[LOGGING] + +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + +# Regular expression of note tags to take in consideration. +#notes-rgx= + + +[SIMILARITIES] + +# Comments are removed from the similarity computation +ignore-comments=yes + +# Docstrings are removed from the similarity computation +ignore-docstrings=yes + +# Imports are removed from the similarity computation +ignore-imports=no + +# Signatures are removed from the similarity computation +ignore-signatures=no + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. Available dictionaries: none. To make it work, +# install the 'python-enchant' package. +spelling-dict= + +# List of comma separated words that should be considered directives if they +# appear and the beginning of a comment and should not be checked. +spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no + + +[STRING] + +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no + +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + +# List of decorators that change the signature of a decorated function. +signature-mutators= + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of names allowed to shadow builtins +allowed-redefined-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io + + +[CLASSES] + +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp, + __post_init__ + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict, + _fields, + _replace, + _source, + _make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=cls + + +[DESIGN] + +# List of qualified class names to ignore when counting class parents (see +# R0901) +ignored-parents= + +# Maximum number of arguments for function / method. +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules= + +# Output a graph (.gv or any supported image format) of external dependencies +# to the given file (report RP0402 must not be disabled). +ext-import-graph= + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be +# disabled). +import-graph= + +# Output a graph (.gv or any supported image format) of internal dependencies +# to the given file (report RP0402 must not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "BaseException, Exception". +overgeneral-exceptions=BaseException, + Exception diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000000..6ac10f57af --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,41 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html +# for details + +--- + +# Required +version: 2 + +# Build documentation in the docs/ directory with Sphinx +sphinx: + builder: dirhtml + configuration: docs/conf.py + fail_on_warning: true + +# Optionally build your docs in additional formats +# such as PDF and ePub +formats: [] + +submodules: + include: all # [] + exclude: [] + recursive: true + +build: + os: ubuntu-20.04 + tools: + python: >- # PyYAML parses it as float `3.1` it it's not an explicit string + 3.10 + +# Optionally set the version of Python and requirements required +# to build docs +python: + install: + - method: pip + path: . + - requirements: requirements-tunnel.txt + - requirements: docs/requirements.txt + system_packages: false + +... diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000000..e956ff2ea9 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,46 @@ +{ + "editor.rulers": [100], + "editor.formatOnSaveMode": "modifications", + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.fixAll": true + }, + "[yaml]": { + "editor.insertSpaces": true, + "editor.tabSize": 2, + "editor.autoIndent": "advanced", + "editor.defaultFormatter": "rvest.vs-code-prettier-eslint" + }, + "[typescript]": { + "editor.formatOnSave": true, + "editor.defaultFormatter": "rvest.vs-code-prettier-eslint" + }, + "typescript.format.semicolons": "remove", + "typescript.preferences.quoteStyle": "single", + "[python]": { + "editor.wordBasedSuggestions": true, + "editor.defaultFormatter": null + }, + "python.testing.unittestEnabled": false, + "python.testing.autoTestDiscoverOnSaveEnabled": true, + "python.testing.pytestEnabled": true, + "python.testing.pytestArgs": ["tests"], + "python.linting.enabled": true, + "python.linting.lintOnSave": true, + "python.linting.ignorePatterns": [ + ".tox/**/*.py", + ".vscode/*.py", + ".venv*/**/*.py", + "venv*/**/*.py", + "docs/**/*.py", + "helper/**/*.py", + "menubar/**/*.py" + ], + "python.linting.pylintEnabled": true, + "python.linting.pylintArgs": ["--generate-members"], + "python.linting.flake8Enabled": true, + "python.linting.flake8Args": ["--config", ".flake8"], + "python.linting.mypyEnabled": true, + "python.formatting.provider": "autopep8", + "autoDocstring.docstringFormat": "sphinx" +} diff --git a/.yamllint b/.yamllint new file mode 100644 index 0000000000..3b3a84c8b2 --- /dev/null +++ b/.yamllint @@ -0,0 +1,8 @@ +--- +extends: default + +rules: + indentation: + level: error + indent-sequences: false +... diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 84e4f570b5..6634e37567 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -55,7 +55,7 @@ further defined and clarified by project maintainers. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the project team at mailsforabhinav@gmail.com. All +reported by contacting the project team at mailsforabhinav+proxy@gmail.com. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000000..e412193b95 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,42 @@ +# Contributing to proxy.py + +This document describes how contributors can participate and iterate quickly while maintaining the `proxy.py` project standards and guidelines. + +## Basic Guidelines + +* Your pull request should NOT introduce any external dependency. +* It is OK to use external dependencies within plugins. + +## Environment Setup + +Contributors must start `proxy.py` from source to verify and develop new features / fixes. See [Run proxy.py from command line using repo source](#from-command-line-using-repo-source) for usage instructions. + +[![WARNING](https://img.shields.io/static/v1?label=MacOS&message=warning&color=red)](https://github.com/abhinavsingh/proxy.py/issues/642#issuecomment-960819271) On `macOS` you must install `Python` using `pyenv`, as `Python` installed via `homebrew` tends to be problematic. See linked thread for more details. + +### Setup Git Hooks + +You SHOULD NOT avoid these steps. Git hooks will help catch test or linting errors locally without pushing to upstream. This will save you a lot of time and allow you to contribute and iterate faster. + +Pre-commit hook ensures tests are passing. + +1. `cd /path/to/proxy.py` +2. `ln -s $(PWD)/git-pre-commit .git/hooks/pre-commit` + +Pre-push hook ensures lint and tests are passing. + +1. `cd /path/to/proxy.py` +2. `ln -s $(PWD)/git-pre-push .git/hooks/pre-push` + +### Sending a Pull Request + +All pull requests are tested using GitHub actions. + +See [GitHub workflow](https://github.com/abhinavsingh/proxy.py/tree/develop/.github/workflows) for list of workflows. + +## Communication + +During the process of PR review, sometimes, you may get asked to update certain project configs. Example, a change in code introduced via your PR will result in a redundant lint guard. So we must make corresponding changes to ensure project health. + +It's highly recommended that you participate in maintaining a high code-quality standard. For any reason, if you are unable to address the requested changes, please communicate the same to the reviewer. + +Thank you!!! diff --git a/Dashboard.png b/Dashboard.png new file mode 100644 index 0000000000..33cb9ed024 Binary files /dev/null and b/Dashboard.png differ diff --git a/Dockerfile b/Dockerfile index d751a12018..0d59cd5bf5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,26 +1,26 @@ -FROM python:3.8-alpine as base -FROM base as builder - -COPY requirements.txt /app/ -COPY setup.py /app/ -COPY README.md /app/ -COPY proxy/ /app/proxy/ -WORKDIR /app -RUN pip install --upgrade pip && \ - pip install --prefix=/deps . - -FROM base - +FROM python:3.10-alpine as base LABEL com.abhinavsingh.name="abhinavsingh/proxy.py" \ - com.abhinavsingh.description="โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on \ - Network monitoring, controls & Application development, testing, debugging." \ - com.abhinavsingh.url="https://github.com/abhinavsingh/proxy.py" \ - com.abhinavsingh.vcs-url="https://github.com/abhinavsingh/proxy.py" \ - com.abhinavsingh.docker.cmd="docker run -it --rm -p 8899:8899 abhinavsingh/proxy.py" + com.abhinavsingh.description="โšก Fast โ€ข ๐Ÿชถ Lightweight โ€ข 0๏ธโƒฃ Dependency โ€ข ๐Ÿ”Œ Pluggable โ€ข \ + ๐Ÿ˜ˆ TLS interception โ€ข ๐Ÿ”’ DNS-over-HTTPS โ€ข ๐Ÿ”ฅ Poor Man's VPN โ€ข โช Reverse & โฉ Forward โ€ข \ + ๐Ÿ‘ฎ๐Ÿฟ \"Proxy Server\" framework โ€ข ๐ŸŒ \"Web Server\" framework โ€ข โžต โžถ โžท โž  \"PubSub\" framework โ€ข \ + ๐Ÿ‘ท \"Work\" acceptor & executor framework" \ + com.abhinavsingh.url="https://github.com/abhinavsingh/proxy.py" \ + com.abhinavsingh.vcs-url="https://github.com/abhinavsingh/proxy.py" \ + com.abhinavsingh.docker.cmd="docker run -it --rm -p 8899:8899 abhinavsingh/proxy.py" +ENV PYTHONUNBUFFERED 1 +ARG PROXYPY_PKG_PATH -COPY --from=builder /deps /usr/local +COPY README.md / +COPY $PROXYPY_PKG_PATH / +RUN pip install --upgrade pip && \ + pip install \ + --no-index \ + --find-links file:/// \ + proxy.py && \ + rm *.whl -# Install openssl to enable TLS interception within container +# Install openssl to enable TLS interception & HTTPS proxy options within container +# NOTE: You can comment out this line if you don't intend to use those features. RUN apk update && apk add openssl EXPOSE 8899/tcp diff --git a/LICENSE b/LICENSE index d607876fbf..57b6da7646 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2013-2020 by Abhinav Singh and contributors. +Copyright (c) 2013-present by Abhinav Singh and contributors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, diff --git a/MANIFEST.in b/MANIFEST.in index 7b741275f0..bdbb7aebbe 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,5 @@ -include LICENSE -include README.md -include requirements.txt +exclude ProxyPy.png +exclude Dashboard.png +exclude shortlink.gif +prune dashboard +prune menubar diff --git a/Makefile b/Makefile index e0331a5ece..d54ab9242f 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,10 @@ SHELL := /bin/bash NS ?= abhinavsingh IMAGE_NAME ?= proxy.py -VERSION ?= v$(shell python -m proxy --version) -LATEST_TAG := $(NS)/$(IMAGE_NAME):latest -IMAGE_TAG := $(NS)/$(IMAGE_NAME):$(VERSION) +# Override to target specific versions of proxy.py +PROXYPY_CONTAINER_VERSION := latest +# Used by container build and run targets +PROXYPY_CONTAINER_TAG := $(NS)/$(IMAGE_NAME):$(PROXYPY_CONTAINER_VERSION) HTTPS_KEY_FILE_PATH := https-key.pem HTTPS_CERT_FILE_PATH := https-cert.pem @@ -15,22 +16,24 @@ CA_KEY_FILE_PATH := ca-key.pem CA_CERT_FILE_PATH := ca-cert.pem CA_SIGNING_KEY_FILE_PATH := ca-signing-key.pem -.PHONY: all https-certificates ca-certificates autopep8 devtools -.PHONY: lib-version lib-clean lib-test lib-package lib-coverage lib-lint -.PHONY: lib-release-test lib-release lib-profile -.PHONY: container container-run container-release -.PHONY: dashboard dashboard-clean +# Dummy invalid hardcoded value +PROXYPY_PKG_PATH := dist/proxy.py.whl +BUILDX_TARGET_PLATFORM := linux/amd64 -all: lib-test +OPEN=$(shell which open) +UNAME := $(shell uname) +ifeq ($(UNAME), Linux) +OPEN=$(shell which xdg-open) +endif -devtools: - pushd dashboard && npm run devtools && popd +.PHONY: all https-certificates sign-https-certificates ca-certificates +.PHONY: lib-check lib-clean lib-test lib-package lib-coverage lib-lint lib-pytest +.PHONY: lib-release-test lib-release lib-profile lib-doc +.PHONY: lib-dep lib-flake8 lib-mypy +.PHONY: container container-run container-release container-build container-buildx +.PHONY: devtools dashboard dashboard-clean -autopep8: - autopep8 --recursive --in-place --aggressive examples - autopep8 --recursive --in-place --aggressive proxy - autopep8 --recursive --in-place --aggressive tests - autopep8 --recursive --in-place --aggressive setup.py +all: lib-test https-certificates: # Generate server key @@ -74,8 +77,8 @@ ca-certificates: python -m proxy.common.pki remove_passphrase \ --private-key-path $(CA_SIGNING_KEY_FILE_PATH) -lib-version: - python version-check.py +lib-check: + python check.py lib-clean: find . -name '*.pyc' -exec rm -f {} + @@ -89,15 +92,31 @@ lib-clean: rm -rf .pytest_cache rm -rf .hypothesis +lib-dep: + pip install --upgrade pip && \ + pip install \ + -r requirements.txt \ + -r requirements-testing.txt \ + -r requirements-release.txt \ + -r requirements-tunnel.txt && \ + pip install "setuptools>=42" + lib-lint: - flake8 --ignore=W504 --max-line-length=127 --max-complexity=19 examples/ proxy/ tests/ setup.py - mypy --strict --ignore-missing-imports examples/ proxy/ tests/ setup.py + python -m tox -e lint + +lib-flake8: + tox -e lint -- flake8 --all-files + +lib-mypy: + tox -e lint -- mypy --all-files -lib-test: lib-clean lib-version lib-lint - pytest -v tests/ +lib-pytest: + python -m tox -e python -- -v -lib-package: lib-clean lib-version - python setup.py sdist +lib-test: lib-clean lib-check lib-lint lib-pytest + +lib-package: lib-clean lib-check + python -m tox -e cleanup-dists,build-dists,metadata-validation lib-release-test: lib-package twine upload --verbose --repository-url https://test.pypi.org/legacy/ dist/* @@ -105,12 +124,32 @@ lib-release-test: lib-package lib-release: lib-package twine upload dist/* +lib-doc: + python -m tox -e build-docs && \ + $(OPEN) .tox/build-docs/docs_out/index.html + lib-coverage: - pytest --cov=proxy --cov-report=html tests/ - open htmlcov/index.html + pytest --cov=proxy --cov=tests --cov-report=html tests/ && \ + $(OPEN) htmlcov/index.html lib-profile: - sudo py-spy record -o profile.svg -t -F -s -- python -m proxy + ulimit -n 65536 && \ + sudo py-spy record \ + -o profile.svg \ + -t -F -s -- \ + python -m proxy \ + --num-acceptors 1 \ + --num-workers 1 \ + --disable-http-proxy \ + --enable-web-server \ + --plugin proxy.plugin.WebServerPlugin \ + --local-executor \ + --backlog 65536 \ + --open-file-limit 65536 \ + --log-file /dev/null + +devtools: + pushd dashboard && npm run devtools && popd dashboard: pushd dashboard && npm run build && popd @@ -118,12 +157,25 @@ dashboard: dashboard-clean: if [[ -d dashboard/public ]]; then rm -rf dashboard/public; fi -container: - docker build -t $(LATEST_TAG) -t $(IMAGE_TAG) . - -container-release: - docker push $(IMAGE_TAG) - docker push $(LATEST_TAG) +container: lib-package + $(MAKE) container-build -e PROXYPY_PKG_PATH=$$(ls dist/*.whl) + +# Usage: +# +# make container-buildx \ +# -e PROXYPY_PKG_PATH=$(ls dist/*.whl) \ +# -e BUILDX_TARGET_PLATFORM=linux/arm64 \ +# -e PROXYPY_CONTAINER_VERSION=latest +container-buildx: + docker buildx build \ + --platform $(BUILDX_TARGET_PLATFORM) \ + -t $(PROXYPY_CONTAINER_TAG) \ + --build-arg PROXYPY_PKG_PATH=$(PROXYPY_PKG_PATH) . + +container-build: + docker build \ + -t $(PROXYPY_CONTAINER_TAG) \ + --build-arg PROXYPY_PKG_PATH=$(PROXYPY_PKG_PATH) . container-run: - docker run -it -p 8899:8899 --rm $(LATEST_TAG) + docker run -it -p 8899:8899 --rm $(PROXYPY_CONTAINER_TAG) diff --git a/ProxyPy.png b/ProxyPy.png index 50ac830a91..220503f924 100644 Binary files a/ProxyPy.png and b/ProxyPy.png differ diff --git a/README.md b/README.md index 2c09bec856..e9ba838e9f 100644 --- a/README.md +++ b/README.md @@ -1,34 +1,32 @@ [![Proxy.Py](https://github.com/raw/abhinavsingh/proxy.py/develop/ProxyPy.png)](https://github.com/abhinavsingh/proxy.py) -[![License](https://img.shields.io/github/license/abhinavsingh/proxy.py.svg)](https://opensource.org/licenses/BSD-3-Clause) -[![PyPi Monthly](https://img.shields.io/pypi/dm/proxy.py.svg?color=green)](https://pypi.org/project/proxy.py/) -[![Docker Pulls](https://img.shields.io/docker/pulls/abhinavsingh/proxy.py?color=green)](https://hub.docker.com/r/abhinavsingh/proxy.py) -[![No Dependencies](https://img.shields.io/static/v1?label=dependencies&message=none&color=green)](https://github.com/abhinavsingh/proxy.py) - -[![Proxy.py Library Build Status](https://github.com/abhinavsingh/proxy.py/workflows/Proxy.py%20Library/badge.svg)](https://github.com/abhinavsingh/proxy.py/actions) -[![Proxy.py Docker Build Status](https://github.com/abhinavsingh/proxy.py/workflows/Proxy.py%20Docker/badge.svg)](https://github.com/abhinavsingh/proxy.py/actions) -[![Proxy.py Docker Build Status](https://github.com/abhinavsingh/proxy.py/workflows/Proxy.py%20Dashboard/badge.svg)](https://github.com/abhinavsingh/proxy.py/actions) -[![Proxy.py Docker Build Status](https://github.com/abhinavsingh/proxy.py/workflows/Proxy.py%20Brew/badge.svg)](https://github.com/abhinavsingh/proxy.py/actions) -[![Coverage](https://codecov.io/gh/abhinavsingh/proxy.py/branch/develop/graph/badge.svg)](https://codecov.io/gh/abhinavsingh/proxy.py) - -[![Tested With MacOS, Ubuntu, Windows, Android, Android Emulator, iOS, iOS Simulator](https://img.shields.io/static/v1?label=tested%20with&message=mac%20OS%20%F0%9F%92%BB%20%7C%20Ubuntu%20%F0%9F%96%A5%20%7C%20Windows%20%F0%9F%92%BB&color=brightgreen)](https://abhinavsingh.com/proxy-py-a-lightweight-single-file-http-proxy-server-in-python/) -[![Android, Android Emulator](https://img.shields.io/static/v1?label=tested%20with&message=Android%20%F0%9F%93%B1%20%7C%20Android%20Emulator%20%F0%9F%93%B1&color=brightgreen)](https://abhinavsingh.com/proxy-py-a-lightweight-single-file-http-proxy-server-in-python/) -[![iOS, iOS Simulator](https://img.shields.io/static/v1?label=tested%20with&message=iOS%20%F0%9F%93%B1%20%7C%20iOS%20Simulator%20%F0%9F%93%B1&color=brightgreen)](https://abhinavsingh.com/proxy-py-a-lightweight-single-file-http-proxy-server-in-python/) - -[![Maintenance](https://img.shields.io/static/v1?label=maintained%3F&message=yes&color=green)](https://github.com/abhinavsingh/proxy.py/graphs/commit-activity) -[![Ask Me Anything](https://img.shields.io/static/v1?label=need%20help%3F&message=ask&color=green)](https://twitter.com/imoracle) -[![Contributions Welcome](https://img.shields.io/static/v1?label=contributions&message=welcome%20%F0%9F%91%8D&color=green)](https://github.com/abhinavsingh/proxy.py/issues) -[![Gitter](https://badges.gitter.im/proxy-py/community.svg)](https://gitter.im/proxy-py/community) - -[![Python 3.x](https://img.shields.io/static/v1?label=Python&message=3.6%20%7C%203.7%20%7C%203.8%20%7C%203.9&color=blue)](https://www.python.org/) +[//]: # (DO-NOT-REMOVE-docs-badges-START) + +[![PyPi Monthly](https://img.shields.io/pypi/dm/proxy.py?style=for-the-badge&color=darkgreen)](https://pypi.org/project/proxy.py/) +[![Docker Pulls](https://img.shields.io/docker/pulls/abhinavsingh/proxy.py?style=for-the-badge&color=darkgreen)](https://hub.docker.com/r/abhinavsingh/proxy.py) +[![No Dependencies](https://img.shields.io/static/v1?label=dependencies&message=0&style=for-the-badge&color=darkgreen)](https://github.com/abhinavsingh/proxy.py) +[![Gitter](https://img.shields.io/gitter/room/abhinavsingh/proxy.py?style=for-the-badge&color=darkgreen)](https://gitter.im/proxy-py/community) +[![License](https://img.shields.io/github/license/abhinavsingh/proxy.py?style=for-the-badge&color=darkgreen)](https://github.com/abhinavsingh/proxy.py/blob/develop/LICENSE) + +[![Tested With MacOS, Ubuntu, Windows, Android, Android Emulator, iOS, iOS Simulator](https://img.shields.io/static/v1?label=tested%20with&message=mac%20OS%20%F0%9F%92%BB%20%7C%20Ubuntu%20%F0%9F%96%A5%20%7C%20Windows%20%F0%9F%92%BB&color=darkgreen&style=for-the-badge)](https://abhinavsingh.com/proxy-py-a-lightweight-single-file-http-proxy-server-in-python/) +[![Android, Android Emulator](https://img.shields.io/static/v1?label=tested%20with&message=Android%20%F0%9F%93%B1%20%7C%20Android%20Emulator%20%F0%9F%93%B1&color=darkgreen&style=for-the-badge)](https://abhinavsingh.com/proxy-py-a-lightweight-single-file-http-proxy-server-in-python/) +[![iOS, iOS Simulator](https://img.shields.io/static/v1?label=tested%20with&message=iOS%20%F0%9F%93%B1%20%7C%20iOS%20Simulator%20%F0%9F%93%B1&color=darkgreen&style=for-the-badge)](https://abhinavsingh.com/proxy-py-a-lightweight-single-file-http-proxy-server-in-python/) + +[![pypi version](https://img.shields.io/pypi/v/proxy.py)](https://pypi.org/project/proxy.py/) +[![Python 3.x](https://img.shields.io/static/v1?label=Python&message=3.6%20%7C%203.7%20%7C%203.8%20%7C%203.9%20%7C%203.10&color=blue)](https://www.python.org/) [![Checked with mypy](https://img.shields.io/static/v1?label=MyPy&message=checked&color=blue)](http://mypy-lang.org/) +[![lib](https://github.com/abhinavsingh/proxy.py/actions/workflows/test-library.yml/badge.svg)](https://github.com/abhinavsingh/proxy.py/actions/workflows/test-library.yml) +[![codecov](https://codecov.io/gh/abhinavsingh/proxy.py/branch/develop/graph/badge.svg?token=Zh9J7b4la2)](https://codecov.io/gh/abhinavsingh/proxy.py) -[![Become a Backer](https://opencollective.com/proxypy/tiers/backer.svg?avatarHeight=72)](https://opencollective.com/proxypy) +[![Contributions Welcome](https://img.shields.io/static/v1?label=Contributions&message=Welcome%20%F0%9F%91%8D&color=darkgreen)](https://github.com/abhinavsingh/proxy.py/issues) +[![Need Help](https://img.shields.io/static/v1?label=Need%20Help%3F&message=Ask&color=darkgreen)](https://twitter.com/imoracle) +[![Sponsored by Jaxl Innovations Private Limited](https://img.shields.io/static/v1?label=Sponsored%20By&message=Jaxl%20Innovations%20Private%20Limited&color=darkgreen)](https://github.com/jaxl-innovations-private-limited) # Table of Contents - [Features](#features) - [Install](#install) + - [Stable vs Develop](#stable-vs-develop) - [Using PIP](#using-pip) - [Stable version](#stable-version-with-pip) - [Development version](#development-version-with-pip) @@ -58,6 +56,9 @@ - [Proxy Pool Plugin](#proxypoolplugin) - [FilterByClientIpPlugin](#filterbyclientipplugin) - [ModifyChunkResponsePlugin](#modifychunkresponseplugin) + - [CloudflareDnsResolverPlugin](#cloudflarednsresolverplugin) + - [CustomDnsResolverPlugin](#customdnsresolverplugin) + - [CustomNetworkInterface](#customnetworkinterface) - [HTTP Web Server Plugins](#http-web-server-plugins) - [Reverse Proxy](#reverse-proxy) - [Web Server Route](#web-server-route) @@ -71,19 +72,12 @@ - [Embed proxy.py](#embed-proxypy) - [Blocking Mode](#blocking-mode) - [Non-blocking Mode](#non-blocking-mode) + - [Ephemeral Port](#ephemeral-port) - [Loading Plugins](#loading-plugins) - [Unit testing with proxy.py](#unit-testing-with-proxypy) - - [proxy.TestCase](#proxytestcase) + - [`proxy.TestCase`](#proxytestcase) - [Override Startup Flags](#override-startup-flags) - - [With unittest.TestCase](#with-unittesttestcase) -- [Plugin Developer and Contributor Guide](#plugin-developer-and-contributor-guide) - - [Everything is a plugin](#everything-is-a-plugin) - - [Internal Architecture](#internal-architecture) - - [Internal Documentation](#internal-documentation) - - [Development Guide](#development-guide) - - [Setup Local Environment](#setup-local-environment) - - [Setup pre-commit hook](#setup-pre-commit-hook) - - [Sending a Pull Request](#sending-a-pull-request) + - [With `unittest.TestCase`](#with-unittesttestcase) - [Utilities](#utilities) - [TCP](#tcp-sockets) - [new_socket_connection](#new_socket_connection) @@ -94,6 +88,9 @@ - [Public Key Infrastructure](#pki) - [API Usage](#api-usage) - [CLI Usage](#cli-usage) +- [Run Dashboard](#run-dashboard) + - [Inspect Traffic](#inspect-traffic) +- [Chrome DevTools Protocol](#chrome-devtools-protocol) - [Frequently Asked Questions](#frequently-asked-questions) - [Threads vs Threadless](#threads-vs-threadless) - [SyntaxError: invalid syntax](#syntaxerror-invalid-syntax) @@ -101,111 +98,194 @@ - [Unable to connect with proxy.py from remote host](#unable-to-connect-with-proxypy-from-remote-host) - [Basic auth not working with a browser](#basic-auth-not-working-with-a-browser) - [Docker image not working on MacOS](#docker-image-not-working-on-macos) - - [ValueError: filedescriptor out of range in select](#valueerror-filedescriptor-out-of-range-in-select) + - [`ValueError: filedescriptor out of range in select`](#valueerror-filedescriptor-out-of-range-in-select) - [None:None in access logs](#nonenone-in-access-logs) + - [OSError when wrapping client for TLS Interception](#oserror-when-wrapping-client-for-tls-interception) +- [Plugin Developer and Contributor Guide](#plugin-developer-and-contributor-guide) + - [High level architecture](#high-level-architecture) + - [Everything is a plugin](#everything-is-a-plugin) + - [Internal Documentation](#internal-documentation) + - [Development Guide](#development-guide) + - [Setup Local Environment](#setup-local-environment) + - [Setup Git Hooks](#setup-git-hooks) + - [Sending a Pull Request](#sending-a-pull-request) +- [Benchmarks](#benchmarks) - [Flags](#flags) - [Changelog](#changelog) - [v2.x](#v2x) - [v1.x](#v1x) - [v0.x](#v0x) +[//]: # (DO-NOT-REMOVE-docs-badges-END) + # Features - Fast & Scalable - Scales by using all available cores on the system - - Threadless executions using coroutine + - Threadless executions using asyncio - Made to handle `tens-of-thousands` connections / sec - ```bash - # On Macbook Pro 2015 / 2.8 GHz Intel Core i7 - โฏ hey -n 10000 -c 100 http://localhost:8899/ - - Summary: - Total: 0.6157 secs - Slowest: 0.1049 secs - Fastest: 0.0007 secs - Average: 0.0055 secs - Requests/sec: 16240.5444 - - Total data: 800000 bytes - Size/request: 80 bytes - - Response time histogram: - 0.001 [1] | - 0.011 [9565] |โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ–  - 0.022 [332] |โ–  + ```console + # On Macbook Pro 2019 / 2.4 GHz 8-Core Intel Core i9 / 32 GB RAM + โฏ ./helper/benchmark.sh + CONCURRENCY: 100 workers, TOTAL REQUESTS: 100000 req, QPS: 8000 req/sec, TIMEOUT: 1 sec + + Summary: + Total: 3.1217 secs + Slowest: 0.0499 secs + Fastest: 0.0004 secs + Average: 0.0030 secs + Requests/sec: 32033.7261 + + Total data: 1900000 bytes + Size/request: 19 bytes + + Response time histogram: + 0.000 [1] | + 0.005 [92268] |โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ–  + 0.010 [7264] |โ– โ– โ–  + 0.015 [318] | + 0.020 [102] | + 0.025 [32] | + 0.030 [6] | + 0.035 [4] | + 0.040 [1] | + 0.045 [2] | + 0.050 [2] | + + + Latency distribution: + 10% in 0.0017 secs + 25% in 0.0020 secs + 50% in 0.0025 secs + 75% in 0.0036 secs + 90% in 0.0050 secs + 95% in 0.0060 secs + 99% in 0.0087 secs + + Details (average, fastest, slowest): + DNS+dialup: 0.0000 secs, 0.0004 secs, 0.0499 secs + DNS-lookup: 0.0000 secs, 0.0000 secs, 0.0000 secs + req write: 0.0000 secs, 0.0000 secs, 0.0020 secs + resp wait: 0.0030 secs, 0.0004 secs, 0.0462 secs + resp read: 0.0000 secs, 0.0000 secs, 0.0027 secs + + Status code distribution: + [200] 100000 responses ``` + PS: `proxy.py` and benchmark tools are running on the same machine during the above load test. + Checkout the repo and try it for yourself. See [Benchmarks](#benchmarks) for more details. + - Lightweight - Uses only `~5-20MB` RAM - No external dependency other than standard Python library - Programmable - - Optionally enable builtin Web Server - - Customize proxy and http routing via [plugins](https://github.com/abhinavsingh/proxy.py/tree/develop/proxy/plugin) - - Enable plugin using command line option e.g. `--plugins proxy.plugin.CacheResponsesPlugin` - - Plugin API is currently in development phase, expect breaking changes. -- Realtime Dashboard - - Optionally enable bundled dashboard. + - Customize proxy behavior using [Proxy Server Plugins](#http-proxy-plugins). Example: + - `--plugins proxy.plugin.ProxyPoolPlugin` + - Optionally, enable builtin [Web Server Plugins](#http-web-server-plugins). Example: + - `--plugins proxy.plugin.ReverseProxyPlugin` + - Plugin API is currently in development phase, expect breaking changes +- Real-time Dashboard + - Optionally, enable [proxy.py dashboard](#run-dashboard). - Available at `http://localhost:8899/dashboard`. - - Inspect, Monitor, Control and Configure `proxy.py` at runtime. - - Extend dashboard using plugins. - - Dashboard is currently in development phase, expect breaking changes. + - [Inspect, Monitor, Control and Configure](#inspect-traffic) `proxy.py` at runtime + - [Chrome DevTools Protocol](#chrome-devtools-protocol) support + - Extend dashboard using plugins + - Dashboard is currently in development phase, expect breaking changes - Secure - - Enable end-to-end encryption between clients and `proxy.py` using TLS + - Enable end-to-end encryption between clients and `proxy.py` - See [End-to-End Encryption](#end-to-end-encryption) +- Private + - Everyone deserves privacy. Browse with malware and adult content protection + - See [DNS-over-HTTPS](#cloudflarednsresolverplugin) - Man-In-The-Middle - Can decrypt TLS traffic between clients and upstream servers - See [TLS Interception](#tls-interception) - Supported proxy protocols - `http(s)` - `http1` - - `http1.1` pipeline + - `http1.1` with pipeline - `http2` - `websockets` +- Support for `HAProxy Protocol` + - See `--enable-proxy-protocol` flag +- Static file server support + - See `--enable-static-server` and `--static-server-dir` flags - Optimized for large file uploads and downloads -- IPv4 and IPv6 support + - See `--client-recvbuf-size` and `--server-recvbuf-size` flag +- `IPv4` and `IPv6` support + - See `--hostname` flag +- Unix domain socket support + - See `--unix-socket-path` flag - Basic authentication support -- Can serve a [PAC (Proxy Auto-configuration)](https://en.wikipedia.org/wiki/Proxy_auto-config) file + - See `--basic-auth` flag +- PAC (Proxy Auto-configuration) support - See `--pac-file` and `--pac-file-url-path` flags # Install +## Stable vs Develop + +`master` branch contains latest stable code and is available via `PyPi` repository + +`develop` branch contains cutting edge changes + +Development branch is kept stable *(most of the times)*. But if you want 100% reliability and serving users in production environment, always use stable version from `PyPi` or `Docker` container from `hub.docker.com`. + ## Using PIP ### Stable Version with PIP Install from `PyPi` -```bash +```console โฏ pip install --upgrade proxy.py ``` or from GitHub `master` branch -```bash +```console โฏ pip install git+https://github.com/abhinavsingh/proxy.py.git@master ``` ### Development Version with PIP -```bash +```console โฏ pip install git+https://github.com/abhinavsingh/proxy.py.git@develop ``` ## Using Docker -#### Stable Version from Docker Hub +Stable version container releases are available for following platforms: + +- `linux/386` +- `linux/amd64` +- `linux/arm/v6` +- `linux/arm/v7` +- `linux/arm64/v8` +- `linux/ppc64le` +- `linux/s390x` -```bash +### Stable Version from Docker Hub + +Run `proxy.py` latest container: + +```console โฏ docker run -it -p 8899:8899 --rm abhinavsingh/proxy.py:latest ``` -#### Build Development Version Locally +To run specific target platform container on multi-platform supported servers: + +```console +โฏ docker run -it -p 8899:8899 --rm --platform linux/arm64/v8 abhinavsingh/proxy.py:latest +``` + +### Build Development Version Locally -```bash +```console โฏ git clone https://github.com/abhinavsingh/proxy.py.git -โฏ cd proxy.py -โฏ make container +โฏ cd proxy.py && make container โฏ docker run -it -p 8899:8899 --rm abhinavsingh/proxy.py:latest ``` @@ -214,15 +294,20 @@ or from GitHub `master` branch ## Using HomeBrew +Updated formulae for `HomeBrew` are maintained in `develop` branch under the `helper/homebrew` directory. + +- `stable` formulae installs the package from `master` branch. +- `develop` formulae installs the package from `develop` branch. + ### Stable Version with HomeBrew -```bash +```console โฏ brew install https://github.com/raw/abhinavsingh/proxy.py/develop/helper/homebrew/stable/proxy.rb ``` ### Development Version with HomeBrew -```bash +```console โฏ brew install https://github.com/raw/abhinavsingh/proxy.py/develop/helper/homebrew/develop/proxy.rb ``` @@ -233,41 +318,51 @@ or from GitHub `master` branch When `proxy.py` is installed using `pip`, an executable named `proxy` is placed under your `$PATH`. -#### Run it +### Run it -Simply type `proxy` on command line to start it with default configuration. +Simply type `proxy` on command line to start with default configuration. -```bash +```console โฏ proxy -...[redacted]... - Loaded plugin proxy.http_proxy.HttpProxyPlugin -...[redacted]... - Starting 8 workers -...[redacted]... - Started server on ::1:8899 +...[redacted]... - Loaded plugin proxy.http.proxy.HttpProxyPlugin +...[redacted]... - Started 8 threadless workers +...[redacted]... - Started 8 acceptors +...[redacted]... - Listening on 127.0.0.1:8899 ``` -#### Understanding logs +### Understanding logs Things to notice from above logs: -- `Loaded plugin` - `proxy.py` will load `proxy.http.proxy.HttpProxyPlugin` by default. - As name suggests, this core plugin adds `http(s)` proxy server capabilities to `proxy.py` +- `Loaded plugin` + - `proxy.py` will load `proxy.http.proxy.HttpProxyPlugin` by default + - As name suggests, this core plugin adds `http(s)` proxy server capabilities to `proxy.py` instance + +- `Started N threadless workers` + - By default, `proxy.py` will start as many worker processes as there are CPU cores on the machine + - Use `--num-workers` flag to customize number of worker processes + - See [Threads vs Threadless](#threads-vs-threadless) to understand how to control execution mode -- `Started N workers` - Use `--num-workers` flag to customize number of worker processes. - By default, `proxy.py` will start as many workers as there are CPU cores on the machine. +- `Started N acceptors` + - By default, `proxy.py` will start as many acceptor processes as there are CPU cores on the machine + - Use `--num-acceptors` flag to customize number of acceptor processes + - See [High Level Architecture](#high-level-architecture) to understand relationship between acceptors and workers -- `Started server on ::1:8899` - By default, `proxy.py` listens on IPv6 `::1`, which - is equivalent of IPv4 `127.0.0.1`. If you want to access `proxy.py` externally, - use `--hostname ::` or `--hostname 0.0.0.0` or bind to any other interface available - on your machine. +- `Started server on ::1:8899` + - By default, `proxy.py` listens on IPv6 `::1`, which is equivalent of IPv4 `127.0.0.1` + - If you want to access `proxy.py` from external host, use `--hostname ::` or `--hostname 0.0.0.0` or bind to any other interface available on your machine. + - See [CustomNetworkInterface](#customnetworkinterface) for how to customize `proxy.py` *public IP seen by upstream servers*. -- `Port 8899` - Use `--port` flag to customize default TCP port. +- `Port 8899` + - Use `--port` flag to customize default TCP port. -#### Enable DEBUG logging +### Enable DEBUG logging -All the logs above are `INFO` level logs, default `--log-level` for `proxy.py`. +All the logs above are `INFO` level logs, default `--log-level` for `proxy.py` Lets start `proxy.py` with `DEBUG` level logging: -```bash +```console โฏ proxy --log-level d ...[redacted]... - Open file descriptor soft limit set to 1024 ...[redacted]... - Loaded plugin proxy.http_proxy.HttpProxyPlugin @@ -275,11 +370,18 @@ Lets start `proxy.py` with `DEBUG` level logging: ...[redacted]... - Started server on ::1:8899 ``` -As we can see, before starting up: +You can use single letter to customize log level. Example: +- `d = DEBUG` +- `i = INFO` +- `w = WARNING` +- `e = ERROR` +- `c = CRITICAL` -- `proxy.py` also tried to set open file limit `ulimit` on the system. -- Default value for `--open-file-limit` used is `1024`. -- `--open-file-limit` flag is a no-op on `Windows` operating systems. +As we can see from the above logs, before starting up: + +- `proxy.py` tried to set open file limit `ulimit` on the system +- Default value for `--open-file-limit` used is `1024` +- `--open-file-limit` flag is a no-op on `Windows` operating systems See [flags](#flags) for full list of available configuration options. @@ -292,50 +394,58 @@ To start `proxy.py` from source code follow these instructions: - Clone repo - ```bash + ```console โฏ git clone https://github.com/abhinavsingh/proxy.py.git โฏ cd proxy.py ``` - Create a Python 3 virtual env - ```bash + ```console โฏ python3 -m venv venv โฏ source venv/bin/activate ``` - Install deps - ```bash - โฏ pip install -r requirements.txt - โฏ pip install -r requirements-testing.txt + ```console + โฏ make lib-dep ``` -- Run tests +- Generate `proxy/common/_scm_version.py` + + NOTE: *Following step is not necessary for editable installs.* + + This file writes SCM detected version to `proxy/common/_scm_version.py` file. + + ```console + โฏ ./write-scm-version.sh + ``` - ```bash +- Optionally, run tests + + ```console โฏ make ``` -- Run proxy.py +- Run `proxy.py` - ```bash + ```console โฏ python -m proxy ``` -Also see [Plugin Developer and Contributor Guide](#plugin-developer-and-contributor-guide) +See [Plugin Developer and Contributor Guide](#plugin-developer-and-contributor-guide) if you plan to work with `proxy.py` source code. ## Docker image -#### Customize startup flags +### Customize startup flags By default `docker` binary is started with IPv4 networking flags: --hostname 0.0.0.0 --port 8899 -To override input flags, start docker image as follows. -For example, to check `proxy.py` version within Docker image: +You can override flag from command line when starting the docker container. For example, to check `proxy.py` version within the docker container, run: โฏ docker run -it \ -p 8899:8899 \ @@ -357,11 +467,11 @@ For example, to check `proxy.py` version within Docker image: Add support for short links in your favorite browsers / applications. -[![Shortlink Plugin](https://github.com/raw/abhinavsingh/proxy.py/develop/shortlink.gif)](https://github.com/abhinavsingh/proxy.py#shortlinkplugin) +[![Shortlink Plugin](https://github.com/raw/abhinavsingh/proxy.py/develop/shortlink.gif)](https://github.com/abhinavsingh/proxy.py#user-content-shortlinkplugin) Start `proxy.py` as: -```bash +```console โฏ proxy \ --plugins proxy.plugin.ShortLinkPlugin ``` @@ -372,17 +482,17 @@ across all browsers. Following short links are enabled by default: -| Short Link | Destination URL | -| :--------: | :--------------: | -| a/ | amazon.com | -| i/ | instagram.com | -| l/ | linkedin.com | -| f/ | facebook.com | -| g/ | google.com | -| t/ | twitter.com | -| w/ | web.whatsapp.com | -| y/ | youtube.com | -| proxy/ | localhost:8899 | +| Short Link | Destination URL | +| :--------: | :--------------: | +| a/ | `amazon.com` | +| i/ | `instagram.com` | +| l/ | `linkedin.com` | +| f/ | `facebook.com` | +| g/ | `google.com` | +| t/ | `twitter.com` | +| w/ | `web.whatsapp.com` | +| y/ | `youtube.com` | +| proxy/ | `localhost:8899` | ### ModifyPostDataPlugin @@ -390,17 +500,17 @@ Modifies POST request body before sending request to upstream server. Start `proxy.py` as: -```bash +```console โฏ proxy \ --plugins proxy.plugin.ModifyPostDataPlugin ``` -By default plugin replaces POST body content with hardcoded `b'{"key": "modified"}'` +By default plugin replaces POST body content with hard-coded `b'{"key": "modified"}'` and enforced `Content-Type: application/json`. Verify the same using `curl -x localhost:8899 -d '{"key": "value"}' http://httpbin.org/post` -```bash +```console { "args": {}, "data": "{\"key\": \"modified\"}", @@ -444,20 +554,20 @@ without need of an actual upstream REST API server. Start `proxy.py` as: -```bash +```console โฏ proxy \ --plugins proxy.plugin.ProposedRestApiPlugin ``` Verify mock API response using `curl -x localhost:8899 http://api.example.com/v1/users/` -```bash +```console {"count": 2, "next": null, "previous": null, "results": [{"email": "you@example.com", "groups": [], "url": "api.example.com/v1/users/1/", "username": "admin"}, {"email": "someone@example.com", "groups": [], "url": "api.example.com/v1/users/2/", "username": "admin"}]} ``` Verify the same by inspecting `proxy.py` logs: -```bash +```console 2019-09-27 12:44:02,212 - INFO - pid:7077 - access_log:1210 - ::1:64792 - GET None:None/v1/users/ - None None - 0 byte ``` @@ -475,7 +585,7 @@ also running on `8899` port. Start `proxy.py` and enable inbuilt web server: -```bash +```console โฏ proxy \ --enable-web-server \ --plugins proxy.plugin.RedirectToCustomServerPlugin @@ -505,18 +615,18 @@ Along with the proxy request log, you must also see a http web server request lo ### FilterByUpstreamHostPlugin Drops traffic by inspecting upstream host. -By default, plugin drops traffic for `google.com` and `www.google.com`. +By default, plugin drops traffic for `facebook.com` and `www.facebok.com`. Start `proxy.py` as: -```bash +```console โฏ proxy \ --plugins proxy.plugin.FilterByUpstreamHostPlugin ``` -Verify using `curl -v -x localhost:8899 http://google.com`: +Verify using `curl -v -x localhost:8899 http://facebook.com`: -```bash +```console ... [redacted] ... < HTTP/1.1 418 I'm a tea pot < Proxy-agent: proxy.py v1.0.0 @@ -529,7 +639,7 @@ Above `418 I'm a tea pot` is sent by our plugin. Verify the same by inspecting logs for `proxy.py`: -```bash +```console 2019-09-24 19:21:37,893 - ERROR - pid:50074 - handle_readables:1347 - HttpProtocolException type raised Traceback (most recent call last): ... [redacted] ... @@ -542,14 +652,14 @@ Caches Upstream Server Responses. Start `proxy.py` as: -```bash +```console โฏ proxy \ --plugins proxy.plugin.CacheResponsesPlugin ``` Verify using `curl -v -x localhost:8899 http://httpbin.org/get`: -```bash +```console ... [redacted] ... < HTTP/1.1 200 OK < Access-Control-Allow-Credentials: true @@ -579,14 +689,14 @@ Verify using `curl -v -x localhost:8899 http://httpbin.org/get`: Get path to the cache file from `proxy.py` logs: -```bash +```console ... [redacted] ... - GET httpbin.org:80/get - 200 OK - 556 bytes ... [redacted] ... - Cached response at /var/folders/k9/x93q0_xn1ls9zy76m2mf2k_00000gn/T/httpbin.org-1569378301.407512.txt ``` Verify contents of the cache file `cat /path/to/your/cache/httpbin.org.txt` -```bash +```console HTTP/1.1 200 OK Access-Control-Allow-Credentials: true Access-Control-Allow-Origin: * @@ -618,14 +728,14 @@ Modifies upstream server responses. Start `proxy.py` as: -```bash +```console โฏ proxy \ --plugins proxy.plugin.ManInTheMiddlePlugin ``` Verify using `curl -v -x localhost:8899 http://google.com`: -```bash +```console ... [redacted] ... < HTTP/1.1 200 OK < Content-Length: 28 @@ -640,26 +750,26 @@ Response body `Hello from man in the middle` is sent by our plugin. Forward incoming proxy requests to a set of upstream proxy servers. -By default, `ProxyPoolPlugin` is hard-coded to use -`localhost:9000` and `localhost:9001` as upstream proxy server. - Let's start upstream proxies first. Start `proxy.py` on port `9000` and `9001` -```bash +```console โฏ proxy --port 9000 ``` -```bash +```console โฏ proxy --port 9001 ``` -Now, start `proxy.py` with `ProxyPoolPlugin` (on default `8899` port): +Now, start `proxy.py` with `ProxyPoolPlugin` (on default `8899` port), +pointing to our upstream proxies at `9000` and `9001` port. -```bash +```console โฏ proxy \ - --plugins proxy.plugin.ProxyPoolPlugin + --plugins proxy.plugin.ProxyPoolPlugin \ + --proxy-pool localhost:9000 \ + --proxy-pool localhost:9001 ``` Make a curl request via `8899` proxy: @@ -676,14 +786,14 @@ plugin blocks traffic from `127.0.0.1` and `::1`. Start `proxy.py` as: -```bash +```console โฏ proxy \ --plugins proxy.plugin.FilterByClientIpPlugin ``` Send a request using `curl -v -x localhost:8899 http://google.com`: -```bash +```console ... [redacted] ... > Proxy-Connection: Keep-Alive > @@ -697,18 +807,18 @@ Modify plugin to your taste e.g. Allow specific IP addresses only. ### ModifyChunkResponsePlugin -This plugin demonstrate how to modify chunked encoded responses. In able to do so, this plugin uses `proxy.py` core to parse the chunked encoded response. Then we reconstruct the response using custom hardcoded chunks, ignoring original chunks received from upstream server. +This plugin demonstrate how to modify chunked encoded responses. In able to do so, this plugin uses `proxy.py` core to parse the chunked encoded response. Then we reconstruct the response using custom hard-coded chunks, ignoring original chunks received from upstream server. Start `proxy.py` as: -```bash +```console โฏ proxy \ --plugins proxy.plugin.ModifyChunkResponsePlugin ``` Verify using `curl -v -x localhost:8899 http://httpbin.org/stream/5`: -```bash +```console ... [redacted] ... modify chunk @@ -718,7 +828,51 @@ plugin * Closing connection 0 ``` -Modify `ModifyChunkResponsePlugin` to your taste. Example, instead of sending hardcoded chunks, parse and modify the original `JSON` chunks received from the upstream server. +Modify `ModifyChunkResponsePlugin` to your taste. Example, instead of sending hard-coded chunks, parse and modify the original `JSON` chunks received from the upstream server. + +### CloudflareDnsResolverPlugin + +This plugin uses `Cloudflare` hosted `DNS-over-HTTPS` [API](https://developers.cloudflare.com/1.1.1.1/encrypted-dns/dns-over-https/make-api-requests/dns-json) (json). + +`DoH` mandates a HTTP2 compliant client. Unfortunately `proxy.py` +does not provide that yet, so we use a dependency. Install it: + +```console +โฏ pip install "httpx[http2]" +``` + +Now start `proxy.py` as: + +```console +โฏ proxy \ + --plugins proxy.plugin.CloudflareDnsResolverPlugin +``` + +By default, `CloudflareDnsResolverPlugin` runs in `security` mode and provides malware protection. +Use `--cloudflare-dns-mode family` to also enable adult content protection too. + +### CustomDnsResolverPlugin + +This plugin demonstrate how to use a custom DNS resolution implementation with `proxy.py`. +This example plugin currently uses Python's in-built resolution mechanism. Customize code +to your taste. Example, query your custom DNS server, implement `DoH` or other mechanisms. + +Start `proxy.py` as: + +```console +โฏ proxy \ + --plugins proxy.plugin.CustomDnsResolverPlugin +``` + +### CustomNetworkInterface + +`HttpProxyBasePlugin.resolve_dns` callback can also be used to configure `network interface` which must be used as the `source_address` for connection to the upstream server. + +See [this thread](https://github.com/abhinavsingh/proxy.py/issues/535#issuecomment-961510862) +for more details. + +PS: There is no plugin named, but [CustomDnsResolverPlugin](#customdnsresolverplugin) +can be easily customized according to your needs. ## HTTP Web Server Plugins @@ -728,7 +882,7 @@ Extend in-built Web Server to add Reverse Proxy capabilities. Start `proxy.py` as: -```bash +```console โฏ proxy --enable-web-server \ --plugins proxy.plugin.ReverseProxyPlugin ``` @@ -736,7 +890,7 @@ Start `proxy.py` as: With default configuration, `ReverseProxyPlugin` plugin is equivalent to following `Nginx` config: -```bash +```console location /get { proxy_pass http://httpbin.org/get } @@ -744,7 +898,7 @@ location /get { Verify using `curl -v localhost:8899/get`: -```bash +```console { "args": {}, "headers": { @@ -763,14 +917,14 @@ Demonstrates inbuilt web server routing using plugin. Start `proxy.py` as: -```bash +```console โฏ proxy --enable-web-server \ --plugins proxy.plugin.WebServerPlugin ``` Verify using `curl -v localhost:8899/http-route-example`, should return: -```bash +```console HTTP route response ``` @@ -783,27 +937,26 @@ on the command line. Plugins are called in the same order as they are passed. Example, say we are using both `FilterByUpstreamHostPlugin` and `RedirectToCustomServerPlugin`. Idea is to drop all incoming `http` -requests for `google.com` and `www.google.com` and redirect other +requests for `facebook.com` and `www.facebook.com` and redirect other `http` requests to our inbuilt web server. Hence, in this scenario it is important to use `FilterByUpstreamHostPlugin` before `RedirectToCustomServerPlugin`. If we enable `RedirectToCustomServerPlugin` before `FilterByUpstreamHostPlugin`, -`google` requests will also get redirected to inbuilt web server, +`facebook` requests will also get redirected to inbuilt web server, instead of being dropped. # End-to-End Encryption -By default, `proxy.py` uses `http` protocol for communication with clients e.g. `curl`, `browser`. -For enabling end-to-end encrypting using `tls` / `https` first generate certificates: +By default, `proxy.py` uses `http` protocol for communication with clients e.g. `curl`, `browser`. For enabling end-to-end encrypting using `tls` / `https` first generate certificates. **Checkout** the repository and run: -```bash +```console make https-certificates ``` Start `proxy.py` as: -```bash +```console โฏ proxy \ --cert-file https-cert.pem \ --key-file https-key.pem @@ -811,7 +964,7 @@ Start `proxy.py` as: Verify using `curl -x https://localhost:8899 --proxy-cacert https-cert.pem https://httpbin.org/get`: -```bash +```console { "args": {}, "headers": { @@ -828,13 +981,13 @@ If you want to avoid passing `--proxy-cacert` flag, also consider signing genera First, generate CA certificates: -```bash +```console make ca-certificates ``` Then, sign SSL certificate: -```bash +```console make sign-https-certificates ``` @@ -845,14 +998,14 @@ Now restart the server with `--cert-file https-signed-cert.pem` flag. Note that By default, `proxy.py` will not decrypt `https` traffic between client and server. To enable TLS interception first generate root CA certificates: -```bash +```console โฏ make ca-certificates ``` Lets also enable `CacheResponsePlugin` so that we can verify decrypted response from the server. Start `proxy.py` as: -```bash +```console โฏ proxy \ --plugins proxy.plugin.CacheResponsesPlugin \ --ca-key-file ca-key.pem \ @@ -860,15 +1013,15 @@ response from the server. Start `proxy.py` as: --ca-signing-key-file ca-signing-key.pem ``` -[![NOTE](https://img.shields.io/static/v1?label=MacOS&message=note&color=yellow)](https://github.com/abhinavsingh/proxy.py#flags) Also provide explicit CA bundle path needed for validation of peer certificates. See `--ca-file` flag. +[![NOTE](https://img.shields.io/static/v1?label=MacOS&message=note&color=yellow)](https://github.com/abhinavsingh/proxy.py#user-content-flags) Also provide explicit CA bundle path needed for validation of peer certificates. See `--ca-file` flag. Verify TLS interception using `curl` -```bash +```console โฏ curl -v -x localhost:8899 --cacert ca-cert.pem https://httpbin.org/get ``` -```bash +```console * issuer: C=US; ST=CA; L=SanFrancisco; O=proxy.py; OU=CA; CN=Proxy PY CA; emailAddress=proxyca@mailserver.com * SSL certificate verify ok. > GET /get HTTP/1.1 @@ -894,7 +1047,7 @@ file from `proxy.py` logs. `โฏ cat /path/to/your/tmp/directory/httpbin.org-1569452863.924174.txt` -```bash +```console HTTP/1.1 200 OK Access-Control-Allow-Credentials: true Access-Control-Allow-Origin: * @@ -933,27 +1086,28 @@ Important notes about TLS Interception with Docker container: - Since `v2.2.0`, `proxy.py` docker container also ships with `openssl`. This allows `proxy.py` to generate certificates on the fly for TLS Interception. -- For security reasons, `proxy.py` docker container doesn't ship with CA certificates. +- For security reasons, `proxy.py` docker container does not ship with + CA certificates. Here is how to start a `proxy.py` docker container with TLS Interception: 1. Generate CA certificates on host computer - ```bash + ```console โฏ make ca-certificates ``` 2. Copy all generated certificates into a separate directory. We'll later mount this directory into our docker container - ```bash + ```console โฏ mkdir /tmp/ca-certificates โฏ cp ca-cert.pem ca-key.pem ca-signing-key.pem /tmp/ca-certificates ``` 3. Start docker container - ```bash + ```console โฏ docker run -it --rm \ -v /tmp/ca-certificates:/tmp/ca-certificates \ -p 8899:8899 \ @@ -971,7 +1125,7 @@ with TLS Interception: 4. From another terminal, try TLS Interception using `curl`. You can omit `--cacert` flag if CA certificate is already trusted by the system. - ```bash + ```console โฏ curl -v \ --cacert ca-cert.pem \ -x 127.0.0.1:8899 \ @@ -980,7 +1134,7 @@ with TLS Interception: 5. Verify `issuer` field from response headers. - ```bash + ```console * Server certificate: * subject: CN=httpbin.org; C=NA; ST=Unavailable; L=Unavailable; O=Unavailable; OU=Unavailable * start date: Jun 17 09:26:57 2020 GMT @@ -992,14 +1146,14 @@ with TLS Interception: 6. Back on docker terminal, copy response dump path logs. - ```bash + ```console ...[redacted]... [I] access_log:338 - 172.17.0.1:56498 - CONNECT httpbin.org:443 - 1031 bytes - 1216.70 ms ...[redacted]... [I] close:49 - Cached response at /tmp/httpbin.org-ae1a927d064e4ab386ea319eb38fe251.txt ``` 7. In another terminal, `cat` the response dump: - ```bash + ```console โฏ docker exec -it $(docker ps | grep proxy.py | awk '{ print $1 }') cat /tmp/httpbin.org-ae1a927d064e4ab386ea319eb38fe251.txt HTTP/1.1 200 OK ...[redacted]... @@ -1013,7 +1167,9 @@ with TLS Interception: **This is a WIP and may not work as documented** -Requires `paramiko` to work. See [requirements-tunnel.txt](https://github.com/abhinavsingh/proxy.py/blob/develop/requirements-tunnel.txt) +Requires `paramiko` to work. + +See [requirements-tunnel.txt](https://github.com/abhinavsingh/proxy.py/blob/develop/requirements-tunnel.txt) ## Proxy Remote Requests Locally @@ -1050,7 +1206,7 @@ running on `localhost`. Start `proxy.py` as: -```bash +```console โฏ # On localhost โฏ proxy --enable-tunnel \ --tunnel-username username \ @@ -1063,7 +1219,7 @@ Start `proxy.py` as: Make a HTTP proxy request on `remote` server and verify that response contains public IP address of `localhost` as origin: -```bash +```console โฏ # On remote โฏ curl -x 127.0.0.1:8899 http://httpbin.org/get { @@ -1080,7 +1236,7 @@ verify that response contains public IP address of `localhost` as origin: Also, verify that `proxy.py` logs on `localhost` contains `remote` IP as client IP. -```bash +```console access_log:328 - remote:52067 - GET httpbin.org:80 ``` @@ -1143,30 +1299,54 @@ Note that: ## Non-blocking Mode Start `proxy.py` in non-blocking embedded mode with default configuration -by using `start` method: Example: +by using `Proxy` context manager: Example: ```python import proxy if __name__ == '__main__': - with proxy.start([]): + with proxy.Proxy([]) as p: # ... your logic here ... ``` Note that: -1. `start` is similar to `main`, except `start` won't block. -1. `start` is a context manager. - It will start `proxy.py` when called and will shut it down - once scope ends. -1. Just like `main`, startup flags with `start` method +1. `Proxy` is similar to `main`, except `Proxy` does not block. +2. Internally `Proxy` is a context manager. +3. It will start `proxy.py` when called and will shut it down + once the scope ends. +4. Just like `main`, startup flags with `Proxy` can be customized by either passing flags as list of - input arguments e.g. `start(['--port', '8899'])` or - by using passing flags as kwargs e.g. `start(port=8899)`. + input arguments e.g. `Proxy(['--port', '8899'])` or + by using passing flags as kwargs e.g. `Proxy(port=8899)`. + +## Ephemeral Port + +Use `--port=0` to bind `proxy.py` on a random port allocated by the kernel. + +In embedded mode, you can access this port. Example: + +```python +import proxy + +if __name__ == '__main__': + with proxy.Proxy([]) as p: + print(p.acceptors.flags.port) +``` + +`acceptors.flags.port` will give you access to the random port allocated by the kernel. ## Loading Plugins -You can, of course, list plugins to load in the input arguments list of `proxy.main`, `proxy.start` or the `Proxy` constructor. Use the `--plugins` flag as when starting from command line: +Users can use `--plugins` flag multiple times to load multiple plugins. +See [Unable to load plugins](#unable-to-load-plugins) if you are running into issues. + +When using in embedded mode, you have a few more options. Example: + +1. Provide a fully-qualified name of the plugin class as `bytes` to the `proxy.main` method or `proxy.Proxy` context manager. +2. Provide `type` instance of the plugin class. This is especially useful if you plan to define plugins at runtime. + +Example, load a single plugin using `--plugins` flag: ```python import proxy @@ -1177,7 +1357,9 @@ if __name__ == '__main__': ]) ``` -However, for simplicity you can pass the list of plugins to load as a keyword argument to `proxy.main`, `proxy.start` or the `Proxy` constructor: +For simplicity, you can also pass the list of plugins as a keyword argument to `proxy.main` or the `Proxy` constructor. + +Example: ```python import proxy @@ -1190,23 +1372,17 @@ if __name__ == '__main__': ]) ``` -Note that it supports: - -1. The fully-qualified name of a class as `bytes` -2. Any `type` instance for a Proxy.py plugin class. This is espacially useful for custom plugins defined locally. - # Unit testing with proxy.py -## proxy.TestCase +## `proxy.TestCase` -To setup and teardown `proxy.py` for your Python unittest classes, +To setup and tear down `proxy.py` for your Python `unittest` classes, simply use `proxy.TestCase` instead of `unittest.TestCase`. Example: ```python import proxy - class TestProxyPyEmbedded(proxy.TestCase): def test_my_application_with_proxy(self) -> None: @@ -1215,10 +1391,10 @@ class TestProxyPyEmbedded(proxy.TestCase): Note that: -1. `proxy.TestCase` overrides `unittest.TestCase.run()` method to setup and teardown `proxy.py`. +1. `proxy.TestCase` overrides `unittest.TestCase.run()` method to setup and tear down `proxy.py`. 2. `proxy.py` server will listen on a random available port on the system. - This random port is available as `self.PROXY_PORT` within your test cases. -3. Only a single worker is started by default (`--num-workers 1`) for faster setup and teardown. + This random port is available as `self.PROXY.acceptors.flags.port` within your test cases. +3. Only a single acceptor and worker is started by default (`--num-workers 1 --num-acceptors 1`) for faster setup and tear down. 4. Most importantly, `proxy.TestCase` also ensures `proxy.py` server is up and running before proceeding with execution of tests. By default, `proxy.TestCase` will wait for `10 seconds` for `proxy.py` server to start, @@ -1233,7 +1409,8 @@ Example: class TestProxyPyEmbedded(TestCase): PROXY_PY_STARTUP_FLAGS = [ - '--num-workers', '1', + '--num-workers', '2', + '--num-acceptors', '1', '--enable-web-server', ] @@ -1241,13 +1418,15 @@ class TestProxyPyEmbedded(TestCase): self.assertTrue(True) ``` -See [test_embed.py](https://github.com/abhinavsingh/proxy.py/blob/develop/tests/test_embed.py) -for full working example. +See [test_embed.py] for full working example. + +[test_embed.py]: +https://github.com/abhinavsingh/proxy.py/blob/develop/tests/testing/test_embed.py -## With unittest.TestCase +## With `unittest.TestCase` If for some reasons you are unable to directly use `proxy.TestCase`, -then simply override `unittest.TestCase.run` yourself to setup and teardown `proxy.py`. +then simply override `unittest.TestCase.run` yourself to setup and tear down `proxy.py`. Example: ```python @@ -1263,84 +1442,14 @@ class TestProxyPyEmbedded(unittest.TestCase): def run(self, result: Optional[unittest.TestResult] = None) -> Any: with proxy.start([ '--num-workers', '1', + '--num-acceptors', '1', '--port', '... random port ...']): super().run(result) ``` -or simply setup / teardown `proxy.py` within +or simply setup / tear down `proxy.py` within `setUpClass` and `teardownClass` class methods. -# Plugin Developer and Contributor Guide - -## Everything is a plugin - -As you might have guessed by now, in `proxy.py` everything is a plugin. - -- We enabled proxy server plugins using `--plugins` flag. - All the [plugin examples](#plugin-examples) were implementing - `HttpProxyBasePlugin`. See documentation of - [HttpProxyBasePlugin](https://github.com/abhinavsingh/proxy.py/blob/b03629fa0df1595eb4995427bc601063be7fdca9/proxy.py#L894-L938) - for available lifecycle hooks. Use `HttpProxyBasePlugin` to modify - behavior of http(s) proxy protocol between client and upstream server. - Example, [FilterByUpstreamHostPlugin](#filterbyupstreamhostplugin). - -- We also enabled inbuilt web server using `--enable-web-server`. - Inbuilt web server implements `HttpProtocolHandlerPlugin` plugin. - See documentation of [HttpProtocolHandlerPlugin](https://github.com/abhinavsingh/proxy.py/blob/b03629fa0df1595eb4995427bc601063be7fdca9/proxy.py#L793-L850) - for available lifecycle hooks. Use `HttpProtocolHandlerPlugin` to add - new features for http(s) clients. Example, - [HttpWebServerPlugin](https://github.com/abhinavsingh/proxy.py/blob/b03629fa0df1595eb4995427bc601063be7fdca9/proxy.py#L1185-L1260). - -- There also is a `--disable-http-proxy` flag. It disables inbuilt proxy server. - Use this flag with `--enable-web-server` flag to run `proxy.py` as a programmable - http(s) server. [HttpProxyPlugin](https://github.com/abhinavsingh/proxy.py/blob/b03629fa0df1595eb4995427bc601063be7fdca9/proxy.py#L941-L1182) - also implements `HttpProtocolHandlerPlugin`. - -## Internal Architecture - -- [HttpProtocolHandler](https://github.com/abhinavsingh/proxy.py/blob/b03629fa0df1595eb4995427bc601063be7fdca9/proxy.py#L1263-L1440) - thread is started with the accepted [TcpClientConnection](https://github.com/abhinavsingh/proxy.py/blob/b03629fa0df1595eb4995427bc601063be7fdca9/proxy.py#L230-L237). - `HttpProtocolHandler` is responsible for parsing incoming client request and invoking - `HttpProtocolHandlerPlugin` lifecycle hooks. - -- `HttpProxyPlugin` which implements `HttpProtocolHandlerPlugin` also has its own plugin - mechanism. Its responsibility is to establish connection between client and - upstream [TcpServerConnection](https://github.com/abhinavsingh/proxy.py/blob/b03629fa0df1595eb4995427bc601063be7fdca9/proxy.py#L204-L227) - and invoke `HttpProxyBasePlugin` lifecycle hooks. - -- `HttpProtocolHandler` threads are started by [Acceptor](https://github.com/abhinavsingh/proxy.py/blob/b03629fa0df1595eb4995427bc601063be7fdca9/proxy.py#L424-L472) - processes. - -- `--num-workers` `Acceptor` processes are started by - [AcceptorPool](https://github.com/abhinavsingh/proxy.py/blob/b03629fa0df1595eb4995427bc601063be7fdca9/proxy.py#L368-L421) - on start-up. - -- `AcceptorPool` listens on server socket and pass the handler to `Acceptor` processes. - Workers are responsible for accepting new client connections and starting - `HttpProtocolHandler` thread. - -## Development Guide - -### Setup Local Environment - -Contributors must start `proxy.py` from source to verify and develop new features / fixes. - -See [Run proxy.py from command line using repo source](#from-command-line-using-repo-source) for details. - -### Setup pre-commit hook - -Pre-commit hook ensures lint checking and tests execution. - -1. `cd /path/to/proxy.py` -2. `ln -s $(PWD)/git-pre-commit .git/hooks/pre-commit` - -### Sending a Pull Request - -Every pull request is tested using GitHub actions. - -See [GitHub workflow](https://github.com/abhinavsingh/proxy.py/tree/develop/.github/workflows) -for list of tests. - # Utilities ## TCP Sockets @@ -1376,36 +1485,34 @@ As a decorator: >>> ... [ use connection ] ... ``` -## Http Client +## HTTP Client ### build_http_request -#### Generate HTTP GET request +- Generate HTTP GET request -```python ->>> build_http_request(b'GET', b'/') -b'GET / HTTP/1.1\r\n\r\n' ->>> -``` + ```python + >>> build_http_request(b'GET', b'/') + b'GET / HTTP/1.1\r\n\r\n' + ``` -#### Generate HTTP GET request with headers +- Generate HTTP GET request with headers -```python ->>> build_http_request(b'GET', b'/', - headers={b'Connection': b'close'}) -b'GET / HTTP/1.1\r\nConnection: close\r\n\r\n' ->>> -``` + ```python + >>> build_http_request(b'GET', b'/', + headers={b'Connection': b'close'}) + b'GET / HTTP/1.1\r\nConnection: close\r\n\r\n' + ``` -#### Generate HTTP POST request with headers and body +- Generate HTTP POST request with headers and body -```python ->>> import json ->>> build_http_request(b'POST', b'/form', - headers={b'Content-type': b'application/json'}, - body=proxy.bytes_(json.dumps({'email': 'hello@world.com'}))) - b'POST /form HTTP/1.1\r\nContent-type: application/json\r\n\r\n{"email": "hello@world.com"}' -``` + ```python + >>> import json + >>> build_http_request(b'POST', b'/form', + headers={b'Content-type': b'application/json'}, + body=proxy.bytes_(json.dumps({'email': 'hello@world.com'}))) + b'POST /form HTTP/1.1\r\nContent-type: application/json\r\n\r\n{"email": "hello@world.com"}' + ``` ### build_http_response @@ -1422,66 +1529,66 @@ build_http_response( ### API Usage -#### gen_private_key +- `gen_private_key` -```python -gen_private_key( - key_path: str, - password: str, - bits: int = 2048, - timeout: int = 10) -> bool -``` - -#### gen_public_key + ```python + gen_private_key( + key_path: str, + password: str, + bits: int = 2048, + timeout: int = 10) -> bool + ``` -```python -gen_public_key( - public_key_path: str, - private_key_path: str, - private_key_password: str, - subject: str, - alt_subj_names: Optional[List[str]] = None, - extended_key_usage: Optional[str] = None, - validity_in_days: int = 365, - timeout: int = 10) -> bool -``` +- `gen_public_key` + + ```python + gen_public_key( + public_key_path: str, + private_key_path: str, + private_key_password: str, + subject: str, + alt_subj_names: Optional[List[str]] = None, + extended_key_usage: Optional[str] = None, + validity_in_days: int = 365, + timeout: int = 10) -> bool + ``` -#### remove_passphrase +- `remove_passphrase` -```python -remove_passphrase( - key_in_path: str, - password: str, - key_out_path: str, - timeout: int = 10) -> bool -``` + ```python + remove_passphrase( + key_in_path: str, + password: str, + key_out_path: str, + timeout: int = 10) -> bool + ``` -#### gen_csr +- `gen_csr` -```python -gen_csr( - csr_path: str, - key_path: str, - password: str, - crt_path: str, - timeout: int = 10) -> bool -``` - -#### sign_csr + ```python + gen_csr( + csr_path: str, + key_path: str, + password: str, + crt_path: str, + timeout: int = 10) -> bool + ``` -```python -sign_csr( - csr_path: str, - crt_path: str, - ca_key_path: str, - ca_key_password: str, - ca_crt_path: str, - serial: str, - alt_subj_names: Optional[List[str]] = None, - extended_key_usage: Optional[str] = None, - validity_in_days: int = 365, - timeout: int = 10) -> bool -``` +- `sign_csr` + + ```python + sign_csr( + csr_path: str, + crt_path: str, + ca_key_path: str, + ca_key_password: str, + ca_crt_path: str, + serial: str, + alt_subj_names: Optional[List[str]] = None, + extended_key_usage: Optional[str] = None, + validity_in_days: int = 365, + timeout: int = 10) -> bool + ``` See [pki.py](https://github.com/abhinavsingh/proxy.py/blob/develop/proxy/common/pki.py) and [test_pki.py](https://github.com/abhinavsingh/proxy.py/blob/develop/tests/common/test_pki.py) @@ -1495,7 +1602,7 @@ Use `proxy.common.pki` module for: 2. Generating CSR requests 3. Signing CSR requests using custom CA. -```bash +```console python -m proxy.common.pki -h usage: pki.py [-h] [--password PASSWORD] [--private-key-path PRIVATE_KEY_PATH] [--public-key-path PUBLIC_KEY_PATH] [--subject SUBJECT] @@ -1520,10 +1627,9 @@ optional arguments: ## Internal Documentation -Browse through internal class hierarchy and documentation using `pydoc3`. -Example: +Code is well documented. Browse through internal class hierarchy and documentation using `pydoc3` -```bash +```console โฏ pydoc3 proxy PACKAGE CONTENTS @@ -1537,20 +1643,89 @@ FILE /Users/abhinav/Dev/proxy.py/proxy/__init__.py ``` +# Run Dashboard + +Dashboard is currently under development and not yet bundled with `pip` packages. +To run dashboard, you must checkout the source. + +Dashboard is written in Typescript and SCSS, so let's build it first using: + +```console +โฏ make dashboard +``` + +Also build the embedded `Chrome DevTools` if you plan on using it: + +```console +โฏ make devtools +``` + +Now start `proxy.py` with dashboard plugin and by overriding root directory for static server: + +```console +โฏ proxy --enable-dashboard --static-server-dir dashboard/public +...[redacted]... - Loaded plugin proxy.http.server.HttpWebServerPlugin +...[redacted]... - Loaded plugin proxy.dashboard.dashboard.ProxyDashboard +...[redacted]... - Loaded plugin proxy.dashboard.inspect_traffic.InspectTrafficPlugin +...[redacted]... - Loaded plugin proxy.http.inspector.DevtoolsProtocolPlugin +...[redacted]... - Loaded plugin proxy.http.proxy.HttpProxyPlugin +...[redacted]... - Listening on ::1:8899 +...[redacted]... - Core Event enabled +``` + +Currently, enabling dashboard will also enable all the dashboard plugins. + +Visit dashboard: + +```console +โฏ open http://localhost:8899/dashboard/ +``` + +## Inspect Traffic + +***This is a WIP and may not work as documented*** + +Wait for embedded `Chrome Dev Console` to load. Currently, detail about all traffic flowing +through `proxy.py` is pushed to the `Inspect Traffic` tab. However, received payloads are not +yet integrated with the embedded developer console. + +Current functionality can be verified by opening the `Dev Console` of dashboard and inspecting +the websocket connection that dashboard established with the `proxy.py` server. + +[![Proxy.Py Dashboard Inspect Traffic](https://github.com/raw/abhinavsingh/proxy.py/develop/Dashboard.png)](https://github.com/abhinavsingh/proxy.py) + +# Chrome DevTools Protocol + +For scenarios where you want direct access to `Chrome DevTools` protocol websocket endpoint, +start `proxy.py` as: + +```console +โฏ proxy --enable-devtools --enable-events +``` + +Now point your CDT instance to `ws://localhost:8899/devtools`. + # Frequently Asked Questions ## Threads vs Threadless -Pre v2.x, `proxy.py` used to spawn new threads for handling -client requests. +### `v1.x` + +`proxy.py` used to spawn new threads for handling client requests. + +### `v2.0+` + +`proxy.py` added support for threadless execution of client requests using `asyncio`. + +### `v2.4.0+` + +Threadless execution was turned ON by default for `Python 3.8+` on `mac` and `linux` environments. -Starting v2.x, `proxy.py` added support for threadless execution of -client requests using `asyncio`. +`proxy.py` threadless execution has been reported safe on these environments by our users. If you are running into trouble, fallback to threaded mode using `--threaded` flag. -In future, threadless execution will be the default mode. +For `windows` and `Python < 3.8`, you can still try out threadless mode by starting `proxy.py` with `--threadless` flag. -Till then if you are interested in trying it out, -start `proxy.py` with `--threadless` flag. +If threadless works for you, consider sending a PR by editing `_env_threadless_compliant` method in the `proxy/common/constants.py` file. ## SyntaxError: invalid syntax @@ -1585,7 +1760,7 @@ Make sure plugin modules are discoverable by adding them to `PYTHONPATH`. Exampl `PYTHONPATH=/path/to/my/app proxy --plugins my_app.proxyPlugin` -```bash +```console ...[redacted]... - Loaded plugin proxy.HttpProxyPlugin ...[redacted]... - Loaded plugin my_app.proxyPlugin ``` @@ -1594,6 +1769,37 @@ OR, simply pass fully-qualified path as parameter, e.g. `proxy --plugins /path/to/my/app/my_app.proxyPlugin` +Here is a quick working example: + +- Contents of `/tmp/plug` folder + +```console +โ•ฐโ”€ ls -1 /tmp/plug โ”€โ•ฏ +my_plugin.py +``` + +- Custom `MyPlugin` class + +```console +โ•ฐโ”€ cat /tmp/plug/my_plugin.py โ”€โ•ฏ +from proxy.http.proxy import HttpProxyBasePlugin + + +class MyPlugin(HttpProxyBasePlugin): + pass +``` + +This is an empty plugin for demonstrating external plugin usage. You must implement necessary methods to make your plugins work for real traffic + +- Start `proxy.py` with `MyPlugin` + +```console +โ•ฐโ”€ PYTHONPATH=/tmp/plug proxy --plugin my_plugin.MyPlugin โ”€โ•ฏ +...[redacted]... - Loaded plugin proxy.http.proxy.HttpProxyPlugin +...[redacted]... - Loaded plugin my_plugin.MyPlugin +...[redacted]... - Listening on ::1:8899 +``` + ## Unable to connect with proxy.py from remote host Make sure `proxy.py` is listening on correct network interface. @@ -1623,7 +1829,7 @@ for some background. ## GCE log viewer integration for proxy.py -A starter [fluentd.conf](https://github.com/abhinavsingh/proxy.py/blob/develop/fluentd.conf) +A starter [fluentd.conf](https://github.com/abhinavsingh/proxy.py/blob/develop/helper/fluentd.conf) template is available. 1. Copy this configuration file as `proxy.py.conf` under @@ -1639,7 +1845,7 @@ template is available. Now `proxy.py` logs can be browsed using [GCE log viewer](https://console.cloud.google.com/logs/viewer). -## ValueError: filedescriptor out of range in select +## `ValueError: filedescriptor out of range in select` `proxy.py` is made to handle thousands of connections per second without any socket leaks. @@ -1650,7 +1856,7 @@ without any socket leaks. If nothing helps, [open an issue](https://github.com/abhinavsingh/proxy.py/issues/new) with `requests per second` sent and output of following debug script: -```bash +```console โฏ ./helper/monitor_open_files.sh ``` @@ -1666,69 +1872,234 @@ few obvious ones include: 1. Client established a connection but never completed the request. 2. A plugin returned a response prematurely, avoiding connection to upstream server. +## OSError when wrapping client for TLS Interception + +With `TLS Interception` on, you might occasionally see following exceptions: + +```console +2021-11-06 23:33:34,540 - pid:91032 [E] server.intercept:678 - OSError when wrapping client +Traceback (most recent call last): + ...[redacted]... + ...[redacted]... + ...[redacted]... +ssl.SSLError: [SSL: TLSV1_ALERT_UNKNOWN_CA] tlsv1 alert unknown ca (_ssl.c:997) +...[redacted]... - CONNECT oauth2.googleapis.com:443 - 0 bytes - 272.08 ms +``` + +Some clients can throw `TLSV1_ALERT_UNKNOWN_CA` if they cannot verify the certificate of the server +because it is signed by an unknown issuer CA. Which is the case when we are doing TLS interception. +This can be for a variety of reasons e.g. certificate pinning etc. + +Another exception you might see is `CERTIFICATE_VERIFY_FAILED`: + +```console +2021-11-06 23:36:02,002 - pid:91033 [E] handler.handle_readables:293 - Exception while receiving from client connection with reason SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self signed certificate in certificate chain (_ssl.c:997)') +Traceback (most recent call last): + ...[redacted]... + ...[redacted]... + ...[redacted]... +ssl.SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self signed certificate in certificate chain (_ssl.c:997) +...[redacted]... - CONNECT init.push.apple.com:443 - 0 bytes - 892.99 ms +``` + +In future, we might support serving original HTTPS content for such clients while still +performing TLS interception in the background. This will keep the clients happy without +impacting our ability to TLS intercept. Unfortunately, this feature is currently not available. + +Another example with `SSLEOFError` exception: + +```console +2021-11-06 23:46:40,446 - pid:91034 [E] server.intercept:678 - OSError when wrapping client +Traceback (most recent call last): + ...[redacted]... + ...[redacted]... + ...[redacted]... +ssl.SSLEOFError: EOF occurred in violation of protocol (_ssl.c:997) +...[redacted]... - CONNECT stock.adobe.io:443 - 0 bytes - 685.32 ms +``` + +# Plugin Developer and Contributor Guide + +## High level architecture + +```console + +-------------+ + | | + | Proxy([]) | + | | + +------+------+ + | + | + +-----------v--------------+ + | | + | AcceptorPool(...) | + | | + +------------+-------------+ + | ++-----------------+ | +-----------------+ +| | | | | +| Acceptor(..) <-------------+-----------> Acceptor(..) | +| | | | ++---+-------------+ +---------+-------+ + | | + | | + | +------++------++------++------++------+ | + | | || || || || | | + +----> || || || || <-----+ + | || || || || | + +------++------++------++------++------+ + Threadless Worker Processes +``` + +`proxy.py` is made with performance in mind. By default, `proxy.py` +will try to utilize all available CPU cores to it for accepting new +client connections. This is achieved by starting `AcceptorPool` which +listens on configured server port. Then, `AcceptorPool` starts `Acceptor` +processes (`--num-acceptors`) to accept incoming client connections. +Alongside, if `--threadless` is enabled, `ThreadlessPool` is setup +which starts `Threadless` processes (`--num-workers`) to handle +the incoming client connections. + +Each `Acceptor` process delegates the accepted client connection +to a threadless process via `Work` class. Currently, `HttpProtocolHandler` +is the default work class. + +`HttpProtocolHandler` simply assumes that incoming clients will follow +HTTP specification. Specific HTTP proxy and HTTP server implementations +are written as plugins of `HttpProtocolHandler`. + +See documentation of `HttpProtocolHandlerPlugin` for available lifecycle hooks. +Use `HttpProtocolHandlerPlugin` to add new features for http(s) clients. Example, +See `HttpWebServerPlugin`. + +## Everything is a plugin + +Within `proxy.py` everything is a plugin. + +- We enabled `proxy server` plugins using `--plugins` flag. + Proxy server `HttpProxyPlugin` is a plugin of `HttpProtocolHandler`. + Further, Proxy server allows plugin through `HttpProxyBasePlugin` specification. + +- All the proxy server [plugin examples](#plugin-examples) were implementing + `HttpProxyBasePlugin`. See documentation of `HttpProxyBasePlugin` for available + lifecycle hooks. Use `HttpProxyBasePlugin` to modify behavior of http(s) proxy protocol + between client and upstream server. Example, + [FilterByUpstreamHostPlugin](#filterbyupstreamhostplugin). + +- We also enabled inbuilt `web server` using `--enable-web-server`. + Web server `HttpWebServerPlugin` is a plugin of `HttpProtocolHandler` + and implements `HttpProtocolHandlerPlugin` specification. + +- There also is a `--disable-http-proxy` flag. It disables inbuilt proxy server. + Use this flag with `--enable-web-server` flag to run `proxy.py` as a programmable + http(s) server. + +## Development Guide + +### Setup Local Environment + +Contributors must start `proxy.py` from source to verify and develop new features / fixes. + +See [Run proxy.py from command line using repo source](#from-command-line-using-repo-source) for details. + + +[![WARNING](https://img.shields.io/static/v1?label=MacOS&message=warning&color=red)](https://github.com/abhinavsingh/proxy.py/issues/642#issuecomment-960819271) On `macOS` +you must install `Python` using `pyenv`, as `Python` installed via `homebrew` tends +to be problematic. See linked thread for more details. + +### Setup Git Hooks + +Pre-commit hook ensures tests are passing. + +1. `cd /path/to/proxy.py` +2. `ln -s $(PWD)/git-pre-commit .git/hooks/pre-commit` + +Pre-push hook ensures lint and tests are passing. + +1. `cd /path/to/proxy.py` +2. `ln -s $(PWD)/git-pre-push .git/hooks/pre-push` + +### Sending a Pull Request + +Every pull request is tested using GitHub actions. + +See [GitHub workflow](https://github.com/abhinavsingh/proxy.py/tree/develop/.github/workflows) +for list of tests. + +# Benchmarks + +Simply run the following command from repo root to start benchmark + +```console +โฏ ./helper/benchmark.sh +``` + # Flags -```bash +```console โฏ proxy -h -usage: proxy [-h] [--threadless] [--backlog BACKLOG] [--enable-events] - [--hostname HOSTNAME] [--port PORT] [--num-workers NUM_WORKERS] +usage: -m [-h] [--enable-events] [--enable-conn-pool] [--threadless] + [--threaded] [--num-workers NUM_WORKERS] [--local-executor] + [--backlog BACKLOG] [--hostname HOSTNAME] [--port PORT] + [--unix-socket-path UNIX_SOCKET_PATH] + [--num-acceptors NUM_ACCEPTORS] [--version] [--log-level LOG_LEVEL] + [--log-file LOG_FILE] [--log-format LOG_FORMAT] + [--open-file-limit OPEN_FILE_LIMIT] + [--plugins PLUGINS [PLUGINS ...]] [--enable-dashboard] + [--work-klass WORK_KLASS] [--pid-file PID_FILE] + [--enable-proxy-protocol] [--client-recvbuf-size CLIENT_RECVBUF_SIZE] [--key-file KEY_FILE] - [--timeout TIMEOUT] [--pid-file PID_FILE] [--version] - [--disable-http-proxy] [--enable-dashboard] [--enable-devtools] - [--enable-static-server] [--enable-web-server] - [--log-level LOG_LEVEL] [--log-file LOG_FILE] - [--log-format LOG_FORMAT] [--open-file-limit OPEN_FILE_LIMIT] - [--plugins PLUGINS] [--ca-key-file CA_KEY_FILE] - [--ca-cert-dir CA_CERT_DIR] [--ca-cert-file CA_CERT_FILE] - [--ca-file CA_FILE] [--ca-signing-key-file CA_SIGNING_KEY_FILE] - [--cert-file CERT_FILE] [--disable-headers DISABLE_HEADERS] - [--server-recvbuf-size SERVER_RECVBUF_SIZE] - [--basic-auth BASIC_AUTH] [--cache-dir CACHE_DIR] - [--static-server-dir STATIC_SERVER_DIR] [--pac-file PAC_FILE] - [--pac-file-url-path PAC_FILE_URL_PATH] + [--timeout TIMEOUT] [--server-recvbuf-size SERVER_RECVBUF_SIZE] + [--disable-http-proxy] [--disable-headers DISABLE_HEADERS] + [--ca-key-file CA_KEY_FILE] [--ca-cert-dir CA_CERT_DIR] + [--ca-cert-file CA_CERT_FILE] [--ca-file CA_FILE] + [--ca-signing-key-file CA_SIGNING_KEY_FILE] [--cert-file CERT_FILE] + [--auth-plugin AUTH_PLUGIN] [--basic-auth BASIC_AUTH] + [--cache-dir CACHE_DIR] + [--filtered-upstream-hosts FILTERED_UPSTREAM_HOSTS] + [--enable-web-server] [--enable-static-server] + [--static-server-dir STATIC_SERVER_DIR] + [--min-compression-length MIN_COMPRESSION_LENGTH] + [--pac-file PAC_FILE] [--pac-file-url-path PAC_FILE_URL_PATH] + [--proxy-pool PROXY_POOL] [--filtered-client-ips FILTERED_CLIENT_IPS] + [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG] + [--cloudflare-dns-mode CLOUDFLARE_DNS_MODE] -proxy.py v2.3.1 +proxy.py v2.3.2.dev190+ge60d80d.d20211124 -optional arguments: +options: -h, --help show this help message and exit - --threadless Default: False. When disabled a new thread is spawned - to handle each client connection. - --backlog BACKLOG Default: 100. Maximum number of pending connections to - proxy server --enable-events Default: False. Enables core to dispatch lifecycle events. Plugins can be used to subscribe for core events. + --enable-conn-pool Default: False. (WIP) Enable upstream connection + pooling. + --threadless Default: True. Enabled by default on Python 3.8+ (mac, + linux). When disabled a new thread is spawned to + handle each client connection. + --threaded Default: False. Disabled by default on Python < 3.8 + and windows. When enabled a new thread is spawned to + handle each client connection. + --num-workers NUM_WORKERS + Defaults to number of CPU cores. + --local-executor Default: False. Disabled by default. When enabled + acceptors will make use of local (same process) + executor instead of distributing load across remote + (other process) executors. Enable this option to + achieve CPU affinity between acceptors and executors, + instead of using underlying OS kernel scheduling + algorithm. + --backlog BACKLOG Default: 100. Maximum number of pending connections to + proxy server --hostname HOSTNAME Default: ::1. Server IP address. --port PORT Default: 8899. Server port. - --num-workers NUM_WORKERS + --unix-socket-path UNIX_SOCKET_PATH + Default: None. Unix socket path to use. When provided + --host and --port flags are ignored + --num-acceptors NUM_ACCEPTORS Defaults to number of CPU cores. - --client-recvbuf-size CLIENT_RECVBUF_SIZE - Default: 1 MB. Maximum amount of data received from - the client in a single recv() operation. Bump this - value for faster uploads at the expense of increased - RAM. - --key-file KEY_FILE Default: None. Server key file to enable end-to-end - TLS encryption with clients. If used, must also pass - --cert-file. - --timeout TIMEOUT Default: 10. Number of seconds after which an inactive - connection must be dropped. Inactivity is defined by - no data sent or received by the client. - --pid-file PID_FILE Default: None. Save parent process ID to a file. --version, -v Prints proxy.py version. - --disable-http-proxy Default: False. Whether to disable - proxy.HttpProxyPlugin. - --enable-dashboard Default: False. Enables proxy.py dashboard. - --enable-devtools Default: False. Enables integration with Chrome - Devtool Frontend. Also see --devtools-ws-path. - --enable-static-server - Default: False. Enable inbuilt static file server. - Optionally, also use --static-server-dir to serve - static content from custom directory. By default, - static file server serves out of installed proxy.py - python module folder. - --enable-web-server Default: False. Whether to enable - proxy.HttpWebServerPlugin. --log-level LOG_LEVEL Valid options: DEBUG, INFO (default), WARNING, ERROR, CRITICAL. Both upper and lowercase values are allowed. @@ -1740,7 +2111,39 @@ optional arguments: --open-file-limit OPEN_FILE_LIMIT Default: 1024. Maximum number of files (TCP connections) that proxy.py can open concurrently. - --plugins PLUGINS Comma separated plugins + --plugins PLUGINS [PLUGINS ...] + Comma separated plugins. You may use --plugins flag + multiple times. + --enable-dashboard Default: False. Enables proxy.py dashboard. + --work-klass WORK_KLASS + Default: proxy.http.HttpProtocolHandler. Work klass to + use for work execution. + --pid-file PID_FILE Default: None. Save "parent" process ID to a file. + --enable-proxy-protocol + Default: False. If used, will enable proxy protocol. + Only version 1 is currently supported. + --client-recvbuf-size CLIENT_RECVBUF_SIZE + Default: 1 MB. Maximum amount of data received from + the client in a single recv() operation. Bump this + value for faster uploads at the expense of increased + RAM. + --key-file KEY_FILE Default: None. Server key file to enable end-to-end + TLS encryption with clients. If used, must also pass + --cert-file. + --timeout TIMEOUT Default: 10.0. Number of seconds after which an + inactive connection must be dropped. Inactivity is + defined by no data sent or received by the client. + --server-recvbuf-size SERVER_RECVBUF_SIZE + Default: 1 MB. Maximum amount of data received from + the server in a single recv() operation. Bump this + value for faster downloads at the expense of increased + RAM. + --disable-http-proxy Default: False. Whether to disable + proxy.HttpProxyPlugin. + --disable-headers DISABLE_HEADERS + Default: None. Comma separated list of headers to + remove before dispatching client request to upstream + server. --ca-key-file CA_KEY_FILE Default: None. CA key to use for signing dynamically generated HTTPS certificates. If used, must also pass @@ -1753,8 +2156,10 @@ optional arguments: Default: None. Signing certificate to use for signing dynamically generated HTTPS certificates. If used, must also pass --ca-key-file and --ca-signing-key-file - --ca-file CA_FILE Default: None. Provide path to custom CA file for peer - certificate validation. Specially useful on MacOS. + --ca-file CA_FILE Default: /Users/abhinavsingh/Dev/proxy.py/venv310/lib/ + python3.10/site-packages/certifi/cacert.pem. Provide + path to custom CA bundle for peer certificate + verification --ca-signing-key-file CA_SIGNING_KEY_FILE Default: None. CA signing key to use for dynamic generation of HTTPS certificates. If used, must also @@ -1763,33 +2168,50 @@ optional arguments: Default: None. Server certificate to enable end-to-end TLS encryption with clients. If used, must also pass --key-file. - --disable-headers DISABLE_HEADERS - Default: None. Comma separated list of headers to - remove before dispatching client request to upstream - server. - --server-recvbuf-size SERVER_RECVBUF_SIZE - Default: 1 MB. Maximum amount of data received from - the server in a single recv() operation. Bump this - value for faster downloads at the expense of increased - RAM. + --auth-plugin AUTH_PLUGIN + Default: proxy.http.proxy.AuthPlugin. Auth plugin to + use instead of default basic auth plugin. --basic-auth BASIC_AUTH Default: No authentication. Specify colon separated user:password to enable basic authentication. --cache-dir CACHE_DIR Default: A temporary directory. Flag only applicable when cache plugin is used with on-disk storage. + --filtered-upstream-hosts FILTERED_UPSTREAM_HOSTS + Default: Blocks Facebook. Comma separated list of IPv4 + and IPv6 addresses. + --enable-web-server Default: False. Whether to enable + proxy.HttpWebServerPlugin. + --enable-static-server + Default: False. Enable inbuilt static file server. + Optionally, also use --static-server-dir to serve + static content from custom directory. By default, + static file server serves out of installed proxy.py + python module folder. --static-server-dir STATIC_SERVER_DIR Default: "public" folder in directory where proxy.py is placed. This option is only applicable when static server is also enabled. See --enable-static-server. + --min-compression-length MIN_COMPRESSION_LENGTH + Default: 20 bytes. Sets the minimum length of a + response that will be compressed (gzipped). --pac-file PAC_FILE A file (Proxy Auto Configuration) or string to serve when the server receives a direct file request. Using this option enables proxy.HttpWebServerPlugin. --pac-file-url-path PAC_FILE_URL_PATH Default: /. Web server path to serve the PAC file. + --proxy-pool PROXY_POOL + List of upstream proxies to use in the pool --filtered-client-ips FILTERED_CLIENT_IPS Default: 127.0.0.1,::1. Comma separated list of IPv4 and IPv6 addresses. + --filtered-url-regex-config FILTERED_URL_REGEX_CONFIG + Default: No config. Comma separated list of IPv4 and + IPv6 addresses. + --cloudflare-dns-mode CLOUDFLARE_DNS_MODE + Default: security. Either "security" (for malware + protection) or "family" (for malware and adult content + protection) Proxy.py not working? Report at: https://github.com/abhinavsingh/proxy.py/issues/new @@ -1797,6 +2219,10 @@ https://github.com/abhinavsingh/proxy.py/issues/new # Changelog +## v2.4.0 + +- No longer support `Python 3.6` due to `asyncio.run` usage in the core. + ## v2.x - No longer ~~a single file module~~. diff --git a/SECURITY.md b/SECURITY.md index 606fe61b6a..a9fbc5725e 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -12,5 +12,5 @@ Follow these steps: 1. Start by [emailing developers](mailto:mailsforabhinav+proxy@gmail.com) -2. If unresponsive, [create a public issue](https://github.com/abhinavsingh/proxy.py/issues/new/choose) +2. If unresponsive, [create a public issue](https://github.com/abhinavsingh/proxy.py/issues/new/choose) without disclosure about the vulnerability itself. 3. [Pull requests](https://github.com/abhinavsingh/proxy.py/pulls) are always welcome diff --git a/check.py b/check.py new file mode 100644 index 0000000000..038758a30f --- /dev/null +++ b/check.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import sys +import subprocess + +from pathlib import Path +from proxy.common.version import __version__ as lib_version + +# This script ensures our versions never run out of sync. +# +# 1. TODO: Version is hardcoded in homebrew stable package +# installer file, but it only needs to match with lib +# versions if current git branch is master + +PY_FILE_PREFIX = b'# -*- coding: utf-8 -*-\n' + \ + b'"""\n' + \ + b' proxy.py\n' + \ + b' ~~~~~~~~\n' + \ + b' \xe2\x9a\xa1\xe2\x9a\xa1\xe2\x9a\xa1 Fast, Lightweight, Pluggable, TLS interception capable' + \ + b' proxy server focused on\n' + \ + b' Network monitoring, controls & Application development, testing, debugging.\n' + \ + b'\n' + \ + b' :copyright: (c) 2013-present by Abhinav Singh and contributors.\n' + \ + b' :license: BSD, see LICENSE for more details.\n' + +REPO_ROOT = Path(__file__).parent +ALL_PY_FILES = ( + list(REPO_ROOT.glob('*.py')) + + list((REPO_ROOT / 'proxy').rglob('*.py')) + + list((REPO_ROOT / 'examples').rglob('*.py')) + + list((REPO_ROOT / 'tests').rglob('*.py')) +) + +# Ensure all python files start with licensing information +for py_file in ALL_PY_FILES: + if py_file.is_file() and py_file.name != '_scm_version.py': + with open(py_file, 'rb') as f: + code = f.read(len(PY_FILE_PREFIX)) + if code != PY_FILE_PREFIX: + print( + 'Expected license not found in {0}'.format( + str(py_file), + ), + ) + sys.exit(1) + +# Update README.md flags section to match current library --help output +lib_help = subprocess.check_output( + ['python', '-m', 'proxy', '-h'], +) +with open('README.md', 'rb+') as f: + c = f.read() + pre_flags, post_flags = c.split(b'# Flags') + help_text, post_changelog = post_flags.split(b'# Changelog') + f.seek(0) + f.write( + pre_flags + b'# Flags\n\n```console\n\xe2\x9d\xaf proxy -h\n' + lib_help + b'```' + + b'\n\n# Changelog' + post_changelog, + ) + +# Version is also hardcoded in README.md flags section +readme_version_cmd = 'cat README.md | grep "proxy.py v" | tail -2 | head -1 | cut -d " " -f 2 | cut -c2-' +readme_version_output = subprocess.check_output( + ['bash', '-c', readme_version_cmd], +) +# Doesn't contain "v" prefix +readme_version = readme_version_output.decode().strip() + +if readme_version != lib_version: + print( + 'Version mismatch found. {0} (readme) vs {1} (lib).'.format( + readme_version, lib_version, + ), + ) + sys.exit(1) diff --git a/codecov.yml b/codecov.yml index f393de9c7d..d120bd4956 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,12 +1,26 @@ +--- codecov: - require_ci_to_pass: yes + require_ci_to_pass: yes # yamllint disable-line rule:truthy notify: - wait_for_ci: yes + wait_for_ci: yes # yamllint disable-line rule:truthy coverage: status: project: - default: + default: false # disable the default status that measures entire project + # examples: # declare a new status context "examples" + tests: # declare a new status context "tests" + # target: 100% # we always want 100% coverage here + paths: + - "tests/" # only include coverage in "tests/" folder + lib: # declare a new status context "lib" + paths: + - "!tests/" # remove all files in "tests/" threshold: 1% patch: default: + target: auto + base: auto threshold: 1% +comment: + require_changes: true +... diff --git a/dashboard/package-lock.json b/dashboard/package-lock.json index 84c042cb3c..a4a79a68e4 100644 --- a/dashboard/package-lock.json +++ b/dashboard/package-lock.json @@ -1,8 +1,5359 @@ { "name": "proxy.py", "version": "1.0.0", - "lockfileVersion": 1, + "lockfileVersion": 2, "requires": true, + "packages": { + "": { + "name": "proxy.py", + "version": "1.0.0", + "license": "BSD-3-Clause", + "devDependencies": { + "@types/jasmine": "^3.6.1", + "@types/jquery": "^3.5.4", + "@types/js-cookie": "^2.2.6", + "@typescript-eslint/eslint-plugin": "^2.34.0", + "@typescript-eslint/parser": "^2.34.0", + "chrome-devtools-frontend": "^1.0.944903", + "eslint": "^6.8.0", + "eslint-config-standard": "^14.1.1", + "eslint-plugin-import": "^2.25.3", + "eslint-plugin-node": "^10.0.0", + "eslint-plugin-promise": "^4.2.1", + "eslint-plugin-standard": "^4.1.0", + "http-server": "^0.12.3", + "jasmine": "^3.6.3", + "jasmine-ts": "^0.3.0", + "jquery": "^3.5.1", + "js-cookie": "^2.2.1", + "jsdom": "^15.2.1", + "ncp": "^2.0.0", + "rollup": "^1.32.1", + "rollup-plugin-copy": "^3.3.0", + "rollup-plugin-javascript-obfuscator": "^1.0.4", + "rollup-plugin-typescript": "^1.0.1", + "ts-node": "^7.0.1", + "typescript": "^3.9.7", + "ws": "^7.4.6" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.10.4.tgz", + "integrity": "sha512-vG6SvB6oYEhvgisZNFRmRCUkLz11c7rp+tbNTynGqc6mS1d5ATd/sGyV6W0KZZnXRKMTzZDRgQT3Ou9jhpAfUg==", + "dev": true, + "dependencies": { + "@babel/highlight": "^7.10.4" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.10.4.tgz", + "integrity": "sha512-3U9y+43hz7ZM+rzG24Qe2mufW5KhvFg/NhnNph+i9mgCtdTCtMJuI1TMkrIUiK7Ix4PYlRF9I5dhqaLYA/ADXw==", + "dev": true + }, + "node_modules/@babel/highlight": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.10.4.tgz", + "integrity": "sha512-i6rgnR/YgPEQzZZnbTHHuZdlE8qyoBNalD6F+q4vAFlcMEcqmkoG+mPqJYJCo63qPf74+Y1UZsl3l6f7/RIkmA==", + "dev": true, + "dependencies": { + "@babel/helper-validator-identifier": "^7.10.4", + "chalk": "^2.0.0", + "js-tokens": "^4.0.0" + } + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.3.tgz", + "integrity": "sha512-eGmwYQn3gxo4r7jdQnkrrN6bY478C3P+a/y72IJukF8LjB6ZHeB3c+Ehacj3sYeSmUXGlnA67/PmbM9CVwL7Dw==", + "dev": true, + "dependencies": { + "@nodelib/fs.stat": "2.0.3", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.scandir/node_modules/@nodelib/fs.stat": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.3.tgz", + "integrity": "sha512-bQBFruR2TAwoevBEd/NWMoAAtNGzTRgdrqnYCc7dhzfoNvqPzLyqlEQnzZ3kVnNrSp25iyxE00/3h2fqGAGArA==", + "dev": true, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.4.tgz", + "integrity": "sha512-1V9XOY4rDW0rehzbrcqAmHnz8e7SKvX27gh8Gt2WgB0+pdzdiLV83p72kZPU+jvMbS1qU5mauP2iOvO8rhmurQ==", + "dev": true, + "dependencies": { + "@nodelib/fs.scandir": "2.1.3", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@types/color-name": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz", + "integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ==", + "dev": true + }, + "node_modules/@types/eslint-visitor-keys": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@types/eslint-visitor-keys/-/eslint-visitor-keys-1.0.0.tgz", + "integrity": "sha512-OCutwjDZ4aFS6PB1UZ988C4YgwlBHJd6wCeQqaLdmadZ/7e+w79+hbMUFC1QXDNCmdyoRfAFdm0RypzwR+Qpag==", + "dev": true + }, + "node_modules/@types/estree": { + "version": "0.0.45", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.45.tgz", + "integrity": "sha512-jnqIUKDUqJbDIUxm0Uj7bnlMnRm1T/eZ9N+AVMqhPgzrba2GhGG5o/jCTwmdPK709nEZsGoMzXEDUjcXHa3W0g==", + "dev": true + }, + "node_modules/@types/fs-extra": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-8.1.1.tgz", + "integrity": "sha512-TcUlBem321DFQzBNuz8p0CLLKp0VvF/XH9E4KHNmgwyp4E3AfgI5cjiIVZWlbfThBop2qxFIh4+LeY6hVWWZ2w==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/glob": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.1.3.tgz", + "integrity": "sha512-SEYeGAIQIQX8NN6LDKprLjbrd5dARM5EXsd8GI/A5l0apYI1fGMWgPHSe4ZKL4eozlAyI+doUE9XbYS4xCkQ1w==", + "dev": true, + "dependencies": { + "@types/minimatch": "*", + "@types/node": "*" + } + }, + "node_modules/@types/jasmine": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/@types/jasmine/-/jasmine-3.6.1.tgz", + "integrity": "sha512-eeSCVhBsgwHNS1FmaMu4zrLxfykCTWJMLFZv7lmyrZQjw7foUUXoPu4GukSN9v7JvUw7X+/aDH3kCaymirBSTg==", + "dev": true + }, + "node_modules/@types/jquery": { + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/@types/jquery/-/jquery-3.5.4.tgz", + "integrity": "sha512-//9CHhaUt/rurMJTxGI+I6DmsNHgYU6d8aSLFfO5dB7+10lwLnaWT0z5GY/yY82Q/M+B+0Qh3TixlJ8vmBeqIw==", + "dev": true, + "dependencies": { + "@types/sizzle": "*" + } + }, + "node_modules/@types/js-cookie": { + "version": "2.2.6", + "resolved": "https://registry.npmjs.org/@types/js-cookie/-/js-cookie-2.2.6.tgz", + "integrity": "sha512-+oY0FDTO2GYKEV0YPvSshGq9t7YozVkgvXLty7zogQNuCxBhT9/3INX9Q7H1aRZ4SUDRXAKlJuA4EA5nTt7SNw==", + "dev": true + }, + "node_modules/@types/json-schema": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.5.tgz", + "integrity": "sha512-7+2BITlgjgDhH0vvwZU/HZJVyk+2XUlvxXe8dFMedNX/aMkaOq++rMAFXc0tM7ij15QaWlbdQASBR9dihi+bDQ==", + "dev": true + }, + "node_modules/@types/json5": { + "version": "0.0.29", + "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz", + "integrity": "sha1-7ihweulOEdK4J7y+UnC86n8+ce4=", + "dev": true + }, + "node_modules/@types/minimatch": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz", + "integrity": "sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==", + "dev": true + }, + "node_modules/@types/node": { + "version": "14.0.27", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.0.27.tgz", + "integrity": "sha512-kVrqXhbclHNHGu9ztnAwSncIgJv/FaxmzXJvGXNdcCpV1b8u1/Mi6z6m0vwy0LzKeXFTPLH0NzwmoJ3fNCIq0g==", + "dev": true + }, + "node_modules/@types/sizzle": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/@types/sizzle/-/sizzle-2.3.2.tgz", + "integrity": "sha512-7EJYyKTL7tFR8+gDbB6Wwz/arpGa0Mywk1TJbNzKzHtzbwVmY4HR9WqS5VV7dsBUKQmPNr192jHr/VpBluj/hg==", + "dev": true + }, + "node_modules/@types/validator": { + "version": "10.11.3", + "resolved": "https://registry.npmjs.org/@types/validator/-/validator-10.11.3.tgz", + "integrity": "sha512-GKF2VnEkMmEeEGvoo03ocrP9ySMuX1ypKazIYMlsjfslfBMhOAtC5dmEWKdJioW4lJN7MZRS88kalTsVClyQ9w==", + "dev": true + }, + "node_modules/@typescript-eslint/eslint-plugin": { + "version": "2.34.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-2.34.0.tgz", + "integrity": "sha512-4zY3Z88rEE99+CNvTbXSyovv2z9PNOVffTWD2W8QF5s2prBQtwN2zadqERcrHpcR7O/+KMI3fcTAmUUhK/iQcQ==", + "dev": true, + "dependencies": { + "@typescript-eslint/experimental-utils": "2.34.0", + "functional-red-black-tree": "^1.0.1", + "regexpp": "^3.0.0", + "tsutils": "^3.17.1" + }, + "engines": { + "node": "^8.10.0 || ^10.13.0 || >=11.10.1" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "@typescript-eslint/parser": "^2.0.0", + "eslint": "^5.0.0 || ^6.0.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/@typescript-eslint/experimental-utils": { + "version": "2.34.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/experimental-utils/-/experimental-utils-2.34.0.tgz", + "integrity": "sha512-eS6FTkq+wuMJ+sgtuNTtcqavWXqsflWcfBnlYhg/nS4aZ1leewkXGbvBhaapn1q6qf4M71bsR1tez5JTRMuqwA==", + "dev": true, + "dependencies": { + "@types/json-schema": "^7.0.3", + "@typescript-eslint/typescript-estree": "2.34.0", + "eslint-scope": "^5.0.0", + "eslint-utils": "^2.0.0" + }, + "engines": { + "node": "^8.10.0 || ^10.13.0 || >=11.10.1" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "*" + } + }, + "node_modules/@typescript-eslint/experimental-utils/node_modules/eslint-utils": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-2.1.0.tgz", + "integrity": "sha512-w94dQYoauyvlDc43XnGB8lU3Zt713vNChgt4EWwhXAP2XkBvndfxF0AgIqKOOasjPIPzj9JqgwkwbCYD0/V3Zg==", + "dev": true, + "dependencies": { + "eslint-visitor-keys": "^1.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/mysticatea" + } + }, + "node_modules/@typescript-eslint/parser": { + "version": "2.34.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-2.34.0.tgz", + "integrity": "sha512-03ilO0ucSD0EPTw2X4PntSIRFtDPWjrVq7C3/Z3VQHRC7+13YB55rcJI3Jt+YgeHbjUdJPcPa7b23rXCBokuyA==", + "dev": true, + "dependencies": { + "@types/eslint-visitor-keys": "^1.0.0", + "@typescript-eslint/experimental-utils": "2.34.0", + "@typescript-eslint/typescript-estree": "2.34.0", + "eslint-visitor-keys": "^1.1.0" + }, + "engines": { + "node": "^8.10.0 || ^10.13.0 || >=11.10.1" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^5.0.0 || ^6.0.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/@typescript-eslint/typescript-estree": { + "version": "2.34.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-2.34.0.tgz", + "integrity": "sha512-OMAr+nJWKdlVM9LOqCqh3pQQPwxHAN7Du8DR6dmwCrAmxtiXQnhHJ6tBNtf+cggqfo51SG/FCwnKhXCIM7hnVg==", + "dev": true, + "dependencies": { + "debug": "^4.1.1", + "eslint-visitor-keys": "^1.1.0", + "glob": "^7.1.6", + "is-glob": "^4.0.1", + "lodash": "^4.17.15", + "semver": "^7.3.2", + "tsutils": "^3.17.1" + }, + "engines": { + "node": "^8.10.0 || ^10.13.0 || >=11.10.1" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "deprecated": "Debug versions >=3.2.0 <3.2.7 || >=4 <4.3.1 have a low-severity ReDos regression when used in a Node.js environment. It is recommended you upgrade to 3.2.7 or 4.3.1. (https://github.com/visionmedia/debug/issues/797)", + "dev": true, + "dependencies": { + "ms": "^2.1.1" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/semver": { + "version": "7.3.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz", + "integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==", + "dev": true, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/abab": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.4.tgz", + "integrity": "sha512-Eu9ELJWCz/c1e9gTiCY+FceWxcqzjYEbqMgtndnuSqZSUCOL73TWNK2mHfIj4Cw2E/ongOp+JISVNCmovt2KYQ==", + "dev": true + }, + "node_modules/acorn": { + "version": "7.4.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.0.tgz", + "integrity": "sha512-+G7P8jJmCHr+S+cLfQxygbWhXy+8YTVGzAkpEbcLo2mLoL7tij/VG41QSHACSf5QgYRhMZYHuNc6drJaO0Da+w==", + "dev": true, + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-globals": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-4.3.4.tgz", + "integrity": "sha512-clfQEh21R+D0leSbUdWf3OcfqyaCSAQ8Ryq00bofSekfr9W8u1jyYZo6ir0xu9Gtcf7BjcHJpnbZH7JOCpP60A==", + "dev": true, + "dependencies": { + "acorn": "^6.0.1", + "acorn-walk": "^6.0.1" + } + }, + "node_modules/acorn-globals/node_modules/acorn": { + "version": "6.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.1.tgz", + "integrity": "sha512-ZVA9k326Nwrj3Cj9jlh3wGFutC2ZornPNARZwsNYqQYgN0EsV2d53w5RN/co65Ohn4sUAUtb1rSUAOD6XN9idA==", + "dev": true, + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.2.0.tgz", + "integrity": "sha512-HiUX/+K2YpkpJ+SzBffkM/AQ2YE03S0U1kjTLVpoJdhZMOWy8qvXVN9JdLqv2QsaQ6MPYQIuNmwD8zOiYUofLQ==", + "dev": true, + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0" + } + }, + "node_modules/acorn-walk": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-6.2.0.tgz", + "integrity": "sha512-7evsyfH1cLOCdAzZAd43Cic04yKydNx0cF+7tiA19p1XnLLPU4dpCQOqpjqwokFe//vS0QqfqqjCS2JkiIs0cA==", + "dev": true, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/ajv": { + "version": "6.12.3", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.3.tgz", + "integrity": "sha512-4K0cK3L1hsqk9xIb2z9vs/XU+PGJZ9PNpJRDS9YLzmNdX6jmVPfamLvTJr0aDAusnHyCHO6MjzlkAsgtqp9teA==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/amdefine": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/amdefine/-/amdefine-1.0.1.tgz", + "integrity": "sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU=", + "dev": true, + "optional": true, + "engines": { + "node": ">=0.4.2" + } + }, + "node_modules/ansi-escapes": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-1.4.0.tgz", + "integrity": "sha1-06ioOzGapneTZisT52HHkRQiMG4=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/array-equal": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz", + "integrity": "sha1-jCpe8kcv2ep0KwTHenUJO6J1fJM=", + "dev": true + }, + "node_modules/array-includes": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.1.4.tgz", + "integrity": "sha512-ZTNSQkmWumEbiHO2GF4GmWxYVTiQyJy2XOTa15sdQSrvKn7l+180egQMqlrMOUMCyLMD7pmyQe4mMDUT6Behrw==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3", + "es-abstract": "^1.19.1", + "get-intrinsic": "^1.1.1", + "is-string": "^1.0.7" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/array-union": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", + "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/array.prototype.flat": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.2.5.tgz", + "integrity": "sha512-KaYU+S+ndVqyUnignHftkwc58o3uVU1jzczILJ1tN2YaIZpFIKBiP/x/j97E5MVPsaCloPbqWLB/8qCTVvT2qg==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3", + "es-abstract": "^1.19.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/arrify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz", + "integrity": "sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/asn1": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", + "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", + "dev": true, + "dependencies": { + "safer-buffer": "~2.1.0" + } + }, + "node_modules/assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=", + "dev": true, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/astral-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-1.0.0.tgz", + "integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/async": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz", + "integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==", + "dev": true, + "dependencies": { + "lodash": "^4.17.14" + } + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=", + "dev": true + }, + "node_modules/aws-sign2": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", + "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=", + "dev": true, + "engines": { + "node": "*" + } + }, + "node_modules/aws4": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.10.0.tgz", + "integrity": "sha512-3YDiu347mtVtjpyV3u5kVqQLP242c06zwDOgpeRnybmXlYYsLbtTrUBUm8i8srONt+FWobl5aibnU1030PeeuA==", + "dev": true + }, + "node_modules/babel-polyfill": { + "version": "6.23.0", + "resolved": "https://registry.npmjs.org/babel-polyfill/-/babel-polyfill-6.23.0.tgz", + "integrity": "sha1-g2TKYt+Or7gwSZ9pkXdGbDsDSZ0=", + "dev": true, + "dependencies": { + "babel-runtime": "^6.22.0", + "core-js": "^2.4.0", + "regenerator-runtime": "^0.10.0" + } + }, + "node_modules/babel-polyfill/node_modules/regenerator-runtime": { + "version": "0.10.5", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.10.5.tgz", + "integrity": "sha1-M2w+/BIgrc7dosn6tntaeVWjNlg=", + "dev": true + }, + "node_modules/babel-runtime": { + "version": "6.26.0", + "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz", + "integrity": "sha1-llxwWGaOgrVde/4E/yM3vItWR/4=", + "dev": true, + "dependencies": { + "core-js": "^2.4.0", + "regenerator-runtime": "^0.11.0" + } + }, + "node_modules/babel-runtime/node_modules/regenerator-runtime": { + "version": "0.11.1", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz", + "integrity": "sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg==", + "dev": true + }, + "node_modules/balanced-match": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", + "dev": true + }, + "node_modules/basic-auth": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-1.1.0.tgz", + "integrity": "sha1-RSIe5Cn37h5QNb4/UVM/HN/SmIQ=", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/bcrypt-pbkdf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", + "dev": true, + "dependencies": { + "tweetnacl": "^0.14.3" + } + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/browser-process-hrtime": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz", + "integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow==", + "dev": true + }, + "node_modules/buffer-from": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", + "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==", + "dev": true + }, + "node_modules/call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "dev": true, + "dependencies": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/camelcase": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-4.1.0.tgz", + "integrity": "sha1-1UVjW+HjPFQmScaRc+Xeas+uNN0=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=", + "dev": true + }, + "node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/chardet": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.4.2.tgz", + "integrity": "sha1-tUc7M9yXxCTl2Y3IfVXU2KKci/I=", + "dev": true + }, + "node_modules/charenc": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", + "integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc=", + "dev": true, + "engines": { + "node": "*" + } + }, + "node_modules/chrome-devtools-frontend": { + "version": "1.0.944903", + "resolved": "https://registry.npmjs.org/chrome-devtools-frontend/-/chrome-devtools-frontend-1.0.944903.tgz", + "integrity": "sha512-0AX3fSoR7l33Kxb4+U1QFbH4SkSKv4mhawDeex0CmbsmsdtfybI7y4NvN4Fen/+w5j/g4m6t79STQ8pjI+NrQA==", + "dev": true + }, + "node_modules/cli-cursor": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz", + "integrity": "sha1-s12sN2R5+sw+lHR9QdDQ9SOP/LU=", + "dev": true, + "dependencies": { + "restore-cursor": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/cli-width": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.0.tgz", + "integrity": "sha1-/xnt6Kml5XkyQUewwR8PvLq+1jk=", + "dev": true + }, + "node_modules/cliui": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-3.2.0.tgz", + "integrity": "sha1-EgYBU3qRbSmUD5NNo7SNWFo5IT0=", + "dev": true, + "dependencies": { + "string-width": "^1.0.1", + "strip-ansi": "^3.0.1", + "wrap-ansi": "^2.0.0" + } + }, + "node_modules/cliui/node_modules/string-width": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", + "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", + "dev": true, + "dependencies": { + "code-point-at": "^1.0.0", + "is-fullwidth-code-point": "^1.0.0", + "strip-ansi": "^3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/code-point-at": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", + "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "dev": true + }, + "node_modules/colorette": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-1.2.1.tgz", + "integrity": "sha512-puCDz0CzydiSYOrnXpz/PKd69zRrribezjtE9yd4zvytoRc8+RY/KJPvtPFKZS3E3wP6neGyMe0vOTlHO5L3Pw==", + "dev": true + }, + "node_modules/colors": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz", + "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==", + "dev": true, + "engines": { + "node": ">=0.1.90" + } + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dev": true, + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", + "dev": true + }, + "node_modules/core-js": { + "version": "2.6.10", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.10.tgz", + "integrity": "sha512-I39t74+4t+zau64EN1fE5v2W31Adtc/REhzWN+gWRRXg6WH5qAsZm62DHpQ1+Yhe4047T55jvzz7MUqF/dBBlA==", + "deprecated": "core-js@<3.3 is no longer maintained and not recommended for usage due to the number of issues. Because of the V8 engine whims, feature detection in old core-js versions could cause a slowdown up to 100x even if nothing is polyfilled. Please, upgrade your dependencies to the actual version of core-js.", + "dev": true, + "hasInstallScript": true + }, + "node_modules/core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", + "dev": true + }, + "node_modules/corser": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/corser/-/corser-2.0.1.tgz", + "integrity": "sha1-jtolLsqrWEDc2XXOuQ2TcMgZ/4c=", + "dev": true, + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/cross-spawn": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz", + "integrity": "sha1-6L0O/uWPz/b4+UUQoKVUu/ojVEk=", + "dev": true, + "dependencies": { + "lru-cache": "^4.0.1", + "shebang-command": "^1.2.0", + "which": "^1.2.9" + } + }, + "node_modules/crypt": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", + "integrity": "sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs=", + "dev": true, + "engines": { + "node": "*" + } + }, + "node_modules/cssom": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.4.4.tgz", + "integrity": "sha512-p3pvU7r1MyyqbTk+WbNJIgJjG2VmTIaB10rI93LzVPrmDJKkzKYMtxxyAvQXR/NS6otuzveI7+7BBq3SjBS2mw==", + "dev": true + }, + "node_modules/cssstyle": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-2.3.0.tgz", + "integrity": "sha512-AZL67abkUzIuvcHqk7c09cezpGNcxUxU4Ioi/05xHk4DQeTkWmGYftIE6ctU6AEt+Gn4n1lDStOtj7FKycP71A==", + "dev": true, + "dependencies": { + "cssom": "~0.3.6" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cssstyle/node_modules/cssom": { + "version": "0.3.8", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz", + "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==", + "dev": true + }, + "node_modules/dashdash": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", + "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", + "dev": true, + "dependencies": { + "assert-plus": "^1.0.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/data-urls": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-1.1.0.tgz", + "integrity": "sha512-YTWYI9se1P55u58gL5GkQHW4P6VJBJ5iBT+B5a7i2Tjadhv52paJG0qHX4A0OR6/t52odI64KP2YvFpkDOi3eQ==", + "dev": true, + "dependencies": { + "abab": "^2.0.0", + "whatwg-mimetype": "^2.2.0", + "whatwg-url": "^7.0.0" + } + }, + "node_modules/debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, + "dependencies": { + "ms": "^2.1.1" + } + }, + "node_modules/decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/deep-is": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", + "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=", + "dev": true + }, + "node_modules/define-properties": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", + "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", + "dev": true, + "dependencies": { + "object-keys": "^1.0.12" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=", + "dev": true, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/diff": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", + "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", + "dev": true, + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/dir-glob": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", + "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", + "dev": true, + "dependencies": { + "path-type": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/dir-glob/node_modules/path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/doctrine": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", + "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", + "dev": true, + "dependencies": { + "esutils": "^2.0.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/domexception": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/domexception/-/domexception-1.0.1.tgz", + "integrity": "sha512-raigMkn7CJNNo6Ihro1fzG7wr3fHuYVytzquZKX5n0yizGsTcYgzdIUwj1X9pK0VvjeihV+XiclP+DjwbsSKug==", + "dev": true, + "dependencies": { + "webidl-conversions": "^4.0.2" + } + }, + "node_modules/ecc-jsbn": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", + "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", + "dev": true, + "dependencies": { + "jsbn": "~0.1.0", + "safer-buffer": "^2.1.0" + } + }, + "node_modules/ecstatic": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/ecstatic/-/ecstatic-3.3.2.tgz", + "integrity": "sha512-fLf9l1hnwrHI2xn9mEDT7KIi22UDqA2jaCwyCbSUJh9a1V+LEUSL/JO/6TIz/QyuBURWUHrFL5Kg2TtO1bkkog==", + "deprecated": "This package is unmaintained and deprecated. See the GH Issue 259.", + "dev": true, + "dependencies": { + "he": "^1.1.1", + "mime": "^1.6.0", + "minimist": "^1.1.0", + "url-join": "^2.0.5" + }, + "bin": { + "ecstatic": "lib/ecstatic.js" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "node_modules/encoding": { + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.12.tgz", + "integrity": "sha1-U4tm8+5izRq1HsMjgp0flIDHS+s=", + "dev": true, + "dependencies": { + "iconv-lite": "~0.4.13" + } + }, + "node_modules/error-ex": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "dev": true, + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, + "node_modules/es-abstract": { + "version": "1.19.1", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.19.1.tgz", + "integrity": "sha512-2vJ6tjA/UfqLm2MPs7jxVybLoB8i1t1Jd9R3kISld20sIxPcTbLuggQOUxeWeAvIUkduv/CfMjuh4WmiXr2v9w==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.2", + "es-to-primitive": "^1.2.1", + "function-bind": "^1.1.1", + "get-intrinsic": "^1.1.1", + "get-symbol-description": "^1.0.0", + "has": "^1.0.3", + "has-symbols": "^1.0.2", + "internal-slot": "^1.0.3", + "is-callable": "^1.2.4", + "is-negative-zero": "^2.0.1", + "is-regex": "^1.1.4", + "is-shared-array-buffer": "^1.0.1", + "is-string": "^1.0.7", + "is-weakref": "^1.0.1", + "object-inspect": "^1.11.0", + "object-keys": "^1.1.1", + "object.assign": "^4.1.2", + "string.prototype.trimend": "^1.0.4", + "string.prototype.trimstart": "^1.0.4", + "unbox-primitive": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/es-to-primitive": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", + "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", + "dev": true, + "dependencies": { + "is-callable": "^1.1.4", + "is-date-object": "^1.0.1", + "is-symbol": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "dev": true, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/escodegen": { + "version": "1.14.3", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.14.3.tgz", + "integrity": "sha512-qFcX0XJkdg+PB3xjZZG/wKSuT1PnQWx57+TVSjIMmILd2yC/6ByYElPwJnslDsuWuSAp4AwJGumarAAmJch5Kw==", + "dev": true, + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^4.2.0", + "esutils": "^2.0.2", + "optionator": "^0.8.1" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=4.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, + "node_modules/escodegen/node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true, + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint": { + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-6.8.0.tgz", + "integrity": "sha512-K+Iayyo2LtyYhDSYwz5D5QdWw0hCacNzyq1Y821Xna2xSJj7cijoLLYmLxTQgcgZ9mC61nryMy9S7GRbYpI5Ig==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.0.0", + "ajv": "^6.10.0", + "chalk": "^2.1.0", + "cross-spawn": "^6.0.5", + "debug": "^4.0.1", + "doctrine": "^3.0.0", + "eslint-scope": "^5.0.0", + "eslint-utils": "^1.4.3", + "eslint-visitor-keys": "^1.1.0", + "espree": "^6.1.2", + "esquery": "^1.0.1", + "esutils": "^2.0.2", + "file-entry-cache": "^5.0.1", + "functional-red-black-tree": "^1.0.1", + "glob-parent": "^5.0.0", + "globals": "^12.1.0", + "ignore": "^4.0.6", + "import-fresh": "^3.0.0", + "imurmurhash": "^0.1.4", + "inquirer": "^7.0.0", + "is-glob": "^4.0.0", + "js-yaml": "^3.13.1", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.3.0", + "lodash": "^4.17.14", + "minimatch": "^3.0.4", + "mkdirp": "^0.5.1", + "natural-compare": "^1.4.0", + "optionator": "^0.8.3", + "progress": "^2.0.0", + "regexpp": "^2.0.1", + "semver": "^6.1.2", + "strip-ansi": "^5.2.0", + "strip-json-comments": "^3.0.1", + "table": "^5.2.3", + "text-table": "^0.2.0", + "v8-compile-cache": "^2.0.3" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^8.10.0 || ^10.13.0 || >=11.10.1" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-config-standard": { + "version": "14.1.1", + "resolved": "https://registry.npmjs.org/eslint-config-standard/-/eslint-config-standard-14.1.1.tgz", + "integrity": "sha512-Z9B+VR+JIXRxz21udPTL9HpFMyoMUEeX1G251EQ6e05WD9aPVtVBn09XUmZ259wCMlCDmYDSZG62Hhm+ZTJcUg==", + "dev": true, + "peerDependencies": { + "eslint": ">=6.2.2", + "eslint-plugin-import": ">=2.18.0", + "eslint-plugin-node": ">=9.1.0", + "eslint-plugin-promise": ">=4.2.1", + "eslint-plugin-standard": ">=4.0.0" + } + }, + "node_modules/eslint-import-resolver-node": { + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.6.tgz", + "integrity": "sha512-0En0w03NRVMn9Uiyn8YRPDKvWjxCWkslUEhGNTdGx15RvPJYQ+lbOlqrlNI2vEAs4pDYK4f/HN2TbDmk5TP0iw==", + "dev": true, + "dependencies": { + "debug": "^3.2.7", + "resolve": "^1.20.0" + } + }, + "node_modules/eslint-module-utils": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.7.1.tgz", + "integrity": "sha512-fjoetBXQZq2tSTWZ9yWVl2KuFrTZZH3V+9iD1V1RfpDgxzJR+mPd/KZmMiA8gbPqdBzpNiEHOuT7IYEWxrH0zQ==", + "dev": true, + "dependencies": { + "debug": "^3.2.7", + "find-up": "^2.1.0", + "pkg-dir": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-plugin-es": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-es/-/eslint-plugin-es-2.0.0.tgz", + "integrity": "sha512-f6fceVtg27BR02EYnBhgWLFQfK6bN4Ll0nQFrBHOlCsAyxeZkn0NHns5O0YZOPrV1B3ramd6cgFwaoFLcSkwEQ==", + "dev": true, + "dependencies": { + "eslint-utils": "^1.4.2", + "regexpp": "^3.0.0" + }, + "engines": { + "node": ">=8.10.0" + }, + "peerDependencies": { + "eslint": ">=4.19.1" + } + }, + "node_modules/eslint-plugin-es/node_modules/regexpp": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.0.0.tgz", + "integrity": "sha512-Z+hNr7RAVWxznLPuA7DIh8UNX1j9CDrUQxskw9IrBE1Dxue2lyXT+shqEIeLUjrokxIP8CMy1WkjgG3rTsd5/g==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/eslint-plugin-import": { + "version": "2.25.3", + "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.25.3.tgz", + "integrity": "sha512-RzAVbby+72IB3iOEL8clzPLzL3wpDrlwjsTBAQXgyp5SeTqqY+0bFubwuo+y/HLhNZcXV4XqTBO4LGsfyHIDXg==", + "dev": true, + "dependencies": { + "array-includes": "^3.1.4", + "array.prototype.flat": "^1.2.5", + "debug": "^2.6.9", + "doctrine": "^2.1.0", + "eslint-import-resolver-node": "^0.3.6", + "eslint-module-utils": "^2.7.1", + "has": "^1.0.3", + "is-core-module": "^2.8.0", + "is-glob": "^4.0.3", + "minimatch": "^3.0.4", + "object.values": "^1.1.5", + "resolve": "^1.20.0", + "tsconfig-paths": "^3.11.0" + }, + "engines": { + "node": ">=4" + }, + "peerDependencies": { + "eslint": "^2 || ^3 || ^4 || ^5 || ^6 || ^7.2.0 || ^8" + } + }, + "node_modules/eslint-plugin-import/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/eslint-plugin-import/node_modules/doctrine": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz", + "integrity": "sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==", + "dev": true, + "dependencies": { + "esutils": "^2.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/eslint-plugin-import/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + }, + "node_modules/eslint-plugin-node": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-node/-/eslint-plugin-node-10.0.0.tgz", + "integrity": "sha512-1CSyM/QCjs6PXaT18+zuAXsjXGIGo5Rw630rSKwokSs2jrYURQc4R5JZpoanNCqwNmepg+0eZ9L7YiRUJb8jiQ==", + "dev": true, + "dependencies": { + "eslint-plugin-es": "^2.0.0", + "eslint-utils": "^1.4.2", + "ignore": "^5.1.1", + "minimatch": "^3.0.4", + "resolve": "^1.10.1", + "semver": "^6.1.0" + }, + "engines": { + "node": ">=8.10.0" + }, + "peerDependencies": { + "eslint": ">=5.16.0" + } + }, + "node_modules/eslint-plugin-node/node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true, + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/eslint-plugin-promise": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-promise/-/eslint-plugin-promise-4.2.1.tgz", + "integrity": "sha512-VoM09vT7bfA7D+upt+FjeBO5eHIJQBUWki1aPvB+vbNiHS3+oGIJGIeyBtKQTME6UPXXy3vV07OL1tHd3ANuDw==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/eslint-plugin-standard": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-standard/-/eslint-plugin-standard-4.1.0.tgz", + "integrity": "sha512-ZL7+QRixjTR6/528YNGyDotyffm5OQst/sGxKDwGb9Uqs4In5Egi4+jbobhqJoyoCM6/7v/1A5fhQ7ScMtDjaQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "peerDependencies": { + "eslint": ">=5.0.0" + } + }, + "node_modules/eslint-scope": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.0.tgz", + "integrity": "sha512-iiGRvtxWqgtx5m8EyQUJihBloE4EnYeGE/bz1wSPwJE6tZuJUtHlhqDM4Xj2ukE8Dyy1+HCZ4hE0fzIVMzb58w==", + "dev": true, + "dependencies": { + "esrecurse": "^4.1.0", + "estraverse": "^4.1.1" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/eslint-utils": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-1.4.3.tgz", + "integrity": "sha512-fbBN5W2xdY45KulGXmLHZ3c3FHfVYmKg0IrAKGOkT/464PQsx2UeIzfz1RmEci+KLm1bBaAzZAh8+/E+XAeZ8Q==", + "dev": true, + "dependencies": { + "eslint-visitor-keys": "^1.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.1.0.tgz", + "integrity": "sha512-8y9YjtM1JBJU/A9Kc+SbaOV4y29sSWckBwMHa+FGtVj5gN/sbnKDf6xJUl+8g7FAij9LVaP8C24DUiH/f/2Z9A==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint/node_modules/ansi-escapes": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.1.tgz", + "integrity": "sha512-JWF7ocqNrp8u9oqpgV+wH5ftbt+cfvv+PTjOvKLT3AdYly/LmORARfEVT1iyjwN+4MqE5UmVKoAdIBqeoCHgLA==", + "dev": true, + "dependencies": { + "type-fest": "^0.11.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint/node_modules/ansi-regex": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/eslint/node_modules/ansi-styles": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz", + "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==", + "dev": true, + "dependencies": { + "@types/color-name": "^1.1.1", + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/eslint/node_modules/chardet": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz", + "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==", + "dev": true + }, + "node_modules/eslint/node_modules/cli-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", + "integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==", + "dev": true, + "dependencies": { + "restore-cursor": "^3.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/eslint/node_modules/cli-width": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-3.0.0.tgz", + "integrity": "sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw==", + "dev": true, + "engines": { + "node": ">= 10" + } + }, + "node_modules/eslint/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/eslint/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "node_modules/eslint/node_modules/cross-spawn": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz", + "integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==", + "dev": true, + "dependencies": { + "nice-try": "^1.0.4", + "path-key": "^2.0.1", + "semver": "^5.5.0", + "shebang-command": "^1.2.0", + "which": "^1.2.9" + }, + "engines": { + "node": ">=4.8" + } + }, + "node_modules/eslint/node_modules/cross-spawn/node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true, + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/eslint/node_modules/debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "deprecated": "Debug versions >=3.2.0 <3.2.7 || >=4 <4.3.1 have a low-severity ReDos regression when used in a Node.js environment. It is recommended you upgrade to 3.2.7 or 4.3.1. (https://github.com/visionmedia/debug/issues/797)", + "dev": true, + "dependencies": { + "ms": "^2.1.1" + } + }, + "node_modules/eslint/node_modules/external-editor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz", + "integrity": "sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew==", + "dev": true, + "dependencies": { + "chardet": "^0.7.0", + "iconv-lite": "^0.4.24", + "tmp": "^0.0.33" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint/node_modules/figures": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", + "integrity": "sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==", + "dev": true, + "dependencies": { + "escape-string-regexp": "^1.0.5" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint/node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/eslint/node_modules/ignore": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", + "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", + "dev": true, + "engines": { + "node": ">= 4" + } + }, + "node_modules/eslint/node_modules/inquirer": { + "version": "7.3.3", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-7.3.3.tgz", + "integrity": "sha512-JG3eIAj5V9CwcGvuOmoo6LB9kbAYT8HXffUl6memuszlwDC/qvFAJw49XJ5NROSFNPxp3iQg1GqkFhaY/CR0IA==", + "dev": true, + "dependencies": { + "ansi-escapes": "^4.2.1", + "chalk": "^4.1.0", + "cli-cursor": "^3.1.0", + "cli-width": "^3.0.0", + "external-editor": "^3.0.3", + "figures": "^3.0.0", + "lodash": "^4.17.19", + "mute-stream": "0.0.8", + "run-async": "^2.4.0", + "rxjs": "^6.6.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0", + "through": "^2.3.6" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/eslint/node_modules/inquirer/node_modules/chalk": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz", + "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/eslint/node_modules/inquirer/node_modules/strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/eslint/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/eslint/node_modules/mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/eslint/node_modules/mute-stream": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", + "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==", + "dev": true + }, + "node_modules/eslint/node_modules/onetime": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.1.tgz", + "integrity": "sha512-ZpZpjcJeugQfWsfyQlshVoowIIQ1qBGSVll4rfDq6JJVO//fesjoX808hXWfBjY+ROZgpKDI5TRSRBSoJiZ8eg==", + "dev": true, + "dependencies": { + "mimic-fn": "^2.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint/node_modules/optionator": { + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz", + "integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==", + "dev": true, + "dependencies": { + "deep-is": "~0.1.3", + "fast-levenshtein": "~2.0.6", + "levn": "~0.3.0", + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2", + "word-wrap": "~1.2.3" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/eslint/node_modules/regexpp": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-2.0.1.tgz", + "integrity": "sha512-lv0M6+TkDVniA3aD1Eg0DVpfU/booSu7Eev3TDO/mZKHBfVjgCGTV4t4buppESEYDtkArYFOxTJWv6S5C+iaNw==", + "dev": true, + "engines": { + "node": ">=6.5.0" + } + }, + "node_modules/eslint/node_modules/restore-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz", + "integrity": "sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==", + "dev": true, + "dependencies": { + "onetime": "^5.1.0", + "signal-exit": "^3.0.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/eslint/node_modules/run-async": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz", + "integrity": "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==", + "dev": true, + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/eslint/node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true, + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/eslint/node_modules/string-width": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.0.tgz", + "integrity": "sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/eslint/node_modules/string-width/node_modules/strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/eslint/node_modules/strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dev": true, + "dependencies": { + "ansi-regex": "^4.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/eslint/node_modules/strip-ansi/node_modules/ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/eslint/node_modules/supports-color": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.1.0.tgz", + "integrity": "sha512-oRSIpR8pxT1Wr2FquTNnGet79b3BWljqOuoW/h4oBhxJ/HUbX5nX6JSruTkvXDCFMwDPvsaTTbvMLKZWSy0R5g==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/eslint/node_modules/type-fest": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.11.0.tgz", + "integrity": "sha512-OdjXJxnCN1AvyLSzeKIgXTXxV+99ZuXl3Hpo9XpJAv9MBcHrrJOQ5kV7ypXOuQie+AmWG25hLbiKdwYTifzcfQ==", + "dev": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/espree": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-6.2.1.tgz", + "integrity": "sha512-ysCxRQY3WaXJz9tdbWOwuWr5Y/XrPTGX9Kiz3yoUXwW0VZ4w30HTkQLaGx/+ttFjF8i+ACbArnB4ce68a9m5hw==", + "dev": true, + "dependencies": { + "acorn": "^7.1.1", + "acorn-jsx": "^5.2.0", + "eslint-visitor-keys": "^1.1.0" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/esprima": { + "version": "2.7.3", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-2.7.3.tgz", + "integrity": "sha1-luO3DVd59q1JzQMmc9HDEnZ7pYE=", + "dev": true, + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/esquery": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.3.1.tgz", + "integrity": "sha512-olpvt9QG0vniUBZspVRN6lwB7hOZoTRtT+jzR+tS4ffYx2mzbw+z0XCOk44aaLYKApNX5nMm+E+P6o25ip/DHQ==", + "dev": true, + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esquery/node_modules/estraverse": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz", + "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==", + "dev": true, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esrecurse": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.2.1.tgz", + "integrity": "sha512-64RBB++fIOAXPw3P9cy89qfMlvZEXZkqqJkjqqXIvzP5ezRZjW+lPWjw35UX/3EhUPFYbg5ER4JYgDw4007/DQ==", + "dev": true, + "dependencies": { + "estraverse": "^4.1.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "dev": true, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estree-walker": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-0.6.1.tgz", + "integrity": "sha512-SqmZANLWS0mnatqbSfRP5g8OXZC12Fgg1IwNtLsyHDzJizORW4khDfjPqJZsemPWBB2uqykUah5YpQ6epsqC/w==", + "dev": true + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/eventemitter3": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.0.tgz", + "integrity": "sha512-qerSRB0p+UDEssxTtm6EDKcE7W4OaoisfIMl4CngyEhjpYglocpNg6UEqCvemdGhosAsg4sO2dXJOdyBifPGCg==", + "dev": true + }, + "node_modules/execa": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz", + "integrity": "sha1-lEvs00zEHuMqY6n68nrVpl/Fl3c=", + "dev": true, + "dependencies": { + "cross-spawn": "^5.0.1", + "get-stream": "^3.0.0", + "is-stream": "^1.1.0", + "npm-run-path": "^2.0.0", + "p-finally": "^1.0.0", + "signal-exit": "^3.0.0", + "strip-eof": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "dev": true + }, + "node_modules/external-editor": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-2.2.0.tgz", + "integrity": "sha512-bSn6gvGxKt+b7+6TKEv1ZycHleA7aHhRHyAqJyp5pbUFuYYNIzpZnQDk7AsYckyWdEnTeAnay0aCy2aV6iTk9A==", + "dev": true, + "dependencies": { + "chardet": "^0.4.0", + "iconv-lite": "^0.4.17", + "tmp": "^0.0.33" + }, + "engines": { + "node": ">=0.12" + } + }, + "node_modules/extsprintf": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", + "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=", + "dev": true, + "engines": [ + "node >=0.6.0" + ] + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", + "dev": true + }, + "node_modules/fastq": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.8.0.tgz", + "integrity": "sha512-SMIZoZdLh/fgofivvIkmknUXyPnvxRE3DhtZ5Me3Mrsk5gyPL42F0xr51TdRXskBxHfMp+07bcYzfsYEsSQA9Q==", + "dev": true, + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/figures": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz", + "integrity": "sha1-OrGi0qYsi/tDGgyUy3l6L84nyWI=", + "dev": true, + "dependencies": { + "escape-string-regexp": "^1.0.5" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/file-entry-cache": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-5.0.1.tgz", + "integrity": "sha512-bCg29ictuBaKUwwArK4ouCaqDgLZcysCFLmM/Yn/FDoqndh/9vNuQfXRDvTuXKLxfD/JtZQGKFT8MGcJBK644g==", + "dev": true, + "dependencies": { + "flat-cache": "^2.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/find-up": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", + "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=", + "dev": true, + "dependencies": { + "locate-path": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/flat-cache": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-2.0.1.tgz", + "integrity": "sha512-LoQe6yDuUMDzQAEH8sgmh4Md6oZnc/7PjtwjNFSzveXqSHt6ka9fPBuso7IGf9Rz4uqnSnWiFH2B/zj24a5ReA==", + "dev": true, + "dependencies": { + "flatted": "^2.0.0", + "rimraf": "2.6.3", + "write": "1.0.3" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/flatted": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.2.tgz", + "integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==", + "dev": true + }, + "node_modules/follow-redirects": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.11.0.tgz", + "integrity": "sha512-KZm0V+ll8PfBrKwMzdo5D13b1bur9Iq9Zd/RMmAoQQcl2PxxFml8cxXPaaPYVbV0RjNjq1CU7zIzAOqtUPudmA==", + "dev": true, + "dependencies": { + "debug": "^3.0.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/forever-agent": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=", + "dev": true, + "engines": { + "node": "*" + } + }, + "node_modules/form-data": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", + "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", + "dev": true, + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 0.12" + } + }, + "node_modules/fs-extra": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", + "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + }, + "engines": { + "node": ">=6 <7 || >=8" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", + "dev": true + }, + "node_modules/function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", + "dev": true + }, + "node_modules/functional-red-black-tree": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz", + "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", + "dev": true + }, + "node_modules/get-caller-file": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.3.tgz", + "integrity": "sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w==", + "dev": true + }, + "node_modules/get-intrinsic": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz", + "integrity": "sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==", + "dev": true, + "dependencies": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/get-symbol-description": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.0.0.tgz", + "integrity": "sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.2", + "get-intrinsic": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/getpass": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", + "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", + "dev": true, + "dependencies": { + "assert-plus": "^1.0.0" + } + }, + "node_modules/glob": { + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/globals": { + "version": "12.4.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz", + "integrity": "sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg==", + "dev": true, + "dependencies": { + "type-fest": "^0.8.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/globby": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/globby/-/globby-10.0.1.tgz", + "integrity": "sha512-sSs4inE1FB2YQiymcmTv6NWENryABjUNPeWhOvmn4SjtKybglsyPZxFB3U1/+L1bYi0rNZDqCLlHyLYDl1Pq5A==", + "dev": true, + "dependencies": { + "@types/glob": "^7.1.1", + "array-union": "^2.1.0", + "dir-glob": "^3.0.1", + "fast-glob": "^3.0.3", + "glob": "^7.1.3", + "ignore": "^5.1.1", + "merge2": "^1.2.3", + "slash": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/globby/node_modules/@nodelib/fs.stat": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.3.tgz", + "integrity": "sha512-bQBFruR2TAwoevBEd/NWMoAAtNGzTRgdrqnYCc7dhzfoNvqPzLyqlEQnzZ3kVnNrSp25iyxE00/3h2fqGAGArA==", + "dev": true, + "engines": { + "node": ">= 8" + } + }, + "node_modules/globby/node_modules/braces": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "dev": true, + "dependencies": { + "fill-range": "^7.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/globby/node_modules/fast-glob": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.4.tgz", + "integrity": "sha512-kr/Oo6PX51265qeuCYsyGypiO5uJFgBS0jksyG7FUeCyQzNwYnzrNIMR1NXfkZXsMYXYLRAHgISHBz8gQcxKHQ==", + "dev": true, + "dependencies": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.0", + "merge2": "^1.3.0", + "micromatch": "^4.0.2", + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/globby/node_modules/fill-range": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "dev": true, + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/globby/node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/globby/node_modules/micromatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.2.tgz", + "integrity": "sha512-y7FpHSbMUMoyPbYUSzO6PaZ6FyRnQOpHuKwbo1G+Knck95XVU4QAiKdGEnj5wwoS7PlOgthX/09u5iFJ+aYf5Q==", + "dev": true, + "dependencies": { + "braces": "^3.0.1", + "picomatch": "^2.0.5" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/globby/node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/google-libphonenumber": { + "version": "3.2.5", + "resolved": "https://registry.npmjs.org/google-libphonenumber/-/google-libphonenumber-3.2.5.tgz", + "integrity": "sha512-Y0r7MFCI11UDLn0KaMPBEInhROyIOkWkQIyvWMFVF2I+h+sHE3vbl5a7FVe39td6u/w+nlKDdUMP9dMOZyv+2Q==", + "dev": true, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.2.tgz", + "integrity": "sha512-IItsdsea19BoLC7ELy13q1iJFNmd7ofZH5+X/pJr90/nRoPEX0DJo1dHDbgtYWOhJhcCgMDTOw84RZ72q6lB+Q==", + "dev": true + }, + "node_modules/har-schema": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", + "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/har-validator": { + "version": "5.1.5", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.5.tgz", + "integrity": "sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==", + "deprecated": "this library is no longer supported", + "dev": true, + "dependencies": { + "ajv": "^6.12.3", + "har-schema": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dev": true, + "dependencies": { + "function-bind": "^1.1.1" + }, + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/has-ansi": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", + "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=", + "dev": true, + "dependencies": { + "ansi-regex": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/has-bigints": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.1.tgz", + "integrity": "sha512-LSBS2LjbNBTf6287JEbEzvJgftkF5qFkmCo9hDRpAzKhUOlJ+hx8dd4USs00SgsUNwc4617J9ki5YtEClM2ffA==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/has-symbols": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz", + "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==", + "dev": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz", + "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", + "dev": true, + "dependencies": { + "has-symbols": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/he": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", + "dev": true, + "bin": { + "he": "bin/he" + } + }, + "node_modules/hosted-git-info": { + "version": "2.8.9", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz", + "integrity": "sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==", + "dev": true + }, + "node_modules/html-encoding-sniffer": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-1.0.2.tgz", + "integrity": "sha512-71lZziiDnsuabfdYiUeWdCVyKuqwWi23L8YeIgV9jSSZHCtb6wB1BKWooH7L3tn4/FuZJMVWyNaIDr4RGmaSYw==", + "dev": true, + "dependencies": { + "whatwg-encoding": "^1.0.1" + } + }, + "node_modules/http-proxy": { + "version": "1.18.1", + "resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.1.tgz", + "integrity": "sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==", + "dev": true, + "dependencies": { + "eventemitter3": "^4.0.0", + "follow-redirects": "^1.0.0", + "requires-port": "^1.0.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/http-server": { + "version": "0.12.3", + "resolved": "https://registry.npmjs.org/http-server/-/http-server-0.12.3.tgz", + "integrity": "sha512-be0dKG6pni92bRjq0kvExtj/NrrAd28/8fCXkaI/4piTwQMSDSLMhWyW0NI1V+DBI3aa1HMlQu46/HjVLfmugA==", + "dev": true, + "dependencies": { + "basic-auth": "^1.0.3", + "colors": "^1.4.0", + "corser": "^2.0.1", + "ecstatic": "^3.3.2", + "http-proxy": "^1.18.0", + "minimist": "^1.2.5", + "opener": "^1.5.1", + "portfinder": "^1.0.25", + "secure-compare": "3.0.1", + "union": "~0.5.0" + }, + "bin": { + "hs": "bin/http-server", + "http-server": "bin/http-server" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/http-signature": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", + "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", + "dev": true, + "dependencies": { + "assert-plus": "^1.0.0", + "jsprim": "^1.2.2", + "sshpk": "^1.7.0" + }, + "engines": { + "node": ">=0.8", + "npm": ">=1.3.7" + } + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dev": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ignore": { + "version": "5.1.4", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.1.4.tgz", + "integrity": "sha512-MzbUSahkTW1u7JpKKjY7LCARd1fU5W2rLdxlM4kdkayuCwZImjkpluF9CM1aLewYJguPDqewLam18Y6AU69A8A==", + "dev": true, + "engines": { + "node": ">= 4" + } + }, + "node_modules/import-fresh": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.1.tgz", + "integrity": "sha512-6e1q1cnWP2RXD9/keSkxHScg508CdXqXWgWBaETNhyuBFz+kUZlKboh+ISK+bU++DmbHimVBrOz/zzPe0sZ3sQ==", + "dev": true, + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=", + "dev": true, + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "dev": true, + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "node_modules/inquirer": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-3.0.6.tgz", + "integrity": "sha1-4EqqnQW3o8ubD0B9BDdfBEcZA0c=", + "dev": true, + "dependencies": { + "ansi-escapes": "^1.1.0", + "chalk": "^1.0.0", + "cli-cursor": "^2.1.0", + "cli-width": "^2.0.0", + "external-editor": "^2.0.1", + "figures": "^2.0.0", + "lodash": "^4.3.0", + "mute-stream": "0.0.7", + "run-async": "^2.2.0", + "rx": "^4.1.0", + "string-width": "^2.0.0", + "strip-ansi": "^3.0.0", + "through": "^2.3.6" + } + }, + "node_modules/inquirer/node_modules/ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/inquirer/node_modules/chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", + "dev": true, + "dependencies": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/inquirer/node_modules/supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=", + "dev": true, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/internal-slot": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.3.tgz", + "integrity": "sha512-O0DB1JC/sPyZl7cIo78n5dR7eUSwwpYPiXRhTzNxZVAMUuB8vlnRFyLxdrVToks6XPLVnFfbzaVd5WLjhgg+vA==", + "dev": true, + "dependencies": { + "get-intrinsic": "^1.1.0", + "has": "^1.0.3", + "side-channel": "^1.0.4" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/invert-kv": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz", + "integrity": "sha1-EEqOSqym09jNFXqO+L+rLXo//bY=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ip-regex": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/ip-regex/-/ip-regex-2.1.0.tgz", + "integrity": "sha1-+ni/XS5pE8kRzp+BnuUUa7bYROk=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0=", + "dev": true + }, + "node_modules/is-bigint": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz", + "integrity": "sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==", + "dev": true, + "dependencies": { + "has-bigints": "^1.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-boolean-object": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz", + "integrity": "sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.2", + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", + "dev": true + }, + "node_modules/is-callable": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.4.tgz", + "integrity": "sha512-nsuwtxZfMX67Oryl9LCQ+upnC0Z0BgpwntpS89m1H/TLF0zNfzfLMV/9Wa/6MZsj0acpEjAO0KF1xT6ZdLl95w==", + "dev": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-core-module": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.8.0.tgz", + "integrity": "sha512-vd15qHsaqrRL7dtH6QNuy0ndJmRDrS9HAM1CAiSifNUFv4x1a0CCVsj18hJ1mShxIG6T2i1sO78MkP56r0nYRw==", + "dev": true, + "dependencies": { + "has": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-date-object": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.5.tgz", + "integrity": "sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==", + "dev": true, + "dependencies": { + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", + "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", + "dev": true, + "dependencies": { + "number-is-nan": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-negative-zero": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.1.tgz", + "integrity": "sha512-2z6JzQvZRa9A2Y7xC6dQQm4FSTSTNWjKIYYTt4246eMTJmIo0Q+ZyOsU66X8lxK1AbB92dFeglPLrhwpeRKO6w==", + "dev": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-number-object": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.6.tgz", + "integrity": "sha512-bEVOqiRcvo3zO1+G2lVMy+gkkEm9Yh7cDMRusKKu5ZJKPUYSJwICTKZrNKHA2EbSP0Tu0+6B/emsYNHZyn6K8g==", + "dev": true, + "dependencies": { + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-promise": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.1.0.tgz", + "integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o=", + "dev": true + }, + "node_modules/is-regex": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz", + "integrity": "sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.2", + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-shared-array-buffer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.1.tgz", + "integrity": "sha512-IU0NmyknYZN0rChcKhRO1X8LYz5Isj/Fsqh8NJOSf+N/hCOTwy29F32Ik7a+QszE63IdvmwdTPDd6cZ5pg4cwA==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-string": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.7.tgz", + "integrity": "sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==", + "dev": true, + "dependencies": { + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-symbol": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.4.tgz", + "integrity": "sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==", + "dev": true, + "dependencies": { + "has-symbols": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=", + "dev": true + }, + "node_modules/is-weakref": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-weakref/-/is-weakref-1.0.1.tgz", + "integrity": "sha512-b2jKc2pQZjaeFYWEf7ScFj+Be1I+PXmlu572Q8coTXZ+LD/QQZ7ShPMst8h16riVgyXTQwUsFEl74mDvc/3MHQ==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", + "dev": true + }, + "node_modules/isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=", + "dev": true + }, + "node_modules/jasmine": { + "version": "3.6.3", + "resolved": "https://registry.npmjs.org/jasmine/-/jasmine-3.6.3.tgz", + "integrity": "sha512-Th91zHsbsALWjDUIiU5d/W5zaYQsZFMPTdeNmi8GivZPmAaUAK8MblSG3yQI4VMGC/abF2us7ex60NH1AAIMTA==", + "dev": true, + "dependencies": { + "glob": "^7.1.6", + "jasmine-core": "~3.6.0" + }, + "bin": { + "jasmine": "bin/jasmine.js" + }, + "engines": { + "node": "^10 || ^12 || ^14" + } + }, + "node_modules/jasmine-core": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/jasmine-core/-/jasmine-core-3.6.0.tgz", + "integrity": "sha512-8uQYa7zJN8hq9z+g8z1bqCfdC8eoDAeVnM5sfqs7KHv9/ifoJ500m018fpFc7RDaO6SWCLCXwo/wPSNcdYTgcw==", + "dev": true + }, + "node_modules/jasmine-ts": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/jasmine-ts/-/jasmine-ts-0.3.0.tgz", + "integrity": "sha512-K5joodjVOh3bnD06CNXC8P5htDq/r0Rhjv66ECOpdIGFLly8kM7V+X/GXcd9kv+xO+tIq3q9Y8B5OF6yr/iiDw==", + "dev": true, + "dependencies": { + "yargs": "^8.0.2" + }, + "bin": { + "jasmine-ts": "lib/index.js" + }, + "engines": { + "node": ">= 5.12" + }, + "peerDependencies": { + "jasmine": ">= 2.0", + "ts-node": ">=3.2.0 <8", + "typescript": ">=2.4.1" + } + }, + "node_modules/javascript-obfuscator": { + "version": "0.18.8", + "resolved": "https://registry.npmjs.org/javascript-obfuscator/-/javascript-obfuscator-0.18.8.tgz", + "integrity": "sha512-lh/PT3dwLgv2vP9ymsmNZqjpbkDq1SQqpxLYtAerLzmYgHlQOzVpUe3NbFb8tUFNylMeg6ez1iEpA7oOQw/7Tw==", + "dev": true, + "hasInstallScript": true, + "dependencies": { + "@babel/runtime": "7.7.4", + "chalk": "3.0.0", + "chance": "1.1.3", + "class-validator": "0.11.0", + "commander": "4.0.1", + "escodegen-wallaby": "1.6.27", + "espree": "^6.1.2", + "estraverse": "4.3.0", + "eventemitter3": "4.0.0", + "inversify": "5.0.1", + "js-string-escape": "1.0.1", + "md5": "2.2.1", + "mkdirp": "0.5.1", + "multimatch": "4.0.0", + "opencollective": "1.0.3", + "reflect-metadata": "0.1.13", + "source-map-support": "0.5.16", + "string-template": "1.0.0", + "tslib": "1.10.0" + }, + "bin": { + "javascript-obfuscator": "bin/javascript-obfuscator" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/javascript-obfuscator/node_modules/@babel/runtime": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.7.4.tgz", + "integrity": "sha512-r24eVUUr0QqNZa+qrImUk8fn5SPhHq+IfYvIoIMg0do3GdK9sMdiLKP3GYVVaxpPKORgm8KRKaNTEhAjgIpLMw==", + "dev": true, + "dependencies": { + "regenerator-runtime": "^0.13.2" + } + }, + "node_modules/javascript-obfuscator/node_modules/acorn": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.1.1.tgz", + "integrity": "sha512-add7dgA5ppRPxCFJoAGfMDi7PIBXq1RtGo7BhbLaxwrXPOmw8gq48Y9ozT01hUKy9byMjlR20EJhu5zlkErEkg==", + "dev": true, + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/javascript-obfuscator/node_modules/acorn-jsx": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.2.0.tgz", + "integrity": "sha512-HiUX/+K2YpkpJ+SzBffkM/AQ2YE03S0U1kjTLVpoJdhZMOWy8qvXVN9JdLqv2QsaQ6MPYQIuNmwD8zOiYUofLQ==", + "dev": true, + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0" + } + }, + "node_modules/javascript-obfuscator/node_modules/ansi-styles": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz", + "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==", + "dev": true, + "dependencies": { + "@types/color-name": "^1.1.1", + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/javascript-obfuscator/node_modules/array-differ": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/array-differ/-/array-differ-3.0.0.tgz", + "integrity": "sha512-THtfYS6KtME/yIAhKjZ2ul7XI96lQGHRputJQHO80LAWQnuGP4iCIN8vdMRboGbIEYBwU33q8Tch1os2+X0kMg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/javascript-obfuscator/node_modules/arrify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/javascript-obfuscator/node_modules/chalk": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz", + "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/javascript-obfuscator/node_modules/chance": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chance/-/chance-1.1.3.tgz", + "integrity": "sha512-XeJsdoVAzDb1WRPRuMBesRSiWpW1uNTo5Fd7mYxPJsAfgX71+jfuCOHOdbyBz2uAUZ8TwKcXgWk3DMedFfJkbg==", + "dev": true + }, + "node_modules/javascript-obfuscator/node_modules/class-validator": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/class-validator/-/class-validator-0.11.0.tgz", + "integrity": "sha512-niAmmSPFku9xsnpYYrddy8NZRrCX3yyoZ/rgPKOilE5BG0Ma1eVCIxpR4X0LasL/6BzbYzsutG+mSbAXlh4zNw==", + "dev": true, + "dependencies": { + "@types/validator": "10.11.3", + "google-libphonenumber": "^3.1.6", + "validator": "12.0.0" + } + }, + "node_modules/javascript-obfuscator/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/javascript-obfuscator/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "node_modules/javascript-obfuscator/node_modules/commander": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-4.0.1.tgz", + "integrity": "sha512-IPF4ouhCP+qdlcmCedhxX4xiGBPyigb8v5NeUp+0LyhwLgxMqyp3S0vl7TAPfS/hiP7FC3caI/PB9lTmP8r1NA==", + "dev": true, + "engines": { + "node": ">= 6" + } + }, + "node_modules/javascript-obfuscator/node_modules/escodegen-wallaby": { + "version": "1.6.27", + "resolved": "https://registry.npmjs.org/escodegen-wallaby/-/escodegen-wallaby-1.6.27.tgz", + "integrity": "sha512-DFXbfLTNa/wiQRgqSseKhmzyDhwWJ4c/6IwsdAgA6qYL0ntsmSuuTMovEbA/8oCanSGkUR1qaLsDgKiWPllnjg==", + "dev": true, + "dependencies": { + "esprima": "^2.7.1", + "estraverse": "^1.9.1", + "esutils": "^2.0.2", + "optionator": "^0.8.1" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=0.10.0" + }, + "optionalDependencies": { + "source-map": "~0.2.0" + } + }, + "node_modules/javascript-obfuscator/node_modules/escodegen-wallaby/node_modules/estraverse": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-1.9.3.tgz", + "integrity": "sha1-r2fy3JIlgkFZUJJgkaQAXSnJu0Q=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/javascript-obfuscator/node_modules/espree": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-6.2.1.tgz", + "integrity": "sha512-ysCxRQY3WaXJz9tdbWOwuWr5Y/XrPTGX9Kiz3yoUXwW0VZ4w30HTkQLaGx/+ttFjF8i+ACbArnB4ce68a9m5hw==", + "dev": true, + "dependencies": { + "acorn": "^7.1.1", + "acorn-jsx": "^5.2.0", + "eslint-visitor-keys": "^1.1.0" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/javascript-obfuscator/node_modules/estraverse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "dev": true, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/javascript-obfuscator/node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/javascript-obfuscator/node_modules/inversify": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/inversify/-/inversify-5.0.1.tgz", + "integrity": "sha512-Ieh06s48WnEYGcqHepdsJUIJUXpwH5o5vodAX+DK2JA/gjy4EbEcQZxw+uFfzysmKjiLXGYwNG3qDZsKVMcINQ==", + "dev": true + }, + "node_modules/javascript-obfuscator/node_modules/minimist": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", + "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", + "dev": true + }, + "node_modules/javascript-obfuscator/node_modules/mkdirp": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", + "deprecated": "Legacy versions of mkdirp are no longer supported. Please update to mkdirp 1.x. (Note that the API surface has changed to use Promises in 1.x.)", + "dev": true, + "dependencies": { + "minimist": "0.0.8" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } + }, + "node_modules/javascript-obfuscator/node_modules/multimatch": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/multimatch/-/multimatch-4.0.0.tgz", + "integrity": "sha512-lDmx79y1z6i7RNx0ZGCPq1bzJ6ZoDDKbvh7jxr9SJcWLkShMzXrHbYVpTdnhNM5MXpDUxCQ4DgqVttVXlBgiBQ==", + "dev": true, + "dependencies": { + "@types/minimatch": "^3.0.3", + "array-differ": "^3.0.0", + "array-union": "^2.1.0", + "arrify": "^2.0.1", + "minimatch": "^3.0.4" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/javascript-obfuscator/node_modules/reflect-metadata": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz", + "integrity": "sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg==", + "dev": true + }, + "node_modules/javascript-obfuscator/node_modules/regenerator-runtime": { + "version": "0.13.5", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.5.tgz", + "integrity": "sha512-ZS5w8CpKFinUzOwW3c83oPeVXoNsrLsaCoLtJvAClH135j/R77RuymhiSErhm2lKcwSCIpmvIWSbDkIfAqKQlA==", + "dev": true + }, + "node_modules/javascript-obfuscator/node_modules/source-map": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.2.0.tgz", + "integrity": "sha1-2rc/vPwrqBm03gO9b26qSBZLP50=", + "dev": true, + "optional": true, + "dependencies": { + "amdefine": ">=0.0.4" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/javascript-obfuscator/node_modules/source-map-support": { + "version": "0.5.16", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.16.tgz", + "integrity": "sha512-efyLRJDr68D9hBBNIPWFjhpFzURh+KJykQwvMyW5UiZzYwoF6l4YMMDIJJEyFWxWCqfyxLzz6tSfUFR+kXXsVQ==", + "dev": true, + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/javascript-obfuscator/node_modules/source-map-support/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/javascript-obfuscator/node_modules/supports-color": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.1.0.tgz", + "integrity": "sha512-oRSIpR8pxT1Wr2FquTNnGet79b3BWljqOuoW/h4oBhxJ/HUbX5nX6JSruTkvXDCFMwDPvsaTTbvMLKZWSy0R5g==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/javascript-obfuscator/node_modules/tslib": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.10.0.tgz", + "integrity": "sha512-qOebF53frne81cf0S9B41ByenJ3/IuH8yJKngAX35CmiZySA0khhkovshKK+jGCaMnVomla7gVlIcc3EvKPbTQ==", + "dev": true + }, + "node_modules/javascript-obfuscator/node_modules/validator": { + "version": "12.0.0", + "resolved": "https://registry.npmjs.org/validator/-/validator-12.0.0.tgz", + "integrity": "sha512-r5zA1cQBEOgYlesRmSEwc9LkbfNLTtji+vWyaHzRZUxCTHdsX3bd+sdHfs5tGZ2W6ILGGsxWxCNwT/h3IY/3ng==", + "dev": true, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/jquery": { + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/jquery/-/jquery-3.5.1.tgz", + "integrity": "sha512-XwIBPqcMn57FxfT+Go5pzySnm4KWkT1Tv7gjrpT1srtf8Weynl6R273VJ5GjkRb51IzMp5nbaPjJXMWeju2MKg==", + "dev": true + }, + "node_modules/js-cookie": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/js-cookie/-/js-cookie-2.2.1.tgz", + "integrity": "sha512-HvdH2LzI/EAZcUwA8+0nKNtWHqS+ZmijLA30RwZA0bo7ToCckjK5MkGhjED9KoRcXO6BaGI3I9UIzSA1FKFPOQ==", + "dev": true + }, + "node_modules/js-string-escape": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/js-string-escape/-/js-string-escape-1.0.1.tgz", + "integrity": "sha1-4mJbrbwNZ8dTPp7cEGjFh65BN+8=", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true + }, + "node_modules/js-yaml": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.0.tgz", + "integrity": "sha512-/4IbIeHcD9VMHFqDR/gQ7EdZdLimOvW2DdcxFjdyyZ9NsbS+ccrXqVWDtab/lRl5AlUqmpBx8EhPaWR+OtY17A==", + "dev": true, + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/js-yaml/node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true, + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=", + "dev": true + }, + "node_modules/jsdom": { + "version": "15.2.1", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-15.2.1.tgz", + "integrity": "sha512-fAl1W0/7T2G5vURSyxBzrJ1LSdQn6Tr5UX/xD4PXDx/PDgwygedfW6El/KIj3xJ7FU61TTYnc/l/B7P49Eqt6g==", + "dev": true, + "dependencies": { + "abab": "^2.0.0", + "acorn": "^7.1.0", + "acorn-globals": "^4.3.2", + "array-equal": "^1.0.0", + "cssom": "^0.4.1", + "cssstyle": "^2.0.0", + "data-urls": "^1.1.0", + "domexception": "^1.0.1", + "escodegen": "^1.11.1", + "html-encoding-sniffer": "^1.0.2", + "nwsapi": "^2.2.0", + "parse5": "5.1.0", + "pn": "^1.1.0", + "request": "^2.88.0", + "request-promise-native": "^1.0.7", + "saxes": "^3.1.9", + "symbol-tree": "^3.2.2", + "tough-cookie": "^3.0.1", + "w3c-hr-time": "^1.0.1", + "w3c-xmlserializer": "^1.1.2", + "webidl-conversions": "^4.0.2", + "whatwg-encoding": "^1.0.5", + "whatwg-mimetype": "^2.3.0", + "whatwg-url": "^7.0.0", + "ws": "^7.0.0", + "xml-name-validator": "^3.0.0" + }, + "engines": { + "node": ">=8" + }, + "peerDependencies": { + "canvas": "^2.5.0" + }, + "peerDependenciesMeta": { + "canvas": { + "optional": true + } + } + }, + "node_modules/json-schema": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", + "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=", + "dev": true + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=", + "dev": true + }, + "node_modules/json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=", + "dev": true + }, + "node_modules/json5": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz", + "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==", + "dev": true, + "dependencies": { + "minimist": "^1.2.0" + }, + "bin": { + "json5": "lib/cli.js" + } + }, + "node_modules/jsonfile": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", + "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=", + "dev": true, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/jsprim": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", + "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", + "dev": true, + "engines": [ + "node >=0.6.0" + ], + "dependencies": { + "assert-plus": "1.0.0", + "extsprintf": "1.3.0", + "json-schema": "0.2.3", + "verror": "1.10.0" + } + }, + "node_modules/lcid": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz", + "integrity": "sha1-MIrMr6C8SDo4Z7S28rlQYlHRuDU=", + "dev": true, + "dependencies": { + "invert-kv": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/levn": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", + "integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=", + "dev": true, + "dependencies": { + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/load-json-file": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-2.0.0.tgz", + "integrity": "sha1-eUfkIUmvgNaWy/eXvKq8/h/inKg=", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.2", + "parse-json": "^2.2.0", + "pify": "^2.0.0", + "strip-bom": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/locate-path": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz", + "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=", + "dev": true, + "dependencies": { + "p-locate": "^2.0.0", + "path-exists": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "dev": true + }, + "node_modules/lodash.sortby": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz", + "integrity": "sha1-7dFMgk4sycHgsKG0K7UhBRakJDg=", + "dev": true + }, + "node_modules/lru-cache": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", + "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", + "dev": true, + "dependencies": { + "pseudomap": "^1.0.2", + "yallist": "^2.1.2" + } + }, + "node_modules/make-error": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.5.tgz", + "integrity": "sha512-c3sIjNUow0+8swNwVpqoH4YCShKNFkMaw6oH1mNS2haDZQqkeZFlHS3dhoeEbKKmJB4vXpJucU6oH75aDYeE9g==", + "dev": true + }, + "node_modules/md5": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/md5/-/md5-2.2.1.tgz", + "integrity": "sha1-U6s41f48iJG6RlMp6iP6wFQBJvk=", + "dev": true, + "dependencies": { + "charenc": "~0.0.1", + "crypt": "~0.0.1", + "is-buffer": "~1.1.1" + } + }, + "node_modules/mem": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/mem/-/mem-1.1.0.tgz", + "integrity": "sha1-Xt1StIXKHZAP5kiVUFOZoN+kX3Y=", + "dev": true, + "dependencies": { + "mimic-fn": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "dev": true, + "engines": { + "node": ">= 8" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "dev": true, + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.44.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.44.0.tgz", + "integrity": "sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.27", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.27.tgz", + "integrity": "sha512-JIhqnCasI9yD+SsmkquHBxTSEuZdQX5BuQnS2Vc7puQQQ+8yiP5AY5uWhpdv4YL4VM5c6iliiYWPgJ/nJQLp7w==", + "dev": true, + "dependencies": { + "mime-db": "1.44.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mimic-fn": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", + "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "dev": true + }, + "node_modules/mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "dev": true, + "dependencies": { + "minimist": "^1.2.5" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } + }, + "node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "node_modules/mute-stream": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz", + "integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=", + "dev": true + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", + "dev": true + }, + "node_modules/ncp": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz", + "integrity": "sha1-GVoh1sRuNh0vsSgbo4uR6d9727M=", + "dev": true, + "bin": { + "ncp": "bin/ncp" + } + }, + "node_modules/nice-try": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz", + "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", + "dev": true + }, + "node_modules/node-fetch": { + "version": "1.6.3", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.6.3.tgz", + "integrity": "sha1-3CNO3WSJmC1Y6PDbT2lQKavNjAQ=", + "dev": true, + "dependencies": { + "encoding": "^0.1.11", + "is-stream": "^1.0.1" + } + }, + "node_modules/normalize-package-data": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz", + "integrity": "sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA==", + "dev": true, + "dependencies": { + "hosted-git-info": "^2.1.4", + "resolve": "^1.10.0", + "semver": "2 || 3 || 4 || 5", + "validate-npm-package-license": "^3.0.1" + } + }, + "node_modules/npm-run-path": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", + "integrity": "sha1-NakjLfo11wZ7TLLd8jV7GHFTbF8=", + "dev": true, + "dependencies": { + "path-key": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/number-is-nan": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", + "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/nwsapi": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.0.tgz", + "integrity": "sha512-h2AatdwYH+JHiZpv7pt/gSX1XoRGb7L/qSIeuqA6GwYoF9w1vP1cw42TO0aI2pNyshRK5893hNSl+1//vHK7hQ==", + "dev": true + }, + "node_modules/oauth-sign": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", + "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==", + "dev": true, + "engines": { + "node": "*" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-inspect": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.11.0.tgz", + "integrity": "sha512-jp7ikS6Sd3GxQfZJPyH3cjcbJF6GZPClgdV+EFygjFLQ5FmW/dRUnTd9PQ9k0JhoNDabWFbpF1yCdSWCC6gexg==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "dev": true, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/object.assign": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.2.tgz", + "integrity": "sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.0", + "define-properties": "^1.1.3", + "has-symbols": "^1.0.1", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/object.values": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.1.5.tgz", + "integrity": "sha512-QUZRW0ilQ3PnPpbNtgdNV1PDbEqLIiSFB3l+EnGtBQ/8SUTLj1PZwtQHABZtLgwpJZTSZhuGLOGk57Drx2IvYg==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3", + "es-abstract": "^1.19.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "dev": true, + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/onetime": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz", + "integrity": "sha1-BnQoIw/WdEOyeUsiu6UotoZ5YtQ=", + "dev": true, + "dependencies": { + "mimic-fn": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/opencollective": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/opencollective/-/opencollective-1.0.3.tgz", + "integrity": "sha1-ruY3K8KBRFg2kMPKja7PwSDdDvE=", + "dev": true, + "dependencies": { + "babel-polyfill": "6.23.0", + "chalk": "1.1.3", + "inquirer": "3.0.6", + "minimist": "1.2.0", + "node-fetch": "1.6.3", + "opn": "4.0.2" + }, + "bin": { + "oc": "dist/bin/opencollective.js", + "opencollective": "dist/bin/opencollective.js" + } + }, + "node_modules/opencollective/node_modules/ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/opencollective/node_modules/chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", + "dev": true, + "dependencies": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/opencollective/node_modules/minimist": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=", + "dev": true + }, + "node_modules/opencollective/node_modules/supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=", + "dev": true, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/opener": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/opener/-/opener-1.5.1.tgz", + "integrity": "sha512-goYSy5c2UXE4Ra1xixabeVh1guIX/ZV/YokJksb6q2lubWu6UbvPQ20p542/sFIll1nl8JnCyK9oBaOcCWXwvA==", + "dev": true, + "bin": { + "opener": "bin/opener-bin.js" + } + }, + "node_modules/opn": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/opn/-/opn-4.0.2.tgz", + "integrity": "sha1-erwi5kTf9jsKltWrfyeQwPAavJU=", + "dev": true, + "dependencies": { + "object-assign": "^4.0.1", + "pinkie-promise": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/optionator": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.2.tgz", + "integrity": "sha1-NkxeQJ0/TWMB1sC0wFu6UBgK62Q=", + "dev": true, + "dependencies": { + "deep-is": "~0.1.3", + "fast-levenshtein": "~2.0.4", + "levn": "~0.3.0", + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2", + "wordwrap": "~1.0.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/os-locale": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-2.1.0.tgz", + "integrity": "sha512-3sslG3zJbEYcaC4YVAvDorjGxc7tv6KVATnLPZONiljsUncvihe9BQoVCEs0RZ1kmf4Hk9OBqlZfJZWI4GanKA==", + "dev": true, + "dependencies": { + "execa": "^0.7.0", + "lcid": "^1.0.0", + "mem": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/os-tmpdir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/p-limit": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", + "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", + "dev": true, + "dependencies": { + "p-try": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/p-locate": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz", + "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=", + "dev": true, + "dependencies": { + "p-limit": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/p-try": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", + "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/parse-json": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz", + "integrity": "sha1-9ID0BDTvgHQfhGkJn43qGPVaTck=", + "dev": true, + "dependencies": { + "error-ex": "^1.2.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/parse5": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-5.1.0.tgz", + "integrity": "sha512-fxNG2sQjHvlVAYmzBZS9YlDp6PTSSDwa98vkD4QgVDDCAo84z5X1t5XyJQ62ImdLXx5NdIIfihey6xpum9/gRQ==", + "dev": true + }, + "node_modules/path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-key": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", + "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "dev": true + }, + "node_modules/path-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-2.0.0.tgz", + "integrity": "sha1-8BLMuEFbcJb8LaoQVMPXI4lZTHM=", + "dev": true, + "dependencies": { + "pify": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/performance-now": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", + "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=", + "dev": true + }, + "node_modules/picomatch": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.2.2.tgz", + "integrity": "sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==", + "dev": true, + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pinkie": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz", + "integrity": "sha1-clVrgM+g1IqXToDnckjoDtT3+HA=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pinkie-promise": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", + "integrity": "sha1-ITXW36ejWMBprJsXh3YogihFD/o=", + "dev": true, + "dependencies": { + "pinkie": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pkg-dir": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-2.0.0.tgz", + "integrity": "sha1-9tXREJ4Z1j7fQo4L1X4Sd3YVM0s=", + "dev": true, + "dependencies": { + "find-up": "^2.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pn": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/pn/-/pn-1.1.0.tgz", + "integrity": "sha512-2qHaIQr2VLRFoxe2nASzsV6ef4yOOH+Fi9FBOVH6cqeSgUnoyySPZkxzLuzd+RYOQTRpROA0ztTMqxROKSb/nA==", + "dev": true + }, + "node_modules/portfinder": { + "version": "1.0.26", + "resolved": "https://registry.npmjs.org/portfinder/-/portfinder-1.0.26.tgz", + "integrity": "sha512-Xi7mKxJHHMI3rIUrnm/jjUgwhbYMkp/XKEcZX3aG4BrumLpq3nmoQMX+ClYnDZnZ/New7IatC1no5RX0zo1vXQ==", + "dev": true, + "dependencies": { + "async": "^2.6.2", + "debug": "^3.1.1", + "mkdirp": "^0.5.1" + }, + "engines": { + "node": ">= 0.12.0" + } + }, + "node_modules/prelude-ls": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", + "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=", + "dev": true, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "dev": true, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/pseudomap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", + "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=", + "dev": true + }, + "node_modules/psl": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz", + "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==", + "dev": true + }, + "node_modules/punycode": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/qs": { + "version": "6.9.4", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.4.tgz", + "integrity": "sha512-A1kFqHekCTM7cz0udomYUoYNWjBebHm/5wzU/XqrBRBNWectVH0QIiN+NEcZ0Dte5hvzHwbr8+XQmguPhJ6WdQ==", + "dev": true, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/read-pkg": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-2.0.0.tgz", + "integrity": "sha1-jvHAYjxqbbDcZxPEv6xGMysjaPg=", + "dev": true, + "dependencies": { + "load-json-file": "^2.0.0", + "normalize-package-data": "^2.3.2", + "path-type": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/read-pkg-up": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-2.0.0.tgz", + "integrity": "sha1-a3KoBImE4MQeeVEP1en6mbO1Sb4=", + "dev": true, + "dependencies": { + "find-up": "^2.0.0", + "read-pkg": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/regexpp": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.1.0.tgz", + "integrity": "sha512-ZOIzd8yVsQQA7j8GCSlPGXwg5PfmA1mrq0JP4nGhh54LaKN3xdai/vHUDu74pKwV8OxseMS65u2NImosQcSD0Q==", + "dev": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/mysticatea" + } + }, + "node_modules/request": { + "version": "2.88.2", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", + "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==", + "deprecated": "request has been deprecated, see https://github.com/request/request/issues/3142", + "dev": true, + "dependencies": { + "aws-sign2": "~0.7.0", + "aws4": "^1.8.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.6", + "extend": "~3.0.2", + "forever-agent": "~0.6.1", + "form-data": "~2.3.2", + "har-validator": "~5.1.3", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.19", + "oauth-sign": "~0.9.0", + "performance-now": "^2.1.0", + "qs": "~6.5.2", + "safe-buffer": "^5.1.2", + "tough-cookie": "~2.5.0", + "tunnel-agent": "^0.6.0", + "uuid": "^3.3.2" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/request-promise-core": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.4.tgz", + "integrity": "sha512-TTbAfBBRdWD7aNNOoVOBH4pN/KigV6LyapYNNlAPA8JwbovRti1E88m3sYAwsLi5ryhPKsE9APwnjFTgdUjTpw==", + "dev": true, + "dependencies": { + "lodash": "^4.17.19" + }, + "engines": { + "node": ">=0.10.0" + }, + "peerDependencies": { + "request": "^2.34" + } + }, + "node_modules/request-promise-native": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/request-promise-native/-/request-promise-native-1.0.9.tgz", + "integrity": "sha512-wcW+sIUiWnKgNY0dqCpOZkUbF/I+YPi+f09JZIDa39Ec+q82CpSYniDp+ISgTTbKmnpJWASeJBPZmoxH84wt3g==", + "deprecated": "request-promise-native has been deprecated because it extends the now deprecated request package, see https://github.com/request/request/issues/3142", + "dev": true, + "dependencies": { + "request-promise-core": "1.1.4", + "stealthy-require": "^1.1.1", + "tough-cookie": "^2.3.3" + }, + "engines": { + "node": ">=0.12.0" + }, + "peerDependencies": { + "request": "^2.34" + } + }, + "node_modules/request-promise-native/node_modules/tough-cookie": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", + "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", + "dev": true, + "dependencies": { + "psl": "^1.1.28", + "punycode": "^2.1.1" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/request/node_modules/qs": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", + "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==", + "dev": true, + "engines": { + "node": ">=0.6" + } + }, + "node_modules/request/node_modules/tough-cookie": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", + "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", + "dev": true, + "dependencies": { + "psl": "^1.1.28", + "punycode": "^2.1.1" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-main-filename": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-1.0.1.tgz", + "integrity": "sha1-l/cXtp1IeE9fUmpsWqj/3aBVpNE=", + "dev": true + }, + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8=", + "dev": true + }, + "node_modules/resolve": { + "version": "1.20.0", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.20.0.tgz", + "integrity": "sha512-wENBPt4ySzg4ybFQW2TT1zMQucPK95HSh/nq2CFTZVOGut2+pQvSsgtda4d26YrYcr067wjbmzOG8byDPBX63A==", + "dev": true, + "dependencies": { + "is-core-module": "^2.2.0", + "path-parse": "^1.0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/restore-cursor": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz", + "integrity": "sha1-n37ih/gv0ybU/RYpI9YhKe7g368=", + "dev": true, + "dependencies": { + "onetime": "^2.0.0", + "signal-exit": "^3.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", + "dev": true, + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/rimraf": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", + "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", + "dev": true, + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + } + }, + "node_modules/rollup": { + "version": "1.32.1", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-1.32.1.tgz", + "integrity": "sha512-/2HA0Ec70TvQnXdzynFffkjA6XN+1e2pEv/uKS5Ulca40g2L7KuOE3riasHoNVHOsFD5KKZgDsMk1CP3Tw9s+A==", + "dev": true, + "dependencies": { + "@types/estree": "*", + "@types/node": "*", + "acorn": "^7.1.0" + }, + "bin": { + "rollup": "dist/bin/rollup" + } + }, + "node_modules/rollup-plugin-copy": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/rollup-plugin-copy/-/rollup-plugin-copy-3.3.0.tgz", + "integrity": "sha512-euDjCUSBXZa06nqnwCNADbkAcYDfzwowfZQkto9K/TFhiH+QG7I4PUsEMwM9tDgomGWJc//z7KLW8t+tZwxADA==", + "dev": true, + "dependencies": { + "@types/fs-extra": "^8.0.1", + "colorette": "^1.1.0", + "fs-extra": "^8.1.0", + "globby": "10.0.1", + "is-plain-object": "^3.0.0" + }, + "engines": { + "node": ">=8.3" + } + }, + "node_modules/rollup-plugin-copy/node_modules/is-plain-object": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-3.0.1.tgz", + "integrity": "sha512-Xnpx182SBMrr/aBik8y+GuR4U1L9FqMSojwDQwPMmxyC6bvEqly9UBCxhauBF5vNh2gwWJNX6oDV7O+OM4z34g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/rollup-plugin-javascript-obfuscator": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/rollup-plugin-javascript-obfuscator/-/rollup-plugin-javascript-obfuscator-1.0.4.tgz", + "integrity": "sha512-pFn5NTqbjWDNMW2WIW9x+GecouGN5Y6fd6oOPLtLwbb0VlBoAiflrbW7WqK1k19ptEIAf5IfAYv0GNIVefhw/A==", + "dev": true, + "dependencies": { + "javascript-obfuscator": "^0.18.1" + } + }, + "node_modules/rollup-plugin-typescript": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/rollup-plugin-typescript/-/rollup-plugin-typescript-1.0.1.tgz", + "integrity": "sha512-rwJDNn9jv/NsKZuyBb/h0jsclP4CJ58qbvZt2Q9zDIGILF2LtdtvCqMOL+Gq9IVq5MTrTlHZNrn8h7VjQgd8tw==", + "deprecated": "This package has been deprecated and is no longer maintained. Please use @rollup/plugin-typescript.", + "dev": true, + "dependencies": { + "resolve": "^1.10.0", + "rollup-pluginutils": "^2.5.0" + }, + "peerDependencies": { + "tslib": "*", + "typescript": ">=2.1.0" + } + }, + "node_modules/rollup-pluginutils": { + "version": "2.8.2", + "resolved": "https://registry.npmjs.org/rollup-pluginutils/-/rollup-pluginutils-2.8.2.tgz", + "integrity": "sha512-EEp9NhnUkwY8aif6bxgovPHMoMoNr2FulJziTndpt5H9RdwC47GSGuII9XxpSdzVGM0GWrNPHV6ie1LTNJPaLQ==", + "dev": true, + "dependencies": { + "estree-walker": "^0.6.1" + } + }, + "node_modules/run-async": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.3.0.tgz", + "integrity": "sha1-A3GrSuC91yDUFm19/aZP96RFpsA=", + "dev": true, + "dependencies": { + "is-promise": "^2.1.0" + }, + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/run-parallel": { + "version": "1.1.9", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.1.9.tgz", + "integrity": "sha512-DEqnSRTDw/Tc3FXf49zedI638Z9onwUotBMiUFKmrO2sdFKIbXamXGQ3Axd4qgphxKB4kw/qP1w5kTxnfU1B9Q==", + "dev": true + }, + "node_modules/rx": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/rx/-/rx-4.1.0.tgz", + "integrity": "sha1-pfE/957zt0D+MKqAP7CfmIBdR4I=", + "dev": true + }, + "node_modules/rxjs": { + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.6.2.tgz", + "integrity": "sha512-BHdBMVoWC2sL26w//BCu3YzKT4s2jip/WhwsGEDmeKYBhKDZeYezVUnHatYB7L85v5xs0BAQmg6BEYJEKxBabg==", + "dev": true, + "dependencies": { + "tslib": "^1.9.0" + }, + "engines": { + "npm": ">=2.0.0" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true + }, + "node_modules/saxes": { + "version": "3.1.11", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-3.1.11.tgz", + "integrity": "sha512-Ydydq3zC+WYDJK1+gRxRapLIED9PWeSuuS41wqyoRmzvhhh9nc+QQrVMKJYzJFULazeGhzSV0QleN2wD3boh2g==", + "dev": true, + "dependencies": { + "xmlchars": "^2.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/secure-compare": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/secure-compare/-/secure-compare-3.0.1.tgz", + "integrity": "sha1-8aAymzCLIh+uN7mXTz1XjQypmeM=", + "dev": true + }, + "node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true, + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=", + "dev": true + }, + "node_modules/shebang-command": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", + "integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=", + "dev": true, + "dependencies": { + "shebang-regex": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/shebang-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", + "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/side-channel": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", + "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/signal-exit": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", + "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=", + "dev": true + }, + "node_modules/slash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/slice-ansi": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-2.1.0.tgz", + "integrity": "sha512-Qu+VC3EwYLldKa1fCxuuvULvSJOKEgk9pi8dZeCVK7TqBfUNTH4sFkk4joj8afVSfAYgJoSOetjx9QWOJ5mYoQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.0", + "astral-regex": "^1.0.0", + "is-fullwidth-code-point": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/slice-ansi/node_modules/is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.13", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.13.tgz", + "integrity": "sha512-SHSKFHadjVA5oR4PPqhtAVdcBWwRYVd6g6cAXnIbRiIwc2EhPrTuKUBdSLvlEKyIP3GCf89fltvcZiP9MMFA1w==", + "dev": true, + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/spdx-correct": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.0.tgz", + "integrity": "sha512-lr2EZCctC2BNR7j7WzJ2FpDznxky1sjfxvvYEyzxNyb6lZXHODmEoJeFu4JupYlkfha1KZpJyoqiJ7pgA1qq8Q==", + "dev": true, + "dependencies": { + "spdx-expression-parse": "^3.0.0", + "spdx-license-ids": "^3.0.0" + } + }, + "node_modules/spdx-exceptions": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.2.0.tgz", + "integrity": "sha512-2XQACfElKi9SlVb1CYadKDXvoajPgBVPn/gOQLrTvHdElaVhr7ZEbqJaRnJLVNeaI4cMEAgVCeBMKF6MWRDCRA==", + "dev": true + }, + "node_modules/spdx-expression-parse": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.0.tgz", + "integrity": "sha512-Yg6D3XpRD4kkOmTpdgbUiEJFKghJH03fiC1OPll5h/0sO6neh2jqRDVHOQ4o/LMea0tgCkbMgea5ip/e+MkWyg==", + "dev": true, + "dependencies": { + "spdx-exceptions": "^2.1.0", + "spdx-license-ids": "^3.0.0" + } + }, + "node_modules/spdx-license-ids": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.5.tgz", + "integrity": "sha512-J+FWzZoynJEXGphVIS+XEh3kFSjZX/1i9gFBaWQcB+/tmpe2qUsSBABpcxqxnAxFdiUFEgAX1bjYGQvIZmoz9Q==", + "dev": true + }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", + "dev": true + }, + "node_modules/sshpk": { + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", + "integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==", + "dev": true, + "dependencies": { + "asn1": "~0.2.3", + "assert-plus": "^1.0.0", + "bcrypt-pbkdf": "^1.0.0", + "dashdash": "^1.12.0", + "ecc-jsbn": "~0.1.1", + "getpass": "^0.1.1", + "jsbn": "~0.1.0", + "safer-buffer": "^2.0.2", + "tweetnacl": "~0.14.0" + }, + "bin": { + "sshpk-conv": "bin/sshpk-conv", + "sshpk-sign": "bin/sshpk-sign", + "sshpk-verify": "bin/sshpk-verify" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stealthy-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz", + "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/string-template": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/string-template/-/string-template-1.0.0.tgz", + "integrity": "sha1-np8iM9wA8hhxjsN5oopWc+zKi5Y=", + "dev": true + }, + "node_modules/string-width": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "dev": true, + "dependencies": { + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^4.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/string-width/node_modules/ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/string-width/node_modules/is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/string-width/node_modules/strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "dev": true, + "dependencies": { + "ansi-regex": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/string.prototype.trimend": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.4.tgz", + "integrity": "sha512-y9xCjw1P23Awk8EvTpcyL2NIr1j7wJ39f+k6lvRnSMz+mz9CGz9NYPelDk42kOz6+ql8xjfK8oYzy3jAP5QU5A==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/string.prototype.trimstart": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.4.tgz", + "integrity": "sha512-jh6e984OBfvxS50tdY2nRZnoC5/mLFKOREQfw8t5yytkoUsJRNxvI/E39qu1sD0OtWI3OC0XgKSmcWwziwYuZw==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "dev": true, + "dependencies": { + "ansi-regex": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/strip-bom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", + "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/strip-eof": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", + "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/symbol-tree": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", + "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", + "dev": true + }, + "node_modules/table": { + "version": "5.4.6", + "resolved": "https://registry.npmjs.org/table/-/table-5.4.6.tgz", + "integrity": "sha512-wmEc8m4fjnob4gt5riFRtTu/6+4rSe12TpAELNSqHMfF3IqnA+CH37USM6/YR3qRZv7e56kAEAtd6nKZaxe0Ug==", + "dev": true, + "dependencies": { + "ajv": "^6.10.2", + "lodash": "^4.17.14", + "slice-ansi": "^2.1.0", + "string-width": "^3.0.0" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/table/node_modules/ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/table/node_modules/emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", + "dev": true + }, + "node_modules/table/node_modules/is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/table/node_modules/string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dev": true, + "dependencies": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/table/node_modules/strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dev": true, + "dependencies": { + "ansi-regex": "^4.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/text-table": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", + "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", + "dev": true + }, + "node_modules/through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=", + "dev": true + }, + "node_modules/tmp": { + "version": "0.0.33", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", + "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", + "dev": true, + "dependencies": { + "os-tmpdir": "~1.0.2" + }, + "engines": { + "node": ">=0.6.0" + } + }, + "node_modules/tough-cookie": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-3.0.1.tgz", + "integrity": "sha512-yQyJ0u4pZsv9D4clxO69OEjLWYw+jbgspjTue4lTQZLfV0c5l1VmK2y1JK8E9ahdpltPOaAThPcp5nKPUgSnsg==", + "dev": true, + "dependencies": { + "ip-regex": "^2.1.0", + "psl": "^1.1.28", + "punycode": "^2.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/tr46": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-1.0.1.tgz", + "integrity": "sha1-qLE/1r/SSJUZZ0zN5VujaTtwbQk=", + "dev": true, + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/ts-node": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-7.0.1.tgz", + "integrity": "sha512-BVwVbPJRspzNh2yfslyT1PSbl5uIk03EZlb493RKHN4qej/D06n1cEhjlOJG69oFsE7OT8XjpTUcYf6pKTLMhw==", + "dev": true, + "dependencies": { + "arrify": "^1.0.0", + "buffer-from": "^1.1.0", + "diff": "^3.1.0", + "make-error": "^1.1.1", + "minimist": "^1.2.0", + "mkdirp": "^0.5.1", + "source-map-support": "^0.5.6", + "yn": "^2.0.0" + }, + "bin": { + "ts-node": "dist/bin.js" + }, + "engines": { + "node": ">=4.2.0" + } + }, + "node_modules/tsconfig-paths": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.12.0.tgz", + "integrity": "sha512-e5adrnOYT6zqVnWqZu7i/BQ3BnhzvGbjEjejFXO20lKIKpwTaupkCPgEfv4GZK1IBciJUEhYs3J3p75FdaTFVg==", + "dev": true, + "dependencies": { + "@types/json5": "^0.0.29", + "json5": "^1.0.1", + "minimist": "^1.2.0", + "strip-bom": "^3.0.0" + } + }, + "node_modules/tslib": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.13.0.tgz", + "integrity": "sha512-i/6DQjL8Xf3be4K/E6Wgpekn5Qasl1usyw++dAA35Ue5orEn65VIxOA+YvNNl9HV3qv70T7CNwjODHZrLwvd1Q==", + "dev": true + }, + "node_modules/tsutils": { + "version": "3.17.1", + "resolved": "https://registry.npmjs.org/tsutils/-/tsutils-3.17.1.tgz", + "integrity": "sha512-kzeQ5B8H3w60nFY2g8cJIuH7JDpsALXySGtwGJ0p2LSjLgay3NdIpqq5SoOBe46bKDW2iq25irHCr8wjomUS2g==", + "dev": true, + "dependencies": { + "tslib": "^1.8.1" + }, + "engines": { + "node": ">= 6" + }, + "peerDependencies": { + "typescript": ">=2.8.0 || >= 3.2.0-dev || >= 3.3.0-dev || >= 3.4.0-dev || >= 3.5.0-dev || >= 3.6.0-dev || >= 3.6.0-beta || >= 3.7.0-dev || >= 3.7.0-beta" + } + }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "dev": true, + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, + "node_modules/tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=", + "dev": true + }, + "node_modules/type-check": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", + "integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=", + "dev": true, + "dependencies": { + "prelude-ls": "~1.1.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/type-fest": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz", + "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/typescript": { + "version": "3.9.7", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.7.tgz", + "integrity": "sha512-BLbiRkiBzAwsjut4x/dsibSTB6yWpwT5qWmC2OfuCg3GgVQCSgMs4vEctYPhsaGtd0AeuuHMkjZ2h2WG8MSzRw==", + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=4.2.0" + } + }, + "node_modules/unbox-primitive": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.1.tgz", + "integrity": "sha512-tZU/3NqK3dA5gpE1KtyiJUrEB0lxnGkMFHptJ7q6ewdZ8s12QrODwNbhIJStmJkd1QDXa1NRA8aF2A1zk/Ypyw==", + "dev": true, + "dependencies": { + "function-bind": "^1.1.1", + "has-bigints": "^1.0.1", + "has-symbols": "^1.0.2", + "which-boxed-primitive": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/union": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/union/-/union-0.5.0.tgz", + "integrity": "sha512-N6uOhuW6zO95P3Mel2I2zMsbsanvvtgn6jVqJv4vbVcz/JN0OkL9suomjQGmWtxJQXOCqUJvquc1sMeNz/IwlA==", + "dev": true, + "dependencies": { + "qs": "^6.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/universalify": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", + "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==", + "dev": true, + "engines": { + "node": ">= 4.0.0" + } + }, + "node_modules/uri-js": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", + "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", + "dev": true, + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/url-join": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/url-join/-/url-join-2.0.5.tgz", + "integrity": "sha1-WvIvGMBSoACkjXuCxenC4v7tpyg=", + "dev": true + }, + "node_modules/uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", + "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", + "dev": true, + "bin": { + "uuid": "bin/uuid" + } + }, + "node_modules/v8-compile-cache": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.1.1.tgz", + "integrity": "sha512-8OQ9CL+VWyt3JStj7HX7/ciTL2V3Rl1Wf5OL+SNTm0yK1KvtReVulksyeRnCANHHuUxHlQig+JJDlUhBt1NQDQ==", + "dev": true + }, + "node_modules/validate-npm-package-license": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", + "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==", + "dev": true, + "dependencies": { + "spdx-correct": "^3.0.0", + "spdx-expression-parse": "^3.0.0" + } + }, + "node_modules/verror": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", + "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", + "dev": true, + "engines": [ + "node >=0.6.0" + ], + "dependencies": { + "assert-plus": "^1.0.0", + "core-util-is": "1.0.2", + "extsprintf": "^1.2.0" + } + }, + "node_modules/w3c-hr-time": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz", + "integrity": "sha512-z8P5DvDNjKDoFIHK7q8r8lackT6l+jo/Ye3HOle7l9nICP9lf1Ci25fy9vHd0JOWewkIFzXIEig3TdKT7JQ5fQ==", + "dev": true, + "dependencies": { + "browser-process-hrtime": "^1.0.0" + } + }, + "node_modules/w3c-xmlserializer": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-1.1.2.tgz", + "integrity": "sha512-p10l/ayESzrBMYWRID6xbuCKh2Fp77+sA0doRuGn4tTIMrrZVeqfpKjXHY+oDh3K4nLdPgNwMTVP6Vp4pvqbNg==", + "dev": true, + "dependencies": { + "domexception": "^1.0.1", + "webidl-conversions": "^4.0.2", + "xml-name-validator": "^3.0.0" + } + }, + "node_modules/webidl-conversions": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz", + "integrity": "sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==", + "dev": true + }, + "node_modules/whatwg-encoding": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-1.0.5.tgz", + "integrity": "sha512-b5lim54JOPN9HtzvK9HFXvBma/rnfFeqsic0hSpjtDbVxR3dJKLc+KB4V6GgiGOvl7CY/KNh8rxSo9DKQrnUEw==", + "dev": true, + "dependencies": { + "iconv-lite": "0.4.24" + } + }, + "node_modules/whatwg-mimetype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz", + "integrity": "sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g==", + "dev": true + }, + "node_modules/whatwg-url": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-7.1.0.tgz", + "integrity": "sha512-WUu7Rg1DroM7oQvGWfOiAK21n74Gg+T4elXEQYkOhtyLeWiJFoOGLXPKI/9gzIie9CtwVLm8wtw6YJdKyxSjeg==", + "dev": true, + "dependencies": { + "lodash.sortby": "^4.7.0", + "tr46": "^1.0.1", + "webidl-conversions": "^4.0.2" + } + }, + "node_modules/which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "dev": true, + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "which": "bin/which" + } + }, + "node_modules/which-boxed-primitive": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz", + "integrity": "sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==", + "dev": true, + "dependencies": { + "is-bigint": "^1.0.1", + "is-boolean-object": "^1.1.0", + "is-number-object": "^1.0.4", + "is-string": "^1.0.5", + "is-symbol": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/which-module": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", + "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=", + "dev": true + }, + "node_modules/word-wrap": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", + "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/wordwrap": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", + "integrity": "sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus=", + "dev": true + }, + "node_modules/wrap-ansi": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", + "integrity": "sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU=", + "dev": true, + "dependencies": { + "string-width": "^1.0.1", + "strip-ansi": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/wrap-ansi/node_modules/string-width": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", + "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", + "dev": true, + "dependencies": { + "code-point-at": "^1.0.0", + "is-fullwidth-code-point": "^1.0.0", + "strip-ansi": "^3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", + "dev": true + }, + "node_modules/write": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/write/-/write-1.0.3.tgz", + "integrity": "sha512-/lg70HAjtkUgWPVZhZcm+T4hkL8Zbtp1nFNOn3lRrxnlv50SRBv7cR7RqR+GMsd3hUXy9hWBo4CHTbFTcOYwig==", + "dev": true, + "dependencies": { + "mkdirp": "^0.5.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/ws": { + "version": "7.4.6", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.6.tgz", + "integrity": "sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A==", + "dev": true, + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/xml-name-validator": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz", + "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==", + "dev": true + }, + "node_modules/xmlchars": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", + "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", + "dev": true + }, + "node_modules/y18n": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-3.2.2.tgz", + "integrity": "sha512-uGZHXkHnhF0XeeAPgnKfPv1bgKAYyVvmNL1xlKsPYZPaIHxGti2hHqvOCQv71XMsLxu1QjergkqogUnms5D3YQ==", + "dev": true + }, + "node_modules/yallist": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", + "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=", + "dev": true + }, + "node_modules/yargs": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-8.0.2.tgz", + "integrity": "sha1-YpmpBVsc78lp/355wdkY3Osiw2A=", + "dev": true, + "dependencies": { + "camelcase": "^4.1.0", + "cliui": "^3.2.0", + "decamelize": "^1.1.1", + "get-caller-file": "^1.0.1", + "os-locale": "^2.0.0", + "read-pkg-up": "^2.0.0", + "require-directory": "^2.1.1", + "require-main-filename": "^1.0.1", + "set-blocking": "^2.0.0", + "string-width": "^2.0.0", + "which-module": "^2.0.0", + "y18n": "^3.2.1", + "yargs-parser": "^7.0.0" + } + }, + "node_modules/yargs-parser": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-7.0.0.tgz", + "integrity": "sha1-jQrELxbqVd69MyyvTEA4s+P139k=", + "dev": true, + "dependencies": { + "camelcase": "^4.1.0" + } + }, + "node_modules/yn": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/yn/-/yn-2.0.0.tgz", + "integrity": "sha1-5a2ryKz0CPY4X8dklWhMiOavaJo=", + "dev": true, + "engines": { + "node": ">=4" + } + } + }, "dependencies": { "@babel/code-frame": { "version": "7.10.4", @@ -265,7 +5616,8 @@ "version": "5.2.0", "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.2.0.tgz", "integrity": "sha512-HiUX/+K2YpkpJ+SzBffkM/AQ2YE03S0U1kjTLVpoJdhZMOWy8qvXVN9JdLqv2QsaQ6MPYQIuNmwD8zOiYUofLQ==", - "dev": true + "dev": true, + "requires": {} }, "acorn-walk": { "version": "6.2.0", @@ -329,14 +5681,16 @@ "dev": true }, "array-includes": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.1.1.tgz", - "integrity": "sha512-c2VXaCHl7zPsvpkFsw4nxvFie4fh1ur9bpcgsVkIjqn0H/Xwdg+7fv3n2r/isyS8EBj5b06M9kHyZuIr4El6WQ==", + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.1.4.tgz", + "integrity": "sha512-ZTNSQkmWumEbiHO2GF4GmWxYVTiQyJy2XOTa15sdQSrvKn7l+180egQMqlrMOUMCyLMD7pmyQe4mMDUT6Behrw==", "dev": true, "requires": { + "call-bind": "^1.0.2", "define-properties": "^1.1.3", - "es-abstract": "^1.17.0", - "is-string": "^1.0.5" + "es-abstract": "^1.19.1", + "get-intrinsic": "^1.1.1", + "is-string": "^1.0.7" } }, "array-union": { @@ -346,13 +5700,14 @@ "dev": true }, "array.prototype.flat": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.2.3.tgz", - "integrity": "sha512-gBlRZV0VSmfPIeWfuuy56XZMvbVfbEUnOXUvt3F/eUUUSyzlgLxhEX4YAEpxNAogRGehPSnfXyPtYyKAhkzQhQ==", + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.2.5.tgz", + "integrity": "sha512-KaYU+S+ndVqyUnignHftkwc58o3uVU1jzczILJ1tN2YaIZpFIKBiP/x/j97E5MVPsaCloPbqWLB/8qCTVvT2qg==", "dev": true, "requires": { + "call-bind": "^1.0.2", "define-properties": "^1.1.3", - "es-abstract": "^1.17.0-next.1" + "es-abstract": "^1.19.0" } }, "arrify": { @@ -490,13 +5845,13 @@ "dev": true }, "call-bind": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.0.tgz", - "integrity": "sha512-AEXsYIyyDY3MCzbwdhzG3Jx1R0J2wetQyUynn6dYHAO+bg8l1k7jwZtRv4ryryFs7EP+NDlikJlVe59jr0cM2w==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", "dev": true, "requires": { "function-bind": "^1.1.1", - "get-intrinsic": "^1.0.0" + "get-intrinsic": "^1.0.2" } }, "callsites": { @@ -541,9 +5896,9 @@ "dev": true }, "chrome-devtools-frontend": { - "version": "1.0.827632", - "resolved": "https://registry.npmjs.org/chrome-devtools-frontend/-/chrome-devtools-frontend-1.0.827632.tgz", - "integrity": "sha512-8qtv6zxlaoZhmtPFrR/dAyXjCKCSXL8s7SeAaHyICpd7p6mfXS+y+b14E66LLbOIEMwpidmy3xUJPePx8NuWHg==", + "version": "1.0.944903", + "resolved": "https://registry.npmjs.org/chrome-devtools-frontend/-/chrome-devtools-frontend-1.0.944903.tgz", + "integrity": "sha512-0AX3fSoR7l33Kxb4+U1QFbH4SkSKv4mhawDeex0CmbsmsdtfybI7y4NvN4Fen/+w5j/g4m6t79STQ8pjI+NrQA==", "dev": true }, "cli-cursor": { @@ -633,12 +5988,6 @@ "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", "dev": true }, - "contains-path": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/contains-path/-/contains-path-0.1.0.tgz", - "integrity": "sha1-/ozxhP9mcLa67wGp1IYaXL7EEgo=", - "dev": true - }, "core-js": { "version": "2.6.10", "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.10.tgz", @@ -718,9 +6067,9 @@ } }, "debug": { - "version": "3.2.6", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", - "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", "dev": true, "requires": { "ms": "^2.1.1" @@ -841,22 +6190,31 @@ } }, "es-abstract": { - "version": "1.17.7", - "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.7.tgz", - "integrity": "sha512-VBl/gnfcJ7OercKA9MVaegWsBHFjV492syMudcnQZvt/Dw8ezpcOHYZXa/J96O8vx+g4x65YKhxOwDUh63aS5g==", + "version": "1.19.1", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.19.1.tgz", + "integrity": "sha512-2vJ6tjA/UfqLm2MPs7jxVybLoB8i1t1Jd9R3kISld20sIxPcTbLuggQOUxeWeAvIUkduv/CfMjuh4WmiXr2v9w==", "dev": true, "requires": { + "call-bind": "^1.0.2", "es-to-primitive": "^1.2.1", "function-bind": "^1.1.1", + "get-intrinsic": "^1.1.1", + "get-symbol-description": "^1.0.0", "has": "^1.0.3", - "has-symbols": "^1.0.1", - "is-callable": "^1.2.2", - "is-regex": "^1.1.1", - "object-inspect": "^1.8.0", + "has-symbols": "^1.0.2", + "internal-slot": "^1.0.3", + "is-callable": "^1.2.4", + "is-negative-zero": "^2.0.1", + "is-regex": "^1.1.4", + "is-shared-array-buffer": "^1.0.1", + "is-string": "^1.0.7", + "is-weakref": "^1.0.1", + "object-inspect": "^1.11.0", "object-keys": "^1.1.1", - "object.assign": "^4.1.1", - "string.prototype.trimend": "^1.0.1", - "string.prototype.trimstart": "^1.0.1" + "object.assign": "^4.1.2", + "string.prototype.trimend": "^1.0.4", + "string.prototype.trimstart": "^1.0.4", + "unbox-primitive": "^1.0.1" } }, "es-to-primitive": { @@ -1236,70 +6594,28 @@ "version": "14.1.1", "resolved": "https://registry.npmjs.org/eslint-config-standard/-/eslint-config-standard-14.1.1.tgz", "integrity": "sha512-Z9B+VR+JIXRxz21udPTL9HpFMyoMUEeX1G251EQ6e05WD9aPVtVBn09XUmZ259wCMlCDmYDSZG62Hhm+ZTJcUg==", - "dev": true + "dev": true, + "requires": {} }, "eslint-import-resolver-node": { - "version": "0.3.4", - "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.4.tgz", - "integrity": "sha512-ogtf+5AB/O+nM6DIeBUNr2fuT7ot9Qg/1harBfBtaP13ekEWFQEEMP94BCB7zaNW3gyY+8SHYF00rnqYwXKWOA==", + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.6.tgz", + "integrity": "sha512-0En0w03NRVMn9Uiyn8YRPDKvWjxCWkslUEhGNTdGx15RvPJYQ+lbOlqrlNI2vEAs4pDYK4f/HN2TbDmk5TP0iw==", "dev": true, "requires": { - "debug": "^2.6.9", - "resolve": "^1.13.1" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "dev": true, - "requires": { - "ms": "2.0.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", - "dev": true - }, - "resolve": { - "version": "1.19.0", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.19.0.tgz", - "integrity": "sha512-rArEXAgsBG4UgRGcynxWIWKFvh/XZCcS8UJdHhwy91zwAvCZIbcs+vAbflgBnNjYMs/i/i+/Ux6IZhML1yPvxg==", - "dev": true, - "requires": { - "is-core-module": "^2.1.0", - "path-parse": "^1.0.6" - } - } + "debug": "^3.2.7", + "resolve": "^1.20.0" } }, "eslint-module-utils": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.6.0.tgz", - "integrity": "sha512-6j9xxegbqe8/kZY8cYpcp0xhbK0EgJlg3g9mib3/miLaExuuwc3n5UEfSnU6hWMbT0FAYVvDbL9RrRgpUeQIvA==", + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.7.1.tgz", + "integrity": "sha512-fjoetBXQZq2tSTWZ9yWVl2KuFrTZZH3V+9iD1V1RfpDgxzJR+mPd/KZmMiA8gbPqdBzpNiEHOuT7IYEWxrH0zQ==", "dev": true, "requires": { - "debug": "^2.6.9", + "debug": "^3.2.7", + "find-up": "^2.1.0", "pkg-dir": "^2.0.0" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "dev": true, - "requires": { - "ms": "2.0.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", - "dev": true - } } }, "eslint-plugin-es": { @@ -1321,24 +6637,24 @@ } }, "eslint-plugin-import": { - "version": "2.22.1", - "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.22.1.tgz", - "integrity": "sha512-8K7JjINHOpH64ozkAhpT3sd+FswIZTfMZTjdx052pnWrgRCVfp8op9tbjpAk3DdUeI/Ba4C8OjdC0r90erHEOw==", + "version": "2.25.3", + "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.25.3.tgz", + "integrity": "sha512-RzAVbby+72IB3iOEL8clzPLzL3wpDrlwjsTBAQXgyp5SeTqqY+0bFubwuo+y/HLhNZcXV4XqTBO4LGsfyHIDXg==", "dev": true, "requires": { - "array-includes": "^3.1.1", - "array.prototype.flat": "^1.2.3", - "contains-path": "^0.1.0", + "array-includes": "^3.1.4", + "array.prototype.flat": "^1.2.5", "debug": "^2.6.9", - "doctrine": "1.5.0", - "eslint-import-resolver-node": "^0.3.4", - "eslint-module-utils": "^2.6.0", + "doctrine": "^2.1.0", + "eslint-import-resolver-node": "^0.3.6", + "eslint-module-utils": "^2.7.1", "has": "^1.0.3", + "is-core-module": "^2.8.0", + "is-glob": "^4.0.3", "minimatch": "^3.0.4", - "object.values": "^1.1.1", - "read-pkg-up": "^2.0.0", - "resolve": "^1.17.0", - "tsconfig-paths": "^3.9.0" + "object.values": "^1.1.5", + "resolve": "^1.20.0", + "tsconfig-paths": "^3.11.0" }, "dependencies": { "debug": { @@ -1351,13 +6667,12 @@ } }, "doctrine": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-1.5.0.tgz", - "integrity": "sha1-N53Ocw9hZvds76TmcHoVmwLFpvo=", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz", + "integrity": "sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==", "dev": true, "requires": { - "esutils": "^2.0.2", - "isarray": "^1.0.0" + "esutils": "^2.0.2" } }, "ms": { @@ -1365,16 +6680,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", "dev": true - }, - "resolve": { - "version": "1.19.0", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.19.0.tgz", - "integrity": "sha512-rArEXAgsBG4UgRGcynxWIWKFvh/XZCcS8UJdHhwy91zwAvCZIbcs+vAbflgBnNjYMs/i/i+/Ux6IZhML1yPvxg==", - "dev": true, - "requires": { - "is-core-module": "^2.1.0", - "path-parse": "^1.0.6" - } } } }, @@ -1410,7 +6715,8 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/eslint-plugin-standard/-/eslint-plugin-standard-4.1.0.tgz", "integrity": "sha512-ZL7+QRixjTR6/528YNGyDotyffm5OQst/sGxKDwGb9Uqs4In5Egi4+jbobhqJoyoCM6/7v/1A5fhQ7ScMtDjaQ==", - "dev": true + "dev": true, + "requires": {} }, "eslint-scope": { "version": "5.1.0", @@ -1675,9 +6981,9 @@ "dev": true }, "get-intrinsic": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.0.1.tgz", - "integrity": "sha512-ZnWP+AmS1VUaLgTRy47+zKtjTxz+0xMpx3I52i+aalBK1QP19ggLF3Db89KJX7kjfOfP2eoa01qc++GwPgufPg==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz", + "integrity": "sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==", "dev": true, "requires": { "function-bind": "^1.1.1", @@ -1691,6 +6997,16 @@ "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=", "dev": true }, + "get-symbol-description": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.0.0.tgz", + "integrity": "sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==", + "dev": true, + "requires": { + "call-bind": "^1.0.2", + "get-intrinsic": "^1.1.1" + } + }, "getpass": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", @@ -1715,9 +7031,9 @@ } }, "glob-parent": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz", - "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==", + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", "dev": true, "requires": { "is-glob": "^4.0.1" @@ -1859,6 +7175,12 @@ "ansi-regex": "^2.0.0" } }, + "has-bigints": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.1.tgz", + "integrity": "sha512-LSBS2LjbNBTf6287JEbEzvJgftkF5qFkmCo9hDRpAzKhUOlJ+hx8dd4USs00SgsUNwc4617J9ki5YtEClM2ffA==", + "dev": true + }, "has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", @@ -1866,11 +7188,20 @@ "dev": true }, "has-symbols": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz", - "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz", + "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==", "dev": true }, + "has-tostringtag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz", + "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", + "dev": true, + "requires": { + "has-symbols": "^1.0.2" + } + }, "he": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", @@ -1878,9 +7209,9 @@ "dev": true }, "hosted-git-info": { - "version": "2.8.5", - "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.5.tgz", - "integrity": "sha512-kssjab8CvdXfcXMXVcvsXum4Hwdq9XGtRD3TteMEvEbq0LXyiNQr6AprqKqfeaDXze7SxWvRxdpwE6ku7ikLkg==", + "version": "2.8.9", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz", + "integrity": "sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==", "dev": true }, "html-encoding-sniffer": { @@ -2027,6 +7358,17 @@ } } }, + "internal-slot": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.3.tgz", + "integrity": "sha512-O0DB1JC/sPyZl7cIo78n5dR7eUSwwpYPiXRhTzNxZVAMUuB8vlnRFyLxdrVToks6XPLVnFfbzaVd5WLjhgg+vA==", + "dev": true, + "requires": { + "get-intrinsic": "^1.1.0", + "has": "^1.0.3", + "side-channel": "^1.0.4" + } + }, "invert-kv": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz", @@ -2045,6 +7387,25 @@ "integrity": "sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0=", "dev": true }, + "is-bigint": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz", + "integrity": "sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==", + "dev": true, + "requires": { + "has-bigints": "^1.0.1" + } + }, + "is-boolean-object": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz", + "integrity": "sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==", + "dev": true, + "requires": { + "call-bind": "^1.0.2", + "has-tostringtag": "^1.0.0" + } + }, "is-buffer": { "version": "1.1.6", "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", @@ -2052,25 +7413,28 @@ "dev": true }, "is-callable": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.2.tgz", - "integrity": "sha512-dnMqspv5nU3LoewK2N/y7KLtxtakvTuaCsU9FU50/QDmdbHNy/4/JuRtMHqRU22o3q+W89YQndQEeCVwK+3qrA==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.4.tgz", + "integrity": "sha512-nsuwtxZfMX67Oryl9LCQ+upnC0Z0BgpwntpS89m1H/TLF0zNfzfLMV/9Wa/6MZsj0acpEjAO0KF1xT6ZdLl95w==", "dev": true }, "is-core-module": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.1.0.tgz", - "integrity": "sha512-YcV7BgVMRFRua2FqQzKtTDMz8iCuLEyGKjr70q8Zm1yy2qKcurbFEd79PAdHV77oL3NrAaOVQIbMmiHQCHB7ZA==", + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.8.0.tgz", + "integrity": "sha512-vd15qHsaqrRL7dtH6QNuy0ndJmRDrS9HAM1CAiSifNUFv4x1a0CCVsj18hJ1mShxIG6T2i1sO78MkP56r0nYRw==", "dev": true, "requires": { "has": "^1.0.3" } }, "is-date-object": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.2.tgz", - "integrity": "sha512-USlDT524woQ08aoZFzh3/Z6ch9Y/EWXEHQ/AaRN0SkKq4t2Jw2R2339tSXmwuVoY7LLlBCbOIlx2myP/L5zk0g==", - "dev": true + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.5.tgz", + "integrity": "sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==", + "dev": true, + "requires": { + "has-tostringtag": "^1.0.0" + } }, "is-extglob": { "version": "2.1.1", @@ -2088,20 +7452,29 @@ } }, "is-glob": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.1.tgz", - "integrity": "sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==", + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", "dev": true, "requires": { "is-extglob": "^2.1.1" } }, "is-negative-zero": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.0.tgz", - "integrity": "sha1-lVOxIbD6wohp2p7UWeIMdUN4hGE=", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.1.tgz", + "integrity": "sha512-2z6JzQvZRa9A2Y7xC6dQQm4FSTSTNWjKIYYTt4246eMTJmIo0Q+ZyOsU66X8lxK1AbB92dFeglPLrhwpeRKO6w==", "dev": true }, + "is-number-object": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.6.tgz", + "integrity": "sha512-bEVOqiRcvo3zO1+G2lVMy+gkkEm9Yh7cDMRusKKu5ZJKPUYSJwICTKZrNKHA2EbSP0Tu0+6B/emsYNHZyn6K8g==", + "dev": true, + "requires": { + "has-tostringtag": "^1.0.0" + } + }, "is-promise": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.1.0.tgz", @@ -2109,14 +7482,21 @@ "dev": true }, "is-regex": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.1.tgz", - "integrity": "sha512-1+QkEcxiLlB7VEyFtyBg94e08OAsvq7FUBgApTq/w2ymCLyKJgDPsybBENVtA7XCQEgEXxKPonG+mvYRxh/LIg==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz", + "integrity": "sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==", "dev": true, "requires": { - "has-symbols": "^1.0.1" + "call-bind": "^1.0.2", + "has-tostringtag": "^1.0.0" } }, + "is-shared-array-buffer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.1.tgz", + "integrity": "sha512-IU0NmyknYZN0rChcKhRO1X8LYz5Isj/Fsqh8NJOSf+N/hCOTwy29F32Ik7a+QszE63IdvmwdTPDd6cZ5pg4cwA==", + "dev": true + }, "is-stream": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", @@ -2124,18 +7504,21 @@ "dev": true }, "is-string": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.5.tgz", - "integrity": "sha512-buY6VNRjhQMiF1qWDouloZlQbRhDPCebwxSjxMjxgemYT46YMd2NR0/H+fBhEfWX4A/w9TBJ+ol+okqJKFE6vQ==", - "dev": true + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.7.tgz", + "integrity": "sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==", + "dev": true, + "requires": { + "has-tostringtag": "^1.0.0" + } }, "is-symbol": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.3.tgz", - "integrity": "sha512-OwijhaRSgqvhm/0ZdAcXNZt9lYdKFpcRDT5ULUuYXPoT794UNOdU+gpT6Rzo7b4V2HUl/op6GqY894AZwv9faQ==", + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.4.tgz", + "integrity": "sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==", "dev": true, "requires": { - "has-symbols": "^1.0.1" + "has-symbols": "^1.0.2" } }, "is-typedarray": { @@ -2144,11 +7527,14 @@ "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=", "dev": true }, - "isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", - "dev": true + "is-weakref": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-weakref/-/is-weakref-1.0.1.tgz", + "integrity": "sha512-b2jKc2pQZjaeFYWEf7ScFj+Be1I+PXmlu572Q8coTXZ+LD/QQZ7ShPMst8h16riVgyXTQwUsFEl74mDvc/3MHQ==", + "dev": true, + "requires": { + "call-bind": "^1.0.0" + } }, "isexe": { "version": "2.0.0", @@ -2233,7 +7619,8 @@ "version": "5.2.0", "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.2.0.tgz", "integrity": "sha512-HiUX/+K2YpkpJ+SzBffkM/AQ2YE03S0U1kjTLVpoJdhZMOWy8qvXVN9JdLqv2QsaQ6MPYQIuNmwD8zOiYUofLQ==", - "dev": true + "dev": true, + "requires": {} }, "ansi-styles": { "version": "4.2.1", @@ -2624,9 +8011,9 @@ } }, "lodash": { - "version": "4.17.19", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz", - "integrity": "sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ==", + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", "dev": true }, "lodash.sortby": { @@ -2814,9 +8201,9 @@ "dev": true }, "object-inspect": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.8.0.tgz", - "integrity": "sha512-jLdtEOB112fORuypAyl/50VRVIBIdVQOSUUGQHzJ4xBSbit81zRarz7GThkEFZy1RceYrWYcPcBFPQwHyAc1gA==", + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.11.0.tgz", + "integrity": "sha512-jp7ikS6Sd3GxQfZJPyH3cjcbJF6GZPClgdV+EFygjFLQ5FmW/dRUnTd9PQ9k0JhoNDabWFbpF1yCdSWCC6gexg==", "dev": true }, "object-keys": { @@ -2838,15 +8225,14 @@ } }, "object.values": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.1.1.tgz", - "integrity": "sha512-WTa54g2K8iu0kmS/us18jEmdv1a4Wi//BZ/DTVYEcH0XhLM5NYdpDHja3gt57VrZLcNAO2WGA+KpWsDBaHt6eA==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.1.5.tgz", + "integrity": "sha512-QUZRW0ilQ3PnPpbNtgdNV1PDbEqLIiSFB3l+EnGtBQ/8SUTLj1PZwtQHABZtLgwpJZTSZhuGLOGk57Drx2IvYg==", "dev": true, "requires": { + "call-bind": "^1.0.2", "define-properties": "^1.1.3", - "es-abstract": "^1.17.0-next.1", - "function-bind": "^1.1.1", - "has": "^1.0.3" + "es-abstract": "^1.19.1" } }, "once": { @@ -3034,9 +8420,9 @@ "dev": true }, "path-parse": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", - "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==", + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", "dev": true }, "path-type": { @@ -3267,11 +8653,12 @@ "dev": true }, "resolve": { - "version": "1.12.0", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.12.0.tgz", - "integrity": "sha512-B/dOmuoAik5bKcD6s6nXDCjzUKnaDvdkRyAk6rsmsKLipWj4797iothd7jmmUhWTfinVMU+wc56rYKsit2Qy4w==", + "version": "1.20.0", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.20.0.tgz", + "integrity": "sha512-wENBPt4ySzg4ybFQW2TT1zMQucPK95HSh/nq2CFTZVOGut2+pQvSsgtda4d26YrYcr067wjbmzOG8byDPBX63A==", "dev": true, "requires": { + "is-core-module": "^2.2.0", "path-parse": "^1.0.6" } }, @@ -3450,6 +8837,17 @@ "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=", "dev": true }, + "side-channel": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", + "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "dev": true, + "requires": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + } + }, "signal-exit": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", @@ -3598,67 +8996,23 @@ } }, "string.prototype.trimend": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.2.tgz", - "integrity": "sha512-8oAG/hi14Z4nOVP0z6mdiVZ/wqjDtWSLygMigTzAb+7aPEDTleeFf+WrF+alzecxIRkckkJVn+dTlwzJXORATw==", + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.4.tgz", + "integrity": "sha512-y9xCjw1P23Awk8EvTpcyL2NIr1j7wJ39f+k6lvRnSMz+mz9CGz9NYPelDk42kOz6+ql8xjfK8oYzy3jAP5QU5A==", "dev": true, "requires": { - "define-properties": "^1.1.3", - "es-abstract": "^1.18.0-next.1" - }, - "dependencies": { - "es-abstract": { - "version": "1.18.0-next.1", - "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.18.0-next.1.tgz", - "integrity": "sha512-I4UGspA0wpZXWENrdA0uHbnhte683t3qT/1VFH9aX2dA5PPSf6QW5HHXf5HImaqPmjXaVeVk4RGWnaylmV7uAA==", - "dev": true, - "requires": { - "es-to-primitive": "^1.2.1", - "function-bind": "^1.1.1", - "has": "^1.0.3", - "has-symbols": "^1.0.1", - "is-callable": "^1.2.2", - "is-negative-zero": "^2.0.0", - "is-regex": "^1.1.1", - "object-inspect": "^1.8.0", - "object-keys": "^1.1.1", - "object.assign": "^4.1.1", - "string.prototype.trimend": "^1.0.1", - "string.prototype.trimstart": "^1.0.1" - } - } + "call-bind": "^1.0.2", + "define-properties": "^1.1.3" } }, "string.prototype.trimstart": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.2.tgz", - "integrity": "sha512-7F6CdBTl5zyu30BJFdzSTlSlLPwODC23Od+iLoVH8X6+3fvDPPuBVVj9iaB1GOsSTSIgVfsfm27R2FGrAPznWg==", + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.4.tgz", + "integrity": "sha512-jh6e984OBfvxS50tdY2nRZnoC5/mLFKOREQfw8t5yytkoUsJRNxvI/E39qu1sD0OtWI3OC0XgKSmcWwziwYuZw==", "dev": true, "requires": { - "define-properties": "^1.1.3", - "es-abstract": "^1.18.0-next.1" - }, - "dependencies": { - "es-abstract": { - "version": "1.18.0-next.1", - "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.18.0-next.1.tgz", - "integrity": "sha512-I4UGspA0wpZXWENrdA0uHbnhte683t3qT/1VFH9aX2dA5PPSf6QW5HHXf5HImaqPmjXaVeVk4RGWnaylmV7uAA==", - "dev": true, - "requires": { - "es-to-primitive": "^1.2.1", - "function-bind": "^1.1.1", - "has": "^1.0.3", - "has-symbols": "^1.0.1", - "is-callable": "^1.2.2", - "is-negative-zero": "^2.0.0", - "is-regex": "^1.1.1", - "object-inspect": "^1.8.0", - "object-keys": "^1.1.1", - "object.assign": "^4.1.1", - "string.prototype.trimend": "^1.0.1", - "string.prototype.trimstart": "^1.0.1" - } - } + "call-bind": "^1.0.2", + "define-properties": "^1.1.3" } }, "strip-ansi": { @@ -3813,9 +9167,9 @@ } }, "tsconfig-paths": { - "version": "3.9.0", - "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.9.0.tgz", - "integrity": "sha512-dRcuzokWhajtZWkQsDVKbWyY+jgcLC5sqJhg2PSgf4ZkH2aHPvaOY8YWGhmjb68b5qqTfasSsDO9k7RUiEmZAw==", + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.12.0.tgz", + "integrity": "sha512-e5adrnOYT6zqVnWqZu7i/BQ3BnhzvGbjEjejFXO20lKIKpwTaupkCPgEfv4GZK1IBciJUEhYs3J3p75FdaTFVg==", "dev": true, "requires": { "@types/json5": "^0.0.29", @@ -3875,6 +9229,18 @@ "integrity": "sha512-BLbiRkiBzAwsjut4x/dsibSTB6yWpwT5qWmC2OfuCg3GgVQCSgMs4vEctYPhsaGtd0AeuuHMkjZ2h2WG8MSzRw==", "dev": true }, + "unbox-primitive": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.1.tgz", + "integrity": "sha512-tZU/3NqK3dA5gpE1KtyiJUrEB0lxnGkMFHptJ7q6ewdZ8s12QrODwNbhIJStmJkd1QDXa1NRA8aF2A1zk/Ypyw==", + "dev": true, + "requires": { + "function-bind": "^1.1.1", + "has-bigints": "^1.0.1", + "has-symbols": "^1.0.2", + "which-boxed-primitive": "^1.0.2" + } + }, "union": { "version": "0.5.0", "resolved": "https://registry.npmjs.org/union/-/union-0.5.0.tgz", @@ -3999,6 +9365,19 @@ "isexe": "^2.0.0" } }, + "which-boxed-primitive": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz", + "integrity": "sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==", + "dev": true, + "requires": { + "is-bigint": "^1.0.1", + "is-boolean-object": "^1.1.0", + "is-number-object": "^1.0.4", + "is-string": "^1.0.5", + "is-symbol": "^1.0.3" + } + }, "which-module": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", @@ -4056,10 +9435,11 @@ } }, "ws": { - "version": "7.4.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.0.tgz", - "integrity": "sha512-kyFwXuV/5ymf+IXhS6f0+eAFvydbaBW3zjpT6hUdAh/hbVjTIB5EHBGi0bPoCLSK2wcuz3BrEkB9LrYv1Nm4NQ==", - "dev": true + "version": "7.4.6", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.6.tgz", + "integrity": "sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A==", + "dev": true, + "requires": {} }, "xml-name-validator": { "version": "3.0.0", @@ -4074,9 +9454,9 @@ "dev": true }, "y18n": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-3.2.1.tgz", - "integrity": "sha1-bRX7qITAhnnA136I53WegR4H+kE=", + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-3.2.2.tgz", + "integrity": "sha512-uGZHXkHnhF0XeeAPgnKfPv1bgKAYyVvmNL1xlKsPYZPaIHxGti2hHqvOCQv71XMsLxu1QjergkqogUnms5D3YQ==", "dev": true }, "yallist": { diff --git a/dashboard/package.json b/dashboard/package.json index 236e2d157d..295ebd2963 100644 --- a/dashboard/package.json +++ b/dashboard/package.json @@ -30,10 +30,10 @@ "@types/js-cookie": "^2.2.6", "@typescript-eslint/eslint-plugin": "^2.34.0", "@typescript-eslint/parser": "^2.34.0", - "chrome-devtools-frontend": "^1.0.827632", + "chrome-devtools-frontend": "^1.0.944903", "eslint": "^6.8.0", "eslint-config-standard": "^14.1.1", - "eslint-plugin-import": "^2.22.1", + "eslint-plugin-import": "^2.25.3", "eslint-plugin-node": "^10.0.0", "eslint-plugin-promise": "^4.2.1", "eslint-plugin-standard": "^4.1.0", @@ -50,6 +50,6 @@ "rollup-plugin-typescript": "^1.0.1", "ts-node": "^7.0.1", "typescript": "^3.9.7", - "ws": "^7.4.0" + "ws": "^7.4.6" } } diff --git a/dashboard/src/core/devtools.ts b/dashboard/src/core/devtools.ts index 906c82f7da..77fc789451 100644 --- a/dashboard/src/core/devtools.ts +++ b/dashboard/src/core/devtools.ts @@ -7,30 +7,40 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. */ -import path = require('path') -import fs = require('fs') +import path = require('path'); +import fs = require('fs'); const ncp = require('ncp').ncp ncp.limit = 16 function setUpDevTools () { - const destinationFolderPath = path.join(path.dirname(__dirname), 'public', 'dashboard', 'devtools') + const destinationFolderPath = path.join( + path.dirname(__dirname), + 'public', + 'dashboard', + 'devtools' + ) const destinationFolderExists = fs.existsSync(destinationFolderPath) if (!destinationFolderExists) { - console.error(destinationFolderPath + ' folder doesn\'t exist, make sure you are in the right directory.') + console.error( + destinationFolderPath + + " folder doesn't exist, make sure you are in the right directory." + ) process.exit(1) } - const chromeDevTools = path.dirname(require.resolve('chrome-devtools-frontend/front_end/inspector.html')) + const chromeDevTools = path.dirname( + require.resolve('chrome-devtools-frontend/front_end/inspector.json') + ) console.log('Destination folder: ' + destinationFolderPath) ncp(chromeDevTools, destinationFolderPath, (err: any) => { if (err) { return console.error(err) } - console.log('Done!!!') }) + console.log('Done!!!') } setUpDevTools() diff --git a/dashboard/src/core/plugins/inspect_traffic.html b/dashboard/src/core/plugins/inspect_traffic.html index 3889c73b78..afd29d2551 100644 --- a/dashboard/src/core/plugins/inspect_traffic.html +++ b/dashboard/src/core/plugins/inspect_traffic.html @@ -7,12 +7,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. --> - + - - - - - - + + + + + diff --git a/dashboard/src/core/plugins/inspect_traffic.js b/dashboard/src/core/plugins/inspect_traffic.js index 3b60c5c0f0..3661cd6786 100644 --- a/dashboard/src/core/plugins/inspect_traffic.js +++ b/dashboard/src/core/plugins/inspect_traffic.js @@ -7,4 +7,7 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. */ -Root.Runtime.startApplication('inspect_traffic'); +import './devtools_app.js'; +import * as Startup from './startup/startup.js'; + +Startup.RuntimeInstantiator.startApplication('inspect_traffic'); diff --git a/dashboard/src/core/plugins/inspect_traffic.json b/dashboard/src/core/plugins/inspect_traffic.json index 31747bbbcd..fc9912b2ce 100644 --- a/dashboard/src/core/plugins/inspect_traffic.json +++ b/dashboard/src/core/plugins/inspect_traffic.json @@ -1,13 +1,21 @@ { - "modules" : [ - { "name": "inspector_main", "type": "autostart" }, - { "name": "emulation" }, - { "name": "mobile_throttling" }, - { "name": "cookie_table" }, - { "name": "har_importer" }, - { "name": "network" } - ], - "extends": "shell", - "has_html": true - } - \ No newline at end of file + "modules" : [ + { "name": "platform", "type": "autostart" }, + { "name": "main", "type": "autostart" }, + { "name": "components", "type": "autostart" }, + { "name": "ui", "type": "autostart" }, + { "name": "sdk", "type": "autostart" }, + { "name": "host", "type": "autostart" }, + { "name": "common", "type": "autostart" }, + { "name": "emulation", "type": "autostart" }, + { "name": "workspace", "type": "autostart" }, + { "name": "bindings", "type": "autostart" }, + { "name": "extensions", "type": "autostart" }, + { "name": "ui_lazy" }, + { "name": "components_lazy" }, + { "name": "network" }, + { "name": "source_frame" } + ], + "extends": "devtools_app", + "has_html": true +} diff --git a/dashboard/src/manifest.json b/dashboard/src/manifest.json index 75ed311dca..ef28d038cb 100644 --- a/dashboard/src/manifest.json +++ b/dashboard/src/manifest.json @@ -1,34 +1,34 @@ { - "name": "Proxy.py Dashboard", - "short_name": "Proxy.py Dashboard", - "start_url": "/dashboard/", - "display": "standalone", - "background_color": "#3E4EB8", - "theme_color": "#2F3BA2", - "icons": [{ - "src": "/dashboard/images/icons/icon-128x128.png", - "sizes": "128x128", - "type": "image/png" - }, { - "src": "/dashboard/images/icons/icon-144x144.png", - "sizes": "144x144", - "type": "image/png" - }, { - "src": "/dashboard/images/icons/icon-152x152.png", - "sizes": "152x152", - "type": "image/png" - }, { - "src": "/dashboard/images/icons/icon-192x192.png", - "sizes": "192x192", - "type": "image/png" - }, { - "src": "/dashboard/images/icons/icon-256x256.png", - "sizes": "256x256", - "type": "image/png" - }, { - "src": "/dashboard/images/icons/icon-512x512.png", - "sizes": "512x512", - "type": "image/png" - } - ] + "name": "Proxy.py Dashboard", + "short_name": "Proxy.py Dashboard", + "start_url": "/dashboard/", + "display": "standalone", + "background_color": "#3E4EB8", + "theme_color": "#2F3BA2", + "icons": [{ + "src": "/dashboard/images/icons/icon-128x128.png", + "sizes": "128x128", + "type": "image/png" + }, { + "src": "/dashboard/images/icons/icon-144x144.png", + "sizes": "144x144", + "type": "image/png" + }, { + "src": "/dashboard/images/icons/icon-152x152.png", + "sizes": "152x152", + "type": "image/png" + }, { + "src": "/dashboard/images/icons/icon-192x192.png", + "sizes": "192x192", + "type": "image/png" + }, { + "src": "/dashboard/images/icons/icon-256x256.png", + "sizes": "256x256", + "type": "image/png" + }, { + "src": "/dashboard/images/icons/icon-512x512.png", + "sizes": "512x512", + "type": "image/png" + } + ] } diff --git a/docs/_ext/spelling_stub_ext.py b/docs/_ext/spelling_stub_ext.py new file mode 100644 index 0000000000..c8989dc149 --- /dev/null +++ b/docs/_ext/spelling_stub_ext.py @@ -0,0 +1,27 @@ +"""Sphinx extension for making the spelling directive noop.""" + +from typing import List + +from sphinx.application import Sphinx +from sphinx.util.docutils import SphinxDirective +from sphinx.util.nodes import nodes + + +class SpellingNoOpDirective(SphinxDirective): + """Definition of the stub spelling directive.""" + + has_content = True + + def run(self) -> List[nodes.Node]: + """Generate nothing in place of the directive.""" + return [] + + +def setup(app: Sphinx) -> None: + """Initialize the extension.""" + app.add_directive('spelling', SpellingNoOpDirective) + + return { + 'parallel_read_safe': True, + 'version': 'builtin', + } diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000000..7391c7911f --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,287 @@ +# pylint: disable=invalid-name +# Requires Python 3.6+ +# Ref: https://www.sphinx-doc.org/en/master/usage/configuration.html +"""Configuration for the Sphinx documentation generator.""" + +import sys +from functools import partial +from pathlib import Path + +from setuptools_scm import get_version + + +# -- Path setup -------------------------------------------------------------- + +PROJECT_ROOT_DIR = Path(__file__).parents[1].resolve() # pylint: disable=no-member +get_scm_version = partial(get_version, root=PROJECT_ROOT_DIR) + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. + + +sys.path.insert(0, str(PROJECT_ROOT_DIR)) + +# Make in-tree extension importable in non-tox setups/envs, like RTD. +# Refs: +# https://github.com/readthedocs/readthedocs.org/issues/6311 +# https://github.com/readthedocs/readthedocs.org/issues/7182 +sys.path.insert(0, str((Path(__file__).parent / '_ext').resolve())) + +# -- Project information ----------------------------------------------------- + +github_url = 'https://github.com' +github_repo_org = 'abhinavsingh' +github_repo_name = 'proxy.py' +github_repo_slug = f'{github_repo_org}/{github_repo_name}' +github_repo_url = f'{github_url}/{github_repo_slug}' +github_sponsors_url = f'{github_url}/sponsors' + +project = github_repo_name.title() +author = f'{project} project contributors' +copyright = author # pylint: disable=redefined-builtin + +# The short X.Y version +version = '.'.join( + get_scm_version( + local_scheme='no-local-version', + ).split('.')[:3], +) + +# The full version, including alpha/beta/rc tags +release = get_scm_version() + +rst_epilog = f""" +.. |project| replace:: {project} +""" + + +# -- General configuration --------------------------------------------------- + + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# today = '' +# Else, today_fmt is used as the format for a strftime call. +today_fmt = '%B %d, %Y' + +# The reST default role (used for this markup: `text`) to use for all +# documents. +# Ref: python-attrs/attrs#571 +default_role = 'any' + +# If true, '()' will be appended to :func: etc. cross-reference text. +add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +show_authors = True + +# The name of the Pygments (syntax highlighting) style to use. +# pygments_style = 'sphinx' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + # stdlib-party extensions: + 'sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.extlinks', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.viewcode', + + # Third-party extensions: + 'myst_parser', # extended markdown; https://pypi.org/project/myst-parser/ + 'sphinxcontrib.apidoc', +] + +# Conditional third-party extensions: +try: + import sphinxcontrib.spelling as _sphinxcontrib_spelling +except ImportError: + extensions.append('spelling_stub_ext') +else: + del _sphinxcontrib_spelling + extensions.append('sphinxcontrib.spelling') + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = 'en' + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'furo' + +html_show_sphinx = True + +html_theme_options = { +} + +html_context = { +} + + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +# html_title = f'{project} Documentation' + +# A shorter title for the navigation bar. Default is the same as html_title. +# html_short_title = 'Documentation' + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +html_last_updated_fmt = '%b %d, %Y' + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# html_use_opensearch = f'https://{github_repo_name.replace(".", "")}.readthedocs.io/en/latest/' + +# The master toctree document. +root_doc = master_doc = 'index' # Sphinx 4+ / 3- # noqa: WPS429 + + +# -- Extension configuration ------------------------------------------------- + +# -- Options for intersphinx extension --------------------------------------- + +intersphinx_mapping = { + 'myst': ('https://myst-parser.rtfd.io/en/latest', None), + 'python': ('https://docs.python.org/3', None), + 'python2': ('https://docs.python.org/2', None), +} + +# -- Options for todo extension ---------------------------------------------- + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + +# -- Options for sphinxcontrib.apidoc extension ------------------------------ + +apidoc_excluded_paths = [ + 'plugin/cache/*', + 'testing/*.py', +] +apidoc_extra_args = [ + '--implicit-namespaces', + '--private', # include โ€œ_privateโ€ modules +] +apidoc_module_dir = str(PROJECT_ROOT_DIR / 'proxy') +apidoc_module_first = False +apidoc_output_dir = 'pkg' +apidoc_separate_modules = True +apidoc_toc_file = None + +# -- Options for sphinxcontrib.spelling extension ---------------------------- + +spelling_ignore_acronyms = True +spelling_ignore_importable_modules = True +spelling_ignore_pypi_package_names = True +spelling_ignore_python_builtins = True +spelling_ignore_wiki_words = True +spelling_show_suggestions = True +spelling_word_list_filename = [ + 'spelling_wordlist.txt', +] + +# -- Options for extlinks extension ------------------------------------------ + +extlinks = { + 'issue': (f'{github_repo_url}/issues/%s', '#'), # noqa: WPS323 + 'pr': (f'{github_repo_url}/pull/%s', 'PR #'), # noqa: WPS323 + 'commit': (f'{github_repo_url}/commit/%s', ''), # noqa: WPS323 + 'gh': (f'{github_url}/%s', 'GitHub: '), # noqa: WPS323 + 'user': (f'{github_sponsors_url}/%s', '@'), # noqa: WPS323 +} + +# -- Options for linkcheck builder ------------------------------------------- + +linkcheck_ignore = [ + r'http://localhost:\d+/', # local URLs +] +linkcheck_workers = 25 + +# -- Options for myst_parser extension ------------------------------------------ + +myst_enable_extensions = [ + 'colon_fence', # allow to optionally use ::: instead of ``` + 'deflist', + 'html_admonition', # allow having HTML admonitions + 'html_image', # allow HTML in Markdown + # FIXME: `linkify` turns "Proxy.Py` into a link so it's disabled now + # Ref: https://github.com/executablebooks/MyST-Parser/issues/428#issuecomment-970277208 + # "linkify", # auto-detect URLs @ plain text, needs myst-parser[linkify] + 'replacements', # allows Jinja2-style replacements + 'smartquotes', # use "cursive" quotes + 'substitution', # replace common ASCII shortcuts into their symbols +] +myst_substitutions = { + 'project': project, +} + +# -- Strict mode ------------------------------------------------------------- + +# The reST default role (used for this markup: `text`) to use for all +# documents. +# Ref: python-attrs/attrs#571 +default_role = 'any' + +nitpicky = True +_any_role = 'any' +_py_obj_role = 'py:obj' +_py_class_role = 'py:class' +nitpick_ignore = [ + (_any_role, ''), + (_any_role, '__init__'), + (_any_role, 'Client'), + (_any_role, 'event_queue'), + (_any_role, 'fd_queue'), + (_any_role, 'flag.flags'), + (_any_role, 'flags.work_klass'), + (_any_role, 'flush'), + (_any_role, 'httpx'), + (_any_role, 'HttpParser.state'), + (_any_role, 'HttpProtocolHandler'), + (_any_role, 'multiprocessing.Manager'), + (_any_role, 'proxy.core.base.tcp_upstream.TcpUpstreamConnectionHandler'), + (_any_role, 'work_klass'), + (_py_class_role, '_asyncio.Task'), + (_py_class_role, 'asyncio.events.AbstractEventLoop'), + (_py_class_role, 'CacheStore'), + (_py_class_role, 'HttpParser'), + (_py_class_role, 'HttpProtocolHandlerPlugin'), + (_py_class_role, 'HttpProxyBasePlugin'), + (_py_class_role, 'HttpWebServerBasePlugin'), + (_py_class_role, 'multiprocessing.context.Process'), + (_py_class_role, 'multiprocessing.synchronize.Lock'), + (_py_class_role, 'NonBlockingQueue'), + (_py_class_role, 'paramiko.channel.Channel'), + (_py_class_role, 'proxy.http.parser.parser.T'), + (_py_class_role, 'proxy.plugin.cache.store.base.CacheStore'), + (_py_class_role, 'proxy.core.pool.AcceptorPool'), + (_py_class_role, 'proxy.core.executors.ThreadlessPool'), + (_py_class_role, 'proxy.core.acceptor.threadless.T'), + (_py_class_role, 'queue.Queue[Any]'), + (_py_class_role, 'TcpClientConnection'), + (_py_class_role, 'TcpServerConnection'), + (_py_class_role, 'unittest.case.TestCase'), + (_py_class_role, 'unittest.result.TestResult'), + (_py_class_role, 'UUID'), + (_py_class_role, 'Url'), + (_py_class_role, 'WebsocketFrame'), + (_py_class_role, 'Work'), + (_py_obj_role, 'proxy.core.acceptor.threadless.T'), +] diff --git a/docs/contributing/code_of_conduct.md b/docs/contributing/code_of_conduct.md new file mode 100644 index 0000000000..cc6912b959 --- /dev/null +++ b/docs/contributing/code_of_conduct.md @@ -0,0 +1,2 @@ +```{include} ../../CODE_OF_CONDUCT.md +``` diff --git a/docs/contributing/guidelines.md b/docs/contributing/guidelines.md new file mode 100644 index 0000000000..c12c38338c --- /dev/null +++ b/docs/contributing/guidelines.md @@ -0,0 +1,34 @@ +```{spelling} +de +facto +Pre +reStructuredText +``` + +```{include} ../../CONTRIBUTING.md +``` + +# Contributing docs + +We use [Sphinx] to generate our docs website. You can trigger +the process locally by executing: + +```shell-session +$ tox -e build-docs +``` + +It is also integrated with [Read The Docs] that builds and +publishes each commit to the main branch and generates live +docs previews for each pull request. + +The sources of the [Sphinx] documents use reStructuredText as a +de-facto standard. But in order to make contributing docs more +beginner-friendly, we have integrated [MyST parser] allowing us +to also accept new documents written in an extended version of +Markdown that supports using Sphinx directives and roles. {ref}`Read +the docs ` to learn more on how to use it. + + +[MyST parser]: https://pypi.org/project/myst-parser/ +[Read The Docs]: https://readthedocs.org +[Sphinx]: https://www.sphinx-doc.org diff --git a/docs/contributing/security.md b/docs/contributing/security.md new file mode 100644 index 0000000000..ff2e94493e --- /dev/null +++ b/docs/contributing/security.md @@ -0,0 +1,2 @@ +```{include} ../../SECURITY.md +``` diff --git a/docs/glossary.md b/docs/glossary.md new file mode 100644 index 0000000000..67a1035800 --- /dev/null +++ b/docs/glossary.md @@ -0,0 +1,25 @@ +(_proxy_py_glossary)= +# {{ project }} Glossary + +```{spelling} +DNS +DoH +Py +``` + +```{glossary} +DoH +DNS over HTTP +DNS-over-HTTP + + [DNS over HTTP] +``` + +```{glossary} +DNS + + Domain Name System +``` + +[DNS over HTTP]: +https://datatracker.ietf.org/doc/html/draft-pauly-dprive-oblivious-doh-04.html diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000000..aefb3dc7c6 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,42 @@ +(_proxy_py_index)= +# {{ project }} Documentation + +```{spelling} +acceptor +acceptors +Changelog +decrypt +decrypted +discoverable +DNS +Facebook +http +macOS +Pre +Py +Scalable +``` + +```{include} ../README.md +:end-before: (DO-NOT-REMOVE-docs-badges-START) +``` + +```{include} ../README.md +:start-after: (DO-NOT-REMOVE-docs-badges-END) +``` + +```{toctree} +:hidden: + +Glossary +``` + +```{toctree} +:caption: Contributing +:hidden: + +Code Of Conduct +contributing/guidelines +contributing/security +Private unsupported (dev) API autodoc +``` diff --git a/docs/pkg/.gitignore b/docs/pkg/.gitignore new file mode 100644 index 0000000000..d6b7ef32c8 --- /dev/null +++ b/docs/pkg/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/docs/requirements.in b/docs/requirements.in new file mode 100644 index 0000000000..86acf774af --- /dev/null +++ b/docs/requirements.in @@ -0,0 +1,5 @@ +myst-parser[linkify] >= 0.15.2 +setuptools-scm >= 6.3.2 +Sphinx >= 4.3.0 +furo >= 2021.11.15 +sphinxcontrib-apidoc >= 0.3.0 diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000000..977c0889a6 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,272 @@ +# +# This file is autogenerated by pip-compile with python 3.10 +# To update, run: +# +# pip-compile --allow-unsafe --generate-hashes --output-file=docs/requirements.txt --strip-extras docs/requirements.in +# +alabaster==0.7.12 \ + --hash=sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359 \ + --hash=sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02 + # via sphinx +attrs==21.2.0 \ + --hash=sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1 \ + --hash=sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb + # via markdown-it-py +babel==2.9.1 \ + --hash=sha256:ab49e12b91d937cd11f0b67cb259a57ab4ad2b59ac7a3b41d6c06c0ac5b0def9 \ + --hash=sha256:bc0c176f9f6a994582230df350aa6e05ba2ebe4b3ac317eab29d9be5d2768da0 + # via sphinx +beautifulsoup4==4.10.0 \ + --hash=sha256:9a315ce70049920ea4572a4055bc4bd700c940521d36fc858205ad4fcde149bf \ + --hash=sha256:c23ad23c521d818955a4151a67d81580319d4bf548d3d49f4223ae041ff98891 + # via furo +certifi==2021.10.8 \ + --hash=sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872 \ + --hash=sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569 + # via requests +charset-normalizer==2.0.7 \ + --hash=sha256:e019de665e2bcf9c2b64e2e5aa025fa991da8720daa3c1138cadd2fd1856aed0 \ + --hash=sha256:f7af805c321bfa1ce6714c51f254e0d5bb5e5834039bc17db7ebe3a4cec9492b + # via requests +docutils==0.17.1 \ + --hash=sha256:686577d2e4c32380bb50cbb22f575ed742d58168cee37e99117a854bcd88f125 \ + --hash=sha256:cf316c8370a737a022b72b56874f6602acf974a37a9fba42ec2876387549fc61 + # via + # myst-parser + # sphinx +furo==2021.11.15 \ + --hash=sha256:17b9fcf4de20f661d13db1ea83f11f7bf30be13738cffc88637889bf79c0469f \ + --hash=sha256:bdca82c3f211a24f850dcb12be3cb0e3f152cd3f2adfc0449bf9db6a07856bd3 + # via -r docs/requirements.in +idna==3.3 \ + --hash=sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff \ + --hash=sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d + # via requests +imagesize==1.3.0 \ + --hash=sha256:1db2f82529e53c3e929e8926a1fa9235aa82d0bd0c580359c67ec31b2fddaa8c \ + --hash=sha256:cd1750d452385ca327479d45b64d9c7729ecf0b3969a58148298c77092261f9d + # via sphinx +jinja2==3.0.3 \ + --hash=sha256:077ce6014f7b40d03b47d1f1ca4b0fc8328a692bd284016f806ed0eaca390ad8 \ + --hash=sha256:611bb273cd68f3b993fabdc4064fc858c5b47a973cb5aa7999ec1ba405c87cd7 + # via + # myst-parser + # sphinx +linkify-it-py==1.0.2 \ + --hash=sha256:4f416e72a41d9a00ecf1270ffb28b033318e458ac1144eb7c326563968a5dd24 \ + --hash=sha256:6c37ef4fc3001b38bc2359ccb5dc7e54388ec5d54fe46d2dbcd9a081f90fdbe3 + # via myst-parser +markdown-it-py==1.1.0 \ + --hash=sha256:36be6bb3ad987bfdb839f5ba78ddf094552ca38ccbd784ae4f74a4e1419fc6e3 \ + --hash=sha256:98080fc0bc34c4f2bcf0846a096a9429acbd9d5d8e67ed34026c03c61c464389 + # via + # mdit-py-plugins + # myst-parser +markupsafe==2.0.1 \ + --hash=sha256:01a9b8ea66f1658938f65b93a85ebe8bc016e6769611be228d797c9d998dd298 \ + --hash=sha256:023cb26ec21ece8dc3907c0e8320058b2e0cb3c55cf9564da612bc325bed5e64 \ + --hash=sha256:0446679737af14f45767963a1a9ef7620189912317d095f2d9ffa183a4d25d2b \ + --hash=sha256:04635854b943835a6ea959e948d19dcd311762c5c0c6e1f0e16ee57022669194 \ + --hash=sha256:0717a7390a68be14b8c793ba258e075c6f4ca819f15edfc2a3a027c823718567 \ + --hash=sha256:0955295dd5eec6cb6cc2fe1698f4c6d84af2e92de33fbcac4111913cd100a6ff \ + --hash=sha256:0d4b31cc67ab36e3392bbf3862cfbadac3db12bdd8b02a2731f509ed5b829724 \ + --hash=sha256:10f82115e21dc0dfec9ab5c0223652f7197feb168c940f3ef61563fc2d6beb74 \ + --hash=sha256:168cd0a3642de83558a5153c8bd34f175a9a6e7f6dc6384b9655d2697312a646 \ + --hash=sha256:1d609f577dc6e1aa17d746f8bd3c31aa4d258f4070d61b2aa5c4166c1539de35 \ + --hash=sha256:1f2ade76b9903f39aa442b4aadd2177decb66525062db244b35d71d0ee8599b6 \ + --hash=sha256:20dca64a3ef2d6e4d5d615a3fd418ad3bde77a47ec8a23d984a12b5b4c74491a \ + --hash=sha256:2a7d351cbd8cfeb19ca00de495e224dea7e7d919659c2841bbb7f420ad03e2d6 \ + --hash=sha256:2d7d807855b419fc2ed3e631034685db6079889a1f01d5d9dac950f764da3dad \ + --hash=sha256:2ef54abee730b502252bcdf31b10dacb0a416229b72c18b19e24a4509f273d26 \ + --hash=sha256:36bc903cbb393720fad60fc28c10de6acf10dc6cc883f3e24ee4012371399a38 \ + --hash=sha256:37205cac2a79194e3750b0af2a5720d95f786a55ce7df90c3af697bfa100eaac \ + --hash=sha256:3c112550557578c26af18a1ccc9e090bfe03832ae994343cfdacd287db6a6ae7 \ + --hash=sha256:3dd007d54ee88b46be476e293f48c85048603f5f516008bee124ddd891398ed6 \ + --hash=sha256:4296f2b1ce8c86a6aea78613c34bb1a672ea0e3de9c6ba08a960efe0b0a09047 \ + --hash=sha256:47ab1e7b91c098ab893b828deafa1203de86d0bc6ab587b160f78fe6c4011f75 \ + --hash=sha256:49e3ceeabbfb9d66c3aef5af3a60cc43b85c33df25ce03d0031a608b0a8b2e3f \ + --hash=sha256:4dc8f9fb58f7364b63fd9f85013b780ef83c11857ae79f2feda41e270468dd9b \ + --hash=sha256:4efca8f86c54b22348a5467704e3fec767b2db12fc39c6d963168ab1d3fc9135 \ + --hash=sha256:53edb4da6925ad13c07b6d26c2a852bd81e364f95301c66e930ab2aef5b5ddd8 \ + --hash=sha256:5855f8438a7d1d458206a2466bf82b0f104a3724bf96a1c781ab731e4201731a \ + --hash=sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a \ + --hash=sha256:5b6d930f030f8ed98e3e6c98ffa0652bdb82601e7a016ec2ab5d7ff23baa78d1 \ + --hash=sha256:5bb28c636d87e840583ee3adeb78172efc47c8b26127267f54a9c0ec251d41a9 \ + --hash=sha256:60bf42e36abfaf9aff1f50f52644b336d4f0a3fd6d8a60ca0d054ac9f713a864 \ + --hash=sha256:611d1ad9a4288cf3e3c16014564df047fe08410e628f89805e475368bd304914 \ + --hash=sha256:6300b8454aa6930a24b9618fbb54b5a68135092bc666f7b06901f897fa5c2fee \ + --hash=sha256:63f3268ba69ace99cab4e3e3b5840b03340efed0948ab8f78d2fd87ee5442a4f \ + --hash=sha256:6557b31b5e2c9ddf0de32a691f2312a32f77cd7681d8af66c2692efdbef84c18 \ + --hash=sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8 \ + --hash=sha256:6a7fae0dd14cf60ad5ff42baa2e95727c3d81ded453457771d02b7d2b3f9c0c2 \ + --hash=sha256:6c4ca60fa24e85fe25b912b01e62cb969d69a23a5d5867682dd3e80b5b02581d \ + --hash=sha256:6fcf051089389abe060c9cd7caa212c707e58153afa2c649f00346ce6d260f1b \ + --hash=sha256:7d91275b0245b1da4d4cfa07e0faedd5b0812efc15b702576d103293e252af1b \ + --hash=sha256:89c687013cb1cd489a0f0ac24febe8c7a666e6e221b783e53ac50ebf68e45d86 \ + --hash=sha256:8d206346619592c6200148b01a2142798c989edcb9c896f9ac9722a99d4e77e6 \ + --hash=sha256:905fec760bd2fa1388bb5b489ee8ee5f7291d692638ea5f67982d968366bef9f \ + --hash=sha256:97383d78eb34da7e1fa37dd273c20ad4320929af65d156e35a5e2d89566d9dfb \ + --hash=sha256:984d76483eb32f1bcb536dc27e4ad56bba4baa70be32fa87152832cdd9db0833 \ + --hash=sha256:99df47edb6bda1249d3e80fdabb1dab8c08ef3975f69aed437cb69d0a5de1e28 \ + --hash=sha256:9f02365d4e99430a12647f09b6cc8bab61a6564363f313126f775eb4f6ef798e \ + --hash=sha256:a30e67a65b53ea0a5e62fe23682cfe22712e01f453b95233b25502f7c61cb415 \ + --hash=sha256:ab3ef638ace319fa26553db0624c4699e31a28bb2a835c5faca8f8acf6a5a902 \ + --hash=sha256:aca6377c0cb8a8253e493c6b451565ac77e98c2951c45f913e0b52facdcff83f \ + --hash=sha256:add36cb2dbb8b736611303cd3bfcee00afd96471b09cda130da3581cbdc56a6d \ + --hash=sha256:b2f4bf27480f5e5e8ce285a8c8fd176c0b03e93dcc6646477d4630e83440c6a9 \ + --hash=sha256:b7f2d075102dc8c794cbde1947378051c4e5180d52d276987b8d28a3bd58c17d \ + --hash=sha256:baa1a4e8f868845af802979fcdbf0bb11f94f1cb7ced4c4b8a351bb60d108145 \ + --hash=sha256:be98f628055368795d818ebf93da628541e10b75b41c559fdf36d104c5787066 \ + --hash=sha256:bf5d821ffabf0ef3533c39c518f3357b171a1651c1ff6827325e4489b0e46c3c \ + --hash=sha256:c47adbc92fc1bb2b3274c4b3a43ae0e4573d9fbff4f54cd484555edbf030baf1 \ + --hash=sha256:cdfba22ea2f0029c9261a4bd07e830a8da012291fbe44dc794e488b6c9bb353a \ + --hash=sha256:d6c7ebd4e944c85e2c3421e612a7057a2f48d478d79e61800d81468a8d842207 \ + --hash=sha256:d7f9850398e85aba693bb640262d3611788b1f29a79f0c93c565694658f4071f \ + --hash=sha256:d8446c54dc28c01e5a2dbac5a25f071f6653e6e40f3a8818e8b45d790fe6ef53 \ + --hash=sha256:deb993cacb280823246a026e3b2d81c493c53de6acfd5e6bfe31ab3402bb37dd \ + --hash=sha256:e0f138900af21926a02425cf736db95be9f4af72ba1bb21453432a07f6082134 \ + --hash=sha256:e9936f0b261d4df76ad22f8fee3ae83b60d7c3e871292cd42f40b81b70afae85 \ + --hash=sha256:f0567c4dc99f264f49fe27da5f735f414c4e7e7dd850cfd8e69f0862d7c74ea9 \ + --hash=sha256:f5653a225f31e113b152e56f154ccbe59eeb1c7487b39b9d9f9cdb58e6c79dc5 \ + --hash=sha256:f826e31d18b516f653fe296d967d700fddad5901ae07c622bb3705955e1faa94 \ + --hash=sha256:f8ba0e8349a38d3001fae7eadded3f6606f0da5d748ee53cc1dab1d6527b9509 \ + --hash=sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51 \ + --hash=sha256:fa130dd50c57d53368c9d59395cb5526eda596d3ffe36666cd81a44d56e48872 + # via jinja2 +mdit-py-plugins==0.2.8 \ + --hash=sha256:1833bf738e038e35d89cb3a07eb0d227ed647ce7dd357579b65343740c6d249c \ + --hash=sha256:5991cef645502e80a5388ec4fc20885d2313d4871e8b8e320ca2de14ac0c015f + # via myst-parser +myst-parser==0.15.2 \ + --hash=sha256:40124b6f27a4c42ac7f06b385e23a9dcd03d84801e9c7130b59b3729a554b1f9 \ + --hash=sha256:f7f3b2d62db7655cde658eb5d62b2ec2a4631308137bd8d10f296a40d57bbbeb + # via -r docs/requirements.in +packaging==21.2 \ + --hash=sha256:096d689d78ca690e4cd8a89568ba06d07ca097e3306a4381635073ca91479966 \ + --hash=sha256:14317396d1e8cdb122989b916fa2c7e9ca8e2be9e8060a6eff75b6b7b4d8a7e0 + # via + # setuptools-scm + # sphinx +pbr==5.7.0 \ + --hash=sha256:4651ca1445e80f2781827305de3d76b3ce53195f2227762684eb08f17bc473b7 \ + --hash=sha256:60002958e459b195e8dbe61bf22bcf344eedf1b4e03a321a5414feb15566100c + # via sphinxcontrib-apidoc +pygments==2.10.0 \ + --hash=sha256:b8e67fe6af78f492b3c4b3e2970c0624cbf08beb1e493b2c99b9fa1b67a20380 \ + --hash=sha256:f398865f7eb6874156579fdf36bc840a03cab64d1cde9e93d68f46a425ec52c6 + # via sphinx +pyparsing==2.4.7 \ + --hash=sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1 \ + --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b + # via packaging +pytz==2021.3 \ + --hash=sha256:3672058bc3453457b622aab7a1c3bfd5ab0bdae451512f6cf25f64ed37f5b87c \ + --hash=sha256:acad2d8b20a1af07d4e4c9d2e9285c5ed9104354062f275f3fcd88dcef4f1326 + # via babel +pyyaml==6.0 \ + --hash=sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293 \ + --hash=sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b \ + --hash=sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57 \ + --hash=sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b \ + --hash=sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4 \ + --hash=sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07 \ + --hash=sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba \ + --hash=sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9 \ + --hash=sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287 \ + --hash=sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513 \ + --hash=sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0 \ + --hash=sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0 \ + --hash=sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92 \ + --hash=sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f \ + --hash=sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2 \ + --hash=sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc \ + --hash=sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c \ + --hash=sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86 \ + --hash=sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4 \ + --hash=sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c \ + --hash=sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34 \ + --hash=sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b \ + --hash=sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c \ + --hash=sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb \ + --hash=sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737 \ + --hash=sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3 \ + --hash=sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d \ + --hash=sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53 \ + --hash=sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78 \ + --hash=sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803 \ + --hash=sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a \ + --hash=sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174 \ + --hash=sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5 + # via myst-parser +requests==2.26.0 \ + --hash=sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24 \ + --hash=sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7 + # via sphinx +setuptools-scm==6.3.2 \ + --hash=sha256:4c64444b1d49c4063ae60bfe1680f611c8b13833d556fd1d6050c0023162a119 \ + --hash=sha256:a49aa8081eeb3514eb9728fa5040f2eaa962d6c6f4ec9c32f6c1fba88f88a0f2 + # via -r docs/requirements.in +snowballstemmer==2.1.0 \ + --hash=sha256:b51b447bea85f9968c13b650126a888aabd4cb4463fca868ec596826325dedc2 \ + --hash=sha256:e997baa4f2e9139951b6f4c631bad912dfd3c792467e2f03d7239464af90e914 + # via sphinx +soupsieve==2.3.1 \ + --hash=sha256:1a3cca2617c6b38c0343ed661b1fa5de5637f257d4fe22bd9f1338010a1efefb \ + --hash=sha256:b8d49b1cd4f037c7082a9683dfa1801aa2597fb11c3a1155b7a5b94829b4f1f9 + # via beautifulsoup4 +sphinx==4.3.0 \ + --hash=sha256:6d051ab6e0d06cba786c4656b0fe67ba259fe058410f49e95bee6e49c4052cbf \ + --hash=sha256:7e2b30da5f39170efcd95c6270f07669d623c276521fee27ad6c380f49d2bf5b + # via + # -r docs/requirements.in + # furo + # myst-parser + # sphinxcontrib-apidoc +sphinxcontrib-apidoc==0.3.0 \ + --hash=sha256:6671a46b2c6c5b0dca3d8a147849d159065e50443df79614f921b42fbd15cb09 \ + --hash=sha256:729bf592cf7b7dd57c4c05794f732dc026127275d785c2a5494521fdde773fb9 + # via -r docs/requirements.in +sphinxcontrib-applehelp==1.0.2 \ + --hash=sha256:806111e5e962be97c29ec4c1e7fe277bfd19e9652fb1a4392105b43e01af885a \ + --hash=sha256:a072735ec80e7675e3f432fcae8610ecf509c5f1869d17e2eecff44389cdbc58 + # via sphinx +sphinxcontrib-devhelp==1.0.2 \ + --hash=sha256:8165223f9a335cc1af7ffe1ed31d2871f325254c0423bc0c4c7cd1c1e4734a2e \ + --hash=sha256:ff7f1afa7b9642e7060379360a67e9c41e8f3121f2ce9164266f61b9f4b338e4 + # via sphinx +sphinxcontrib-htmlhelp==2.0.0 \ + --hash=sha256:d412243dfb797ae3ec2b59eca0e52dac12e75a241bf0e4eb861e450d06c6ed07 \ + --hash=sha256:f5f8bb2d0d629f398bf47d0d69c07bc13b65f75a81ad9e2f71a63d4b7a2f6db2 + # via sphinx +sphinxcontrib-jsmath==1.0.1 \ + --hash=sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178 \ + --hash=sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8 + # via sphinx +sphinxcontrib-qthelp==1.0.3 \ + --hash=sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72 \ + --hash=sha256:bd9fc24bcb748a8d51fd4ecaade681350aa63009a347a8c14e637895444dfab6 + # via sphinx +sphinxcontrib-serializinghtml==1.1.5 \ + --hash=sha256:352a9a00ae864471d3a7ead8d7d79f5fc0b57e8b3f95e9867eb9eb28999b92fd \ + --hash=sha256:aa5f6de5dfdf809ef505c4895e51ef5c9eac17d0f287933eb49ec495280b6952 + # via sphinx +tomli==1.2.2 \ + --hash=sha256:c6ce0015eb38820eaf32b5db832dbc26deb3dd427bd5f6556cf0acac2c214fee \ + --hash=sha256:f04066f68f5554911363063a30b108d2b5a5b1a010aa8b6132af78489fe3aade + # via setuptools-scm +uc-micro-py==1.0.1 \ + --hash=sha256:316cfb8b6862a0f1d03540f0ae6e7b033ff1fa0ddbe60c12cbe0d4cec846a69f \ + --hash=sha256:b7cdf4ea79433043ddfe2c82210208f26f7962c0cfbe3bacb05ee879a7fdb596 + # via linkify-it-py +urllib3==1.26.7 \ + --hash=sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece \ + --hash=sha256:c4fdf4019605b6e5423637e01bc9fe4daef873709a7973e195ceba0a62bbc844 + # via requests + +# The following packages are considered to be unsafe in a requirements file: +setuptools==59.0.1 \ + --hash=sha256:899d27ec8104a68d4ba813b1afd66708a1a10e9391e79be92c8c60f9c77d05e5 \ + --hash=sha256:dedb38ba61844d9df36072dad313cb79426fd50497aaac9c0da4cd50dbeeb110 + # via + # setuptools-scm + # sphinx diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt new file mode 100644 index 0000000000..5562179d58 --- /dev/null +++ b/docs/spelling_wordlist.txt @@ -0,0 +1,13 @@ +IPv +Nginx +Pluggable +scm +Threadless +threadless +youtube +socio +sexualized +https +www +html +faq diff --git a/examples/README.md b/examples/README.md index 84bf0f4452..90455dc4f4 100644 --- a/examples/README.md +++ b/examples/README.md @@ -6,6 +6,7 @@ Looking for `proxy.py` plugin examples? Check [proxy/plugin](https://github.com Table of Contents ================= +* [Generic Work Acceptor and Executor](#generic-work-acceptor-and-executor) * [WebSocket Client](#websocket-client) * [TCP Echo Server](#tcp-echo-server) * [TCP Echo Client](#tcp-echo-client) @@ -14,6 +15,17 @@ Table of Contents * [PubSub Eventing](#pubsub-eventing) * [Https Connect Tunnel](#https-connect-tunnel) +## Generic Work Acceptor and Executor + +1. Makes use of `proxy.core.AcceptorPool` and `proxy.core.Work` +2. Demonstrates how to perform generic work using `proxy.py` core. + +Start `web_scraper.py` as: + +```console +โฏ PYTHONPATH=. python examples/web_scraper.py +``` + ## WebSocket Client 1. Makes use of `proxy.http.websocket.WebsocketClient` which is built on-top of `asyncio` @@ -22,7 +34,7 @@ Table of Contents Start `websocket_client.py` as: -```bash +```console โฏ PYTHONPATH=. python examples/websocket_client.py Received b'hello' after 306 millisec Received b'hello' after 308 millisec @@ -44,7 +56,7 @@ Received b'hello' after 309 millisec Start `tcp_echo_server.py` as: -```bash +```console โฏ PYTHONPATH=. python examples/tcp_echo_server.py Connection accepted from ('::1', 53285, 0, 0) Connection closed by client ('::1', 53285, 0, 0) @@ -57,7 +69,7 @@ Connection closed by client ('::1', 53285, 0, 0) Start `tcp_echo_client.py` as: -```bash +```console โฏ PYTHONPATH=. python examples/tcp_echo_client.py b'hello' b'hello' @@ -81,7 +93,7 @@ KeyboardInterrupt Start `ssl_echo_server.py` as: -```bash +```console โฏ PYTHONPATH=. python examples/ssl_echo_server.py ``` @@ -92,7 +104,7 @@ Start `ssl_echo_server.py` as: Start `ssl_echo_client.py` as: -```bash +```console โฏ PYTHONPATH=. python examples/ssl_echo_client.py ``` @@ -107,7 +119,7 @@ Start `ssl_echo_client.py` as: Start `pubsub_eventing.py` as: -```bash +```console โฏ PYTHONPATH=. python examples/pubsub_eventing.py DEBUG:proxy.core.event.subscriber:Subscribed relay sub id 5eb22010764f4d44900f41e2fb408ca6 from core events publisher starting diff --git a/examples/https_connect_tunnel.py b/examples/https_connect_tunnel.py index 950186468f..e60edaeda0 100644 --- a/examples/https_connect_tunnel.py +++ b/examples/https_connect_tunnel.py @@ -9,30 +9,33 @@ :license: BSD, see LICENSE for more details. """ import time + from typing import Any, Optional -from proxy.proxy import Proxy +from proxy import Proxy from proxy.common.utils import build_http_response -from proxy.http.codes import httpStatusCodes +from proxy.http import httpStatusCodes from proxy.http.parser import httpParserStates -from proxy.http.methods import httpMethods -from proxy.core.acceptor import AcceptorPool from proxy.core.base import BaseTcpTunnelHandler class HttpsConnectTunnelHandler(BaseTcpTunnelHandler): """A https CONNECT tunnel.""" - PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT = memoryview(build_http_response( - httpStatusCodes.OK, - reason=b'Connection established' - )) - - PROXY_TUNNEL_UNSUPPORTED_SCHEME = memoryview(build_http_response( - httpStatusCodes.BAD_REQUEST, - headers={b'Connection': b'close'}, - reason=b'Unsupported protocol scheme' - )) + PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT = memoryview( + build_http_response( + httpStatusCodes.OK, + reason=b'Connection established', + ), + ) + + PROXY_TUNNEL_UNSUPPORTED_SCHEME = memoryview( + build_http_response( + httpStatusCodes.BAD_REQUEST, + headers={b'Connection': b'close'}, + reason=b'Unsupported protocol scheme', + ), + ) def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) @@ -47,9 +50,10 @@ def handle_data(self, data: memoryview) -> Optional[bool]: self.request.parse(data) # Drop the request if not a CONNECT request - if self.request.method != httpMethods.CONNECT: - self.client.queue( - HttpsConnectTunnelHandler.PROXY_TUNNEL_UNSUPPORTED_SCHEME) + if not self.request.is_https_tunnel(): + self.work.queue( + HttpsConnectTunnelHandler.PROXY_TUNNEL_UNSUPPORTED_SCHEME, + ) return True # CONNECT requests are short and we need not worry about @@ -60,25 +64,26 @@ def handle_data(self, data: memoryview) -> Optional[bool]: self.connect_upstream() # Queue tunnel established response to client - self.client.queue( - HttpsConnectTunnelHandler.PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT) + self.work.queue( + HttpsConnectTunnelHandler.PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT, + ) return None def main() -> None: # This example requires `threadless=True` - pool = AcceptorPool( - flags=Proxy.initialize(port=12345, num_workers=1, threadless=True), - work_klass=HttpsConnectTunnelHandler) - try: - pool.setup() - while True: - time.sleep(1) - except KeyboardInterrupt: - pass - finally: - pool.shutdown() + with Proxy( + work_klass=HttpsConnectTunnelHandler, + threadless=True, + num_workers=1, + port=12345, + ): + try: + while True: + time.sleep(1) + except KeyboardInterrupt: + pass if __name__ == '__main__': diff --git a/examples/pubsub_eventing.py b/examples/pubsub_eventing.py index 3e247c38eb..607cf4536e 100644 --- a/examples/pubsub_eventing.py +++ b/examples/pubsub_eventing.py @@ -9,99 +9,97 @@ :license: BSD, see LICENSE for more details. """ import time -import threading import multiprocessing import logging -from typing import Dict, Any +from typing import Dict, Any, Optional -from proxy.core.event import EventQueue, EventSubscriber, EventDispatcher, eventNames +from proxy.common.constants import DEFAULT_LOG_FORMAT +from proxy.core.event import EventManager, EventQueue, EventSubscriber, eventNames -# Enable debug logging to view core event logs -logging.basicConfig(level=logging.DEBUG) +logging.basicConfig(level=logging.DEBUG, format=DEFAULT_LOG_FORMAT) -# Eventing requires a multiprocess safe queue -# so that events can be safely published and received -# between processes. -manager = multiprocessing.Manager() - -main_publisher_request_id = '1234' -process_publisher_request_id = '12345' -num_events_received = [0, 0] +logger = logging.getLogger(__name__) -def publisher_process(shutdown_event: multiprocessing.synchronize.Event, - dispatcher_queue: EventQueue) -> None: - print('publisher starting') - try: - while not shutdown_event.is_set(): - dispatcher_queue.publish( - request_id=process_publisher_request_id, - event_name=eventNames.WORK_STARTED, - event_payload={'time': time.time()}, - publisher_id='eventing_pubsub_process' - ) - except KeyboardInterrupt: - pass - print('publisher shutdown') +num_events_received = [0, 0] +# Execute within a separate thread context def on_event(payload: Dict[str, Any]) -> None: '''Subscriber callback.''' global num_events_received - if payload['request_id'] == main_publisher_request_id: + if payload['request_id'] == '1234': num_events_received[0] += 1 else: num_events_received[1] += 1 - # print(payload) -if __name__ == '__main__': - start_time = time.time() - - # Start dispatcher thread - dispatcher_queue = EventQueue(manager.Queue()) - dispatcher_shutdown_event = threading.Event() - dispatcher = EventDispatcher( - shutdown=dispatcher_shutdown_event, - event_queue=dispatcher_queue) - dispatcher_thread = threading.Thread(target=dispatcher.run) - dispatcher_thread.start() - - # Create a subscriber - subscriber = EventSubscriber(dispatcher_queue) - # Internally, subscribe will start a separate thread - # to receive incoming published messages - subscriber.subscribe(on_event) - - # Start a publisher process to demonstrate safe exchange - # of messages between processes. - publisher_shutdown_event = multiprocessing.Event() - publisher = multiprocessing.Process( - target=publisher_process, args=( - publisher_shutdown_event, dispatcher_queue, )) - publisher.start() - +def publisher_process( + shutdown_event: multiprocessing.synchronize.Event, + dispatcher_queue: EventQueue, +) -> None: + logger.info('publisher started') try: - while True: - # Dispatch event from main process + while not shutdown_event.is_set(): dispatcher_queue.publish( - request_id=main_publisher_request_id, + request_id='12345', event_name=eventNames.WORK_STARTED, event_payload={'time': time.time()}, - publisher_id='eventing_pubsub_main' + publisher_id='eventing_pubsub_process', ) except KeyboardInterrupt: - print('bye!!!') - finally: - # Stop publisher - publisher_shutdown_event.set() - publisher.join() - # Stop subscriber thread - subscriber.unsubscribe() - # Signal dispatcher to shutdown - dispatcher_shutdown_event.set() - # Wait for dispatcher shutdown - dispatcher_thread.join() - print('Received {0} events from main thread, {1} events from another process, in {2} seconds'.format( - num_events_received[0], num_events_received[1], time.time() - start_time)) + pass + logger.info('publisher shutdown') + + +if __name__ == '__main__': + start_time = time.time() + + # Start eventing core + subscriber: Optional[EventSubscriber] = None + with EventManager() as event_manager: + assert event_manager.queue + + # Create a subscriber. + # Internally, subscribe will start a separate thread + # to receive incoming published messages. + subscriber = EventSubscriber(event_manager.queue, callback=on_event) + subscriber.setup() + + # Start a publisher process to demonstrate safe exchange + # of messages between processes. + publisher_shutdown_event = multiprocessing.Event() + publisher = multiprocessing.Process( + target=publisher_process, args=( + publisher_shutdown_event, event_manager.queue, ), + ) + publisher.start() + + # Dispatch event from main process too + # to demonstrate safe exchange of messages + # between threads. + try: + while True: + event_manager.queue.publish( + request_id='1234', + event_name=eventNames.WORK_STARTED, + event_payload={'time': time.time()}, + publisher_id='eventing_pubsub_main', + ) + except KeyboardInterrupt: + logger.info('bye!!!') + finally: + # Stop publisher process + publisher_shutdown_event.set() + publisher.join() + # Stop subscriber thread + subscriber.unsubscribe() + logger.info( + 'Received {0} events from main thread, {1} events from another process, in {2} seconds'.format( + num_events_received[0], num_events_received[1], time.time( + ) - start_time, + ), + ) + if subscriber: + subscriber.shutdown(do_unsubscribe=False) diff --git a/examples/ssl_echo_client.py b/examples/ssl_echo_client.py index 227b26c94f..3c8f2d8cc5 100644 --- a/examples/ssl_echo_client.py +++ b/examples/ssl_echo_client.py @@ -8,9 +8,13 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ +import logging + from proxy.core.connection import TcpServerConnection from proxy.common.constants import DEFAULT_BUFFER_SIZE +logger = logging.getLogger(__name__) + if __name__ == '__main__': client = TcpServerConnection('::', 12345) client.connect() @@ -24,6 +28,6 @@ data = client.recv(DEFAULT_BUFFER_SIZE) if data is None: break - print(data.tobytes()) + logger.info(data.tobytes()) finally: client.close() diff --git a/examples/ssl_echo_server.py b/examples/ssl_echo_server.py index 013bc3a5f0..65432f3719 100644 --- a/examples/ssl_echo_server.py +++ b/examples/ssl_echo_server.py @@ -11,9 +11,8 @@ import time from typing import Optional -from proxy.proxy import Proxy +from proxy import Proxy from proxy.common.utils import wrap_socket -from proxy.core.acceptor import AcceptorPool from proxy.core.connection import TcpClientConnection from proxy.core.base import BaseTcpServerHandler @@ -27,38 +26,37 @@ def initialize(self) -> None: # here using wrap_socket() utility. assert self.flags.keyfile is not None and self.flags.certfile is not None conn = wrap_socket( - self.client.connection, + self.work.connection, self.flags.keyfile, - self.flags.certfile) + self.flags.certfile, + ) conn.setblocking(False) # Upgrade plain TcpClientConnection to SSL connection object - self.client = TcpClientConnection( - conn=conn, addr=self.client.addr) + self.work = TcpClientConnection( + conn=conn, addr=self.work.addr, + ) def handle_data(self, data: memoryview) -> Optional[bool]: # echo back to client - self.client.queue(data) + self.work.queue(data) return None def main() -> None: # This example requires `threadless=True` - pool = AcceptorPool( - flags=Proxy.initialize( - port=12345, - num_workers=1, - threadless=True, - keyfile='https-key.pem', - certfile='https-signed-cert.pem'), - work_klass=EchoSSLServerHandler) - try: - pool.setup() - while True: - time.sleep(1) - except KeyboardInterrupt: - pass - finally: - pool.shutdown() + with Proxy( + work_klass=EchoSSLServerHandler, + threadless=True, + num_workers=1, + port=12345, + keyfile='https-key.pem', + certfile='https-signed-cert.pem', + ): + try: + while True: + time.sleep(1) + except KeyboardInterrupt: + pass if __name__ == '__main__': diff --git a/examples/tcp_echo_client.py b/examples/tcp_echo_client.py index decabb505f..e62230f13e 100644 --- a/examples/tcp_echo_client.py +++ b/examples/tcp_echo_client.py @@ -8,9 +8,13 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ +import logging + from proxy.common.utils import socket_connection from proxy.common.constants import DEFAULT_BUFFER_SIZE +logger = logging.getLogger(__name__) + if __name__ == '__main__': with socket_connection(('::', 12345)) as client: while True: @@ -18,4 +22,4 @@ data = client.recv(DEFAULT_BUFFER_SIZE) if data is None: break - print(data) + logger.info(data) diff --git a/examples/tcp_echo_server.py b/examples/tcp_echo_server.py index c468b7eaf8..cd4924150f 100644 --- a/examples/tcp_echo_server.py +++ b/examples/tcp_echo_server.py @@ -11,8 +11,7 @@ import time from typing import Optional -from proxy.proxy import Proxy -from proxy.core.acceptor import AcceptorPool +from proxy import Proxy from proxy.core.base import BaseTcpServerHandler @@ -20,27 +19,27 @@ class EchoServerHandler(BaseTcpServerHandler): """Sets client socket to non-blocking during initialization.""" def initialize(self) -> None: - self.client.connection.setblocking(False) + self.work.connection.setblocking(False) def handle_data(self, data: memoryview) -> Optional[bool]: # echo back to client - self.client.queue(data) + self.work.queue(data) return None def main() -> None: # This example requires `threadless=True` - pool = AcceptorPool( - flags=Proxy.initialize(port=12345, num_workers=1, threadless=True), - work_klass=EchoServerHandler) - try: - pool.setup() - while True: - time.sleep(1) - except KeyboardInterrupt: - pass - finally: - pool.shutdown() + with Proxy( + work_klass=EchoServerHandler, + threadless=True, + num_workers=1, + port=12345, + ): + try: + while True: + time.sleep(1) + except KeyboardInterrupt: + pass if __name__ == '__main__': diff --git a/examples/web_scraper.py b/examples/web_scraper.py new file mode 100644 index 0000000000..4b925876c5 --- /dev/null +++ b/examples/web_scraper.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import time + +from typing import Dict + +from proxy import Proxy +from proxy.core.acceptor import Work +from proxy.common.types import Readables, Writables + + +class WebScraper(Work): + """Demonstrates how to orchestrate a generic work acceptors and executors + workflow using proxy.py core. + + By default, `WebScraper` expects to receive work from a file on disk. + Each line in the file must be a URL to scrape. Received URL is scrapped + by the implementation in this class. + + After scrapping, results are published to the eventing core. One or several + result subscriber can then handle the result as necessary. Currently, result + subscribers consume the scrapped response and write discovered URL in the + file on the disk. This creates a feedback loop. Allowing WebScraper to + continue endlessly. + + NOTE: No loop detection is performed currently. + + NOTE: File descriptor need not point to a file on disk. + Example, file descriptor can be a database connection. + For simplicity, imagine a Redis server connection handling + only PUBSUB protocol. + """ + + async def get_events(self) -> Dict[int, int]: + """Return sockets and events (read or write) that we are interested in.""" + return {} + + async def handle_events( + self, + readables: Readables, + writables: Writables, + ) -> bool: + """Handle readable and writable sockets. + + Return True to shutdown work.""" + return False + + +if __name__ == '__main__': + with Proxy( + work_klass=WebScraper, + threadless=True, + num_workers=1, + port=12345, + ) as pool: + while True: + time.sleep(1) diff --git a/examples/websocket_client.py b/examples/websocket_client.py index c87ed3e4fb..48a217b4d3 100644 --- a/examples/websocket_client.py +++ b/examples/websocket_client.py @@ -9,8 +9,9 @@ :license: BSD, see LICENSE for more details. """ import time -from proxy.http.websocket import WebsocketClient, WebsocketFrame, websocketOpcodes +import logging +from proxy.http.websocket import WebsocketClient, WebsocketFrame, websocketOpcodes # globals client: WebsocketClient @@ -18,14 +19,20 @@ static_frame = memoryview(WebsocketFrame.text(b'hello')) num_echos = 10 +logger = logging.getLogger(__name__) + def on_message(frame: WebsocketFrame) -> None: """WebsocketClient on_message callback.""" global client, num_echos, last_dispatch_time - print('Received %r after %d millisec' % - (frame.data, (time.time() - last_dispatch_time) * 1000)) - assert(frame.data == b'hello' and frame.opcode == - websocketOpcodes.TEXT_FRAME) + logger.info( + 'Received %r after %d millisec' % + (frame.data, (time.time() - last_dispatch_time) * 1000), + ) + assert( + frame.data == b'hello' and frame.opcode == + websocketOpcodes.TEXT_FRAME + ) if num_echos > 0: client.queue(static_frame) last_dispatch_time = time.time() @@ -40,7 +47,8 @@ def on_message(frame: WebsocketFrame) -> None: b'echo.websocket.org', 80, b'/', - on_message=on_message) + on_message=on_message, + ) # Perform handshake client.handshake() # Queue some data for client diff --git a/git-pre-commit b/git-pre-commit index 2aad486415..aeef83fa29 100755 --- a/git-pre-commit +++ b/git-pre-commit @@ -1,3 +1,3 @@ #!/bin/bash -make +make lib-pytest diff --git a/git-pre-push b/git-pre-push new file mode 100755 index 0000000000..2aad486415 --- /dev/null +++ b/git-pre-push @@ -0,0 +1,3 @@ +#!/bin/bash + +make diff --git a/helper/benchmark.sh b/helper/benchmark.sh new file mode 100755 index 0000000000..aaa62321f0 --- /dev/null +++ b/helper/benchmark.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# +# proxy.py +# ~~~~~~~~ +# โšกโšกโšก Fast, Lightweight, Programmable, TLS interception capable +# proxy server for Application debugging, testing and development. +# +# :copyright: (c) 2013-present by Abhinav Singh and contributors. +# :license: BSD, see LICENSE for more details. +# +usage() { + echo "Usage: ./helper/benchmark.sh" + echo "You must run this script from proxy.py repo root." +} + +DIRNAME=$(dirname "$0") +if [ "$DIRNAME" != "./helper" ]; then + usage + exit 1 +fi + +BASENAME=$(basename "$0") +if [ "$BASENAME" != "benchmark.sh" ]; then + usage + exit 1 +fi + +PWD=$(pwd) +if [ $(basename $PWD) != "proxy.py" ]; then + usage + exit 1 +fi + +TIMEOUT=1 +QPS=8000 +CONCURRENCY=100 +TOTAL_REQUESTS=100000 +OPEN_FILE_LIMIT=65536 +BACKLOG=OPEN_FILE_LIMIT +PID_FILE=/tmp/proxy.pid + +ulimit -n $OPEN_FILE_LIMIT + +PID=$(cat $PID_FILE) +if [[ -z "$PID" ]]; then + echo "Either pid file doesn't exist or no pid found in the pid file" + exit 1 +fi +ADDR=$(lsof -Pan -p $PID -i | grep -v COMMAND | awk '{ print $9 }') + +PRE_RUN_OPEN_FILES=$(./helper/monitor_open_files.sh) + +echo "CONCURRENCY: $CONCURRENCY workers, TOTAL REQUESTS: $TOTAL_REQUESTS req, QPS: $QPS req/sec, TIMEOUT: $TIMEOUT sec" +hey \ + -n $TOTAL_REQUESTS \ + -c $CONCURRENCY \ + -q $QPS \ + -t $TIMEOUT \ + http://$ADDR/http-route-example + +POST_RUN_OPEN_FILES=$(./helper/monitor_open_files.sh) + +echo $output + +echo "Open files diff:" +diff <( echo "$PRE_RUN_OPEN_FILES" ) <( echo "$POST_RUN_OPEN_FILES" ) + +# while true; do netstat -ant | grep .8899 | awk '{print $6}' | sort | uniq -c | sort -n; sleep 1; done diff --git a/helper/chrome_with_rdp.sh b/helper/chrome_with_rdp.sh new file mode 100755 index 0000000000..1b59a464d7 --- /dev/null +++ b/helper/chrome_with_rdp.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# proxy.py +# ~~~~~~~~ +# โšกโšกโšก Fast, Lightweight, Programmable, TLS interception capable +# proxy server for Application debugging, testing and development. +# +# :copyright: (c) 2013-present by Abhinav Singh and contributors. +# :license: BSD, see LICENSE for more details. +# +# Usage +# ./chrome_with_proxy + +/Applications/Google\ Chrome.app/Contents/MacOS/Google\ Chrome \ + --no-first-run \ + --no-default-browser-check \ + --remote-debugging-port=9222 \ + --user-data-dir="$(mktemp -d -t 'chrome-remote_data_dir')" diff --git a/helper/homebrew/develop/proxy.rb b/helper/homebrew/develop/proxy.rb index d695e53ed8..99a480e819 100644 --- a/helper/homebrew/develop/proxy.rb +++ b/helper/homebrew/develop/proxy.rb @@ -4,15 +4,10 @@ class Proxy < Formula desc "โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on Network monitoring, controls & Application development, testing, debugging." homepage "https://github.com/abhinavsingh/proxy.py" - url "https://github.com/abhinavsingh/proxy.py/archive/develop.zip" + url "https://github.com/abhinavsingh/proxy.py.git", :using => :git, :branch => "develop" version "develop" - depends_on "python" - - resource "typing-extensions" do - url "https://files.pythonhosted.org/packages/6a/28/d32852f2af6b5ead85d396249d5bdf450833f3a69896d76eb480d9c5e406/typing_extensions-3.7.4.2.tar.gz" - sha256 "79ee589a3caca649a9bfd2a8de4709837400dfa00b6cc81962a1e6a1815969ae" - end + depends_on "python@3.10" def install virtualenv_install_with_resources diff --git a/helper/homebrew/stable/proxy.rb b/helper/homebrew/stable/proxy.rb index 83b38cbb1a..7761ac62fd 100644 --- a/helper/homebrew/stable/proxy.rb +++ b/helper/homebrew/stable/proxy.rb @@ -4,15 +4,10 @@ class Proxy < Formula desc "โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on Network monitoring, controls & Application development, testing, debugging." homepage "https://github.com/abhinavsingh/proxy.py" - url "https://github.com/abhinavsingh/proxy.py/archive/master.zip" - version "2.2.0" + url "https://github.com/abhinavsingh/proxy.py.git", :using => :git, :branch => "master" + version "stable" - depends_on "python" - - resource "typing-extensions" do - url "https://files.pythonhosted.org/packages/6a/28/d32852f2af6b5ead85d396249d5bdf450833f3a69896d76eb480d9c5e406/typing_extensions-3.7.4.2.tar.gz" - sha256 "79ee589a3caca649a9bfd2a8de4709837400dfa00b6cc81962a1e6a1815969ae" - end + depends_on "python@3.10" def install virtualenv_install_with_resources diff --git a/helper/monitor_open_files.sh b/helper/monitor_open_files.sh index 7bfa48c631..7a8caa0eb1 100755 --- a/helper/monitor_open_files.sh +++ b/helper/monitor_open_files.sh @@ -1,5 +1,5 @@ #!/bin/bash - +# # proxy.py # ~~~~~~~~ # โšกโšกโšก Fast, Lightweight, Programmable, TLS interception capable diff --git a/menubar/proxy.py.xcodeproj/project.pbxproj b/menubar/proxy.py.xcodeproj/project.pbxproj index 5c44621506..d86998c5c8 100644 --- a/menubar/proxy.py.xcodeproj/project.pbxproj +++ b/menubar/proxy.py.xcodeproj/project.pbxproj @@ -447,7 +447,7 @@ "@executable_path/../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = 10.15; - PRODUCT_BUNDLE_IDENTIFIER = "com.abhinavsingh.proxy-py"; + PRODUCT_BUNDLE_IDENTIFIER = com.jaxl.proxy; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_VERSION = 5.0; }; @@ -471,7 +471,7 @@ "@executable_path/../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = 10.15; - PRODUCT_BUNDLE_IDENTIFIER = "com.abhinavsingh.proxy-py"; + PRODUCT_BUNDLE_IDENTIFIER = com.jaxl.proxy; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_VERSION = 5.0; }; diff --git a/menubar/proxy.py.xcodeproj/project.xcworkspace/xcuserdata/abhinavsingh.xcuserdatad/UserInterfaceState.xcuserstate b/menubar/proxy.py.xcodeproj/project.xcworkspace/xcuserdata/abhinavsingh.xcuserdatad/UserInterfaceState.xcuserstate index f4e17991b4..160c36502d 100644 Binary files a/menubar/proxy.py.xcodeproj/project.xcworkspace/xcuserdata/abhinavsingh.xcuserdatad/UserInterfaceState.xcuserstate and b/menubar/proxy.py.xcodeproj/project.xcworkspace/xcuserdata/abhinavsingh.xcuserdatad/UserInterfaceState.xcuserstate differ diff --git a/menubar/proxy.py/Info.plist b/menubar/proxy.py/Info.plist index ada1ded305..9e185bfbe8 100644 --- a/menubar/proxy.py/Info.plist +++ b/menubar/proxy.py/Info.plist @@ -20,8 +20,12 @@ 1.0 CFBundleVersion 1 + LSApplicationCategoryType + public.app-category.utilities LSMinimumSystemVersion $(MACOSX_DEPLOYMENT_TARGET) + LSUIElement + NSHumanReadableCopyright Copyright ยฉ 2013-present by Abhinav Singh and contributors. All rights reserved. NSMainStoryboardFile @@ -30,8 +34,6 @@ NSApplication NSSupportsAutomaticTermination - LSUIElement - NSSupportsSuddenTermination diff --git a/proxy/__init__.py b/proxy/__init__.py index ca7c5269d0..94b6d7203d 100755 --- a/proxy/__init__.py +++ b/proxy/__init__.py @@ -7,11 +7,15 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + eventing + Submodules + Subpackages """ -from .proxy import entry_point -from .proxy import main, start -from .proxy import Proxy -from .testing.test_case import TestCase +from .proxy import entry_point, main, Proxy +from .testing import TestCase __all__ = [ # PyPi package entry_point. See @@ -19,7 +23,7 @@ 'entry_point', # Embed proxy.py. See # https://github.com/abhinavsingh/proxy.py#embed-proxypy - 'main', 'start', + 'main', # Unit testing with proxy.py. See # https://github.com/abhinavsingh/proxy.py#unit-testing-with-proxypy 'TestCase', diff --git a/proxy/common/.gitignore b/proxy/common/.gitignore new file mode 100644 index 0000000000..7d536eca4b --- /dev/null +++ b/proxy/common/.gitignore @@ -0,0 +1,2 @@ +# Build-time setuptools-scm generated version module +/_scm_version.py diff --git a/proxy/common/__init__.py b/proxy/common/__init__.py index 232621f0b5..02bacd6ce5 100644 --- a/proxy/common/__init__.py +++ b/proxy/common/__init__.py @@ -7,4 +7,8 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + Submodules """ diff --git a/proxy/common/_compat.py b/proxy/common/_compat.py new file mode 100644 index 0000000000..c3ec75e411 --- /dev/null +++ b/proxy/common/_compat.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. + + Compatibility code for using Proxy.py across various versions of Python. + + .. spelling:: + + compat + py +""" + +import platform + + +SYS_PLATFORM = platform.system() +IS_WINDOWS = SYS_PLATFORM == 'Windows' diff --git a/proxy/common/_scm_version.pyi b/proxy/common/_scm_version.pyi new file mode 100644 index 0000000000..4b96455246 --- /dev/null +++ b/proxy/common/_scm_version.pyi @@ -0,0 +1,6 @@ +# This stub file is necessary because `_scm_version.py` +# autogenerated on build and absent on mypy checks time +from typing import Tuple, Union + +version: str +version_tuple: Tuple[Union[int, str], ...] diff --git a/proxy/common/_version.py b/proxy/common/_version.py new file mode 100644 index 0000000000..cea99cdfa7 --- /dev/null +++ b/proxy/common/_version.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. + + Version definition. +""" +from typing import Tuple, Union + +try: + # pylint: disable=unused-import + from ._scm_version import version as __version__, version_tuple as _ver_tup # noqa: WPS433, WPS436 +except ImportError: + from pkg_resources import get_distribution as _get_dist # noqa: WPS433 + __version__ = _get_dist('proxy.py').version # noqa: WPS440 + + +def _to_int_or_str(inp: str) -> Union[int, str]: + try: + return int(inp) + except ValueError: + return inp + + +def _split_version_parts(inp: str) -> Tuple[str, ...]: + public_version, _plus, local_version = inp.partition('+') + return (*public_version.split('.'), local_version) + + +try: + VERSION = _ver_tup +except NameError: + VERSION = tuple( + map(_to_int_or_str, _split_version_parts(__version__)), + ) + + +__all__ = '__version__', 'VERSION' diff --git a/proxy/common/backports.py b/proxy/common/backports.py new file mode 100644 index 0000000000..b6f5dfd927 --- /dev/null +++ b/proxy/common/backports.py @@ -0,0 +1,118 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import time +import threading + +from typing import Any, Deque +from queue import Empty +from collections import deque + + +class cached_property: + """Decorator for read-only properties evaluated only once within TTL period. + It can be used to create a cached property like this:: + + import random + + # the class containing the property must be a new-style class + class MyClass: + # create property whose value is cached for ten minutes + @cached_property(ttl=600) + def randint(self): + # will only be evaluated every 10 min. at maximum. + return random.randint(0, 100) + + The value is cached in the '_cached_properties' attribute of the object instance that + has the property getter method wrapped by this decorator. The '_cached_properties' + attribute value is a dictionary which has a key for every property of the + object which is wrapped by this decorator. Each entry in the cache is + created only when the property is accessed for the first time and is a + two-element tuple with the last computed property value and the last time + it was updated in seconds since the epoch. + + The default time-to-live (TTL) is 300 seconds (5 minutes). Set the TTL to + zero for the cached value to never expire. + + To expire a cached property value manually just do:: + del instance._cached_properties[] + + Adopted from https://wiki.python.org/moin/PythonDecoratorLibrary#Cached_Properties + ยฉ 2011 Christopher Arndt, MIT License. + + NOTE: We need this function only because Python in-built are only available + for 3.8+. Hence, we must get rid of this function once proxy.py no longer + support version older than 3.8. + + .. spelling:: + + backports + getter + Arndt + """ + + def __init__(self, ttl: float = 300.0): + self.ttl = ttl + + def __call__(self, fget: Any, doc: Any = None) -> 'cached_property': + self.fget = fget + self.__doc__ = doc or fget.__doc__ + self.__name__ = fget.__name__ + self.__module__ = fget.__module__ + return self + + def __get__(self, inst: Any, owner: Any) -> Any: + now = time.time() + try: + value, last_update = inst._cached_properties[self.__name__] + if self.ttl > 0 and now - last_update > self.ttl: # noqa: WPS333 + raise AttributeError + except (KeyError, AttributeError): + value = self.fget(inst) + try: + cache = inst._cached_properties + except AttributeError: + cache, inst._cached_properties = {}, {} + finally: + cache[self.__name__] = (value, now) + return value + + +class NonBlockingQueue: + '''Simple, unbounded, non-blocking FIFO queue. + + Supports only a single consumer. + + NOTE: This is available in Python since 3.7 as SimpleQueue. + Here because proxy.py still supports 3.6 + ''' + + def __init__(self) -> None: + self._queue: Deque[Any] = deque() + self._count: threading.Semaphore = threading.Semaphore(0) + + def put(self, item: Any) -> None: + '''Put the item on the queue.''' + self._queue.append(item) + self._count.release() + + def get(self) -> Any: + '''Remove and return an item from the queue.''' + if not self._count.acquire(False, None): + raise Empty + return self._queue.popleft() + + def empty(self) -> bool: + '''Return True if the queue is empty, False otherwise (not reliable!).''' + return len(self._queue) == 0 + + def qsize(self) -> int: + '''Return the approximate size of the queue (not reliable!).''' + return len(self._queue) diff --git a/proxy/common/constants.py b/proxy/common/constants.py index 21e3955089..c941676f0d 100644 --- a/proxy/common/constants.py +++ b/proxy/common/constants.py @@ -9,25 +9,41 @@ :license: BSD, see LICENSE for more details. """ import os +import sys import time +import secrets import pathlib +import sysconfig import ipaddress -from typing import List +from typing import Any, List +from ._compat import IS_WINDOWS # noqa: WPS436 from .version import __version__ + +def _env_threadless_compliant() -> bool: + """Returns true for Python 3.8+ across all platforms + except Windows.""" + return not IS_WINDOWS and sys.version_info >= (3, 8) + + PROXY_PY_START_TIME = time.time() # /path/to/proxy.py/proxy folder PROXY_PY_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) +# Path to virtualenv/lib/python3.X/site-packages +PROXY_PY_SITE_PACKAGES = sysconfig.get_path('purelib') +assert PROXY_PY_SITE_PACKAGES + CRLF = b'\r\n' COLON = b':' WHITESPACE = b' ' COMMA = b',' DOT = b'.' SLASH = b'/' +HTTP_1_0 = b'HTTP/1.0' HTTP_1_1 = b'HTTP/1.1' PROXY_AGENT_HEADER_KEY = b'Proxy-agent' @@ -45,7 +61,9 @@ DEFAULT_CA_KEY_FILE = None DEFAULT_CA_SIGNING_KEY_FILE = None DEFAULT_CERT_FILE = None -DEFAULT_CA_FILE = None +DEFAULT_CA_FILE = pathlib.Path( + PROXY_PY_SITE_PACKAGES, +) / 'certifi' / 'cacert.pem' DEFAULT_CLIENT_RECVBUF_SIZE = DEFAULT_BUFFER_SIZE DEFAULT_DEVTOOLS_WS_PATH = b'/devtools' DEFAULT_DISABLE_HEADERS: List[bytes] = [] @@ -60,32 +78,60 @@ DEFAULT_IPV6_HOSTNAME = ipaddress.IPv6Address('::1') DEFAULT_KEY_FILE = None DEFAULT_LOG_FILE = None -DEFAULT_LOG_FORMAT = '%(asctime)s - pid:%(process)d [%(levelname)-.1s] %(funcName)s:%(lineno)d - %(message)s' +DEFAULT_LOG_FORMAT = '%(asctime)s - pid:%(process)d [%(levelname)-.1s] %(module)s.%(funcName)s:%(lineno)d - %(message)s' DEFAULT_LOG_LEVEL = 'INFO' +DEFAULT_WEB_ACCESS_LOG_FORMAT = '{client_addr} - {request_method} {request_path} - {connection_time_ms}ms' +DEFAULT_HTTP_ACCESS_LOG_FORMAT = '{client_ip}:{client_port} - ' + \ + '{request_method} {server_host}:{server_port}{request_path} - ' + \ + '{response_code} {response_reason} - {response_bytes} bytes - ' + \ + '{connection_time_ms}ms' +DEFAULT_HTTPS_ACCESS_LOG_FORMAT = '{client_ip}:{client_port} - ' + \ + '{request_method} {server_host}:{server_port} - ' + \ + '{response_bytes} bytes - {connection_time_ms}ms' +DEFAULT_NUM_ACCEPTORS = 0 DEFAULT_NUM_WORKERS = 0 DEFAULT_OPEN_FILE_LIMIT = 1024 DEFAULT_PAC_FILE = None DEFAULT_PAC_FILE_URL_PATH = b'/' DEFAULT_PID_FILE = None -DEFAULT_PLUGINS = '' +DEFAULT_PLUGINS: List[Any] = [] DEFAULT_PORT = 8899 DEFAULT_SERVER_RECVBUF_SIZE = DEFAULT_BUFFER_SIZE DEFAULT_STATIC_SERVER_DIR = os.path.join(PROXY_PY_DIR, "public") -DEFAULT_THREADLESS = False -DEFAULT_TIMEOUT = 10 +DEFAULT_MIN_COMPRESSION_LIMIT = 20 # In bytes +DEFAULT_THREADLESS = _env_threadless_compliant() +DEFAULT_LOCAL_EXECUTOR = False +DEFAULT_TIMEOUT = 10.0 DEFAULT_VERSION = False DEFAULT_HTTP_PORT = 80 +DEFAULT_HTTPS_PORT = 443 DEFAULT_MAX_SEND_SIZE = 16 * 1024 +DEFAULT_WORK_KLASS = 'proxy.http.HttpProtocolHandler' +DEFAULT_ENABLE_PROXY_PROTOCOL = False +# 25 milliseconds to keep the loops hot +# Will consume ~0.3-0.6% CPU when idle. +DEFAULT_SELECTOR_SELECT_TIMEOUT = 25 / 1000 +DEFAULT_INACTIVE_CONN_CLEANUP_TIMEOUT = 1 # in seconds + +DEFAULT_DEVTOOLS_DOC_URL = 'http://proxy' +DEFAULT_DEVTOOLS_FRAME_ID = secrets.token_hex(8) +DEFAULT_DEVTOOLS_LOADER_ID = secrets.token_hex(8) DEFAULT_DATA_DIRECTORY_PATH = os.path.join(str(pathlib.Path.home()), '.proxy') # Cor plugins enabled by default or via flags +DEFAULT_ABC_PLUGINS = [ + 'HttpProtocolHandlerPlugin', + 'HttpProxyBasePlugin', + 'HttpWebServerBasePlugin', + 'ProxyDashboardWebsocketPlugin', +] PLUGIN_HTTP_PROXY = 'proxy.http.proxy.HttpProxyPlugin' PLUGIN_WEB_SERVER = 'proxy.http.server.HttpWebServerPlugin' PLUGIN_PAC_FILE = 'proxy.http.server.HttpWebServerPacFilePlugin' PLUGIN_DEVTOOLS_PROTOCOL = 'proxy.http.inspector.DevtoolsProtocolPlugin' -PLUGIN_DASHBOARD = 'proxy.dashboard.dashboard.ProxyDashboard' -PLUGIN_INSPECT_TRAFFIC = 'proxy.dashboard.inspect_traffic.InspectTrafficPlugin' +PLUGIN_DASHBOARD = 'proxy.dashboard.ProxyDashboard' +PLUGIN_INSPECT_TRAFFIC = 'proxy.dashboard.InspectTrafficPlugin' PLUGIN_PROXY_AUTH = 'proxy.http.proxy.AuthPlugin' PY2_DEPRECATION_MESSAGE = '''DEPRECATION: proxy.py no longer supports Python 2.7. Kindly upgrade to Python 3+. ' diff --git a/proxy/common/flag.py b/proxy/common/flag.py index 13ae8e6f37..d4b7da733d 100644 --- a/proxy/common/flag.py +++ b/proxy/common/flag.py @@ -8,27 +8,60 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ +import os +import sys +import base64 +import socket import argparse -from typing import Optional, List, Any +import ipaddress +import collections +import multiprocessing + +from typing import Optional, List, Any, cast + +from ._compat import IS_WINDOWS # noqa: WPS436 +from .plugins import Plugins +from .types import IpAddress +from .utils import bytes_, is_py2, set_open_file_limit +from .constants import COMMA, DEFAULT_DATA_DIRECTORY_PATH, DEFAULT_NUM_ACCEPTORS, DEFAULT_NUM_WORKERS +from .constants import DEFAULT_DEVTOOLS_WS_PATH, DEFAULT_DISABLE_HEADERS, PY2_DEPRECATION_MESSAGE +from .constants import PLUGIN_DASHBOARD, PLUGIN_DEVTOOLS_PROTOCOL, DEFAULT_MIN_COMPRESSION_LIMIT +from .constants import PLUGIN_HTTP_PROXY, PLUGIN_INSPECT_TRAFFIC, PLUGIN_PAC_FILE +from .constants import PLUGIN_WEB_SERVER, PLUGIN_PROXY_AUTH +from .logger import Logger from .version import __version__ __homepage__ = 'https://github.com/abhinavsingh/proxy.py' +# TODO: Currently `initialize` staticmethod contains knowledge +# about several common flags defined by proxy.py core. + +# This logic must be decoupled. flags.add_argument must +# also provide a callback to resolve the final flag value +# based upon availability in input_args, **opts and +# default values. + +# Supporting such a framework is complex but achievable. +# One problem is that resolution of certain flags +# can depend upon availability of other flags. + +# This will lead us into dependency graph modeling domain. class FlagParser: """Wrapper around argparse module. - proxy.py core and plugin classes must import `flag.flags` and - use `add_argument` to define their own flags within respective - class files. + Import `flag.flags` and use `add_argument` API + to define custom flags within respective Python files. + + Best Practice:: + + 1. Define flags at the top of your class files. + 2. DO NOT add flags within your class `__init__` method OR + within class methods. It MAY result into runtime exception, + especially if your class is initialized multiple times or if + class method registering the flag gets invoked multiple times. - Best Practice: - 1. Define flags at the top of your class files. - 2. DO NOT add flags within your class `__init__` method OR - within class methods. It MAY result into runtime exception, - especially if your class is initialized multiple times or if - class method registering the flag gets invoked multiple times. """ def __init__(self) -> None: @@ -36,7 +69,7 @@ def __init__(self) -> None: self.actions: List[str] = [] self.parser = argparse.ArgumentParser( description='proxy.py v%s' % __version__, - epilog='Proxy.py not working? Report at: %s/issues/new' % __homepage__ + epilog='Proxy.py not working? Report at: %s/issues/new' % __homepage__, ) def add_argument(self, *args: Any, **kwargs: Any) -> argparse.Action: @@ -46,10 +79,312 @@ def add_argument(self, *args: Any, **kwargs: Any) -> argparse.Action: return action def parse_args( - self, input_args: Optional[List[str]]) -> argparse.Namespace: + self, input_args: Optional[List[str]], + ) -> argparse.Namespace: """Parse flags from input arguments.""" self.args = self.parser.parse_args(input_args) return self.args + @staticmethod + def initialize( + input_args: Optional[List[str]] = None, + **opts: Any, + ) -> argparse.Namespace: + if input_args is None: + input_args = [] + + if is_py2(): + print(PY2_DEPRECATION_MESSAGE) + sys.exit(1) + + # Discover flags from requested plugin. + # This will also surface external plugin flags + # under --help. + Plugins.discover(input_args) + + # Parse flags + args = flags.parse_args(input_args) + + # Print version and exit + if args.version: + print(__version__) + sys.exit(0) + + # proxy.py currently cannot serve over HTTPS and also perform TLS interception + # at the same time. Check if user is trying to enable both feature + # at the same time. + # + # TODO: Use parser.add_mutually_exclusive_group() + # and remove this logic from here. + if (args.cert_file and args.key_file) and \ + (args.ca_key_file and args.ca_cert_file and args.ca_signing_key_file): + print( + 'You can either enable end-to-end encryption OR TLS interception,' + 'not both together.', + ) + sys.exit(1) + + # Setup logging module + Logger.setup(args.log_file, args.log_level, args.log_format) + + # Setup limits + set_open_file_limit(args.open_file_limit) + + # Load work_klass + work_klass = opts.get('work_klass', args.work_klass) + work_klass = Plugins.importer(bytes_(work_klass))[0] \ + if isinstance(work_klass, str) \ + else work_klass + + # Generate auth_code required for basic authentication if enabled + auth_code = None + basic_auth = opts.get('basic_auth', args.basic_auth) + # Destroy passed credentials via flags or options + args.basic_auth = None + if 'basic_auth' in opts: + del opts['basic_auth'] + + # Resolve auth module. + auth_plugins = [] + auth_plugin = opts.get('auth_plugin', args.auth_plugin) + if basic_auth: + auth_code = base64.b64encode(bytes_(basic_auth)) + if basic_auth or auth_plugin != PLUGIN_PROXY_AUTH: + # No basic auth provided + # Here auth_plugin is set to default plugin + # We want to avoid loading the auth plugin (w/o basic auth) + # unless user overrides the default auth plugin. + auth_plugins.append(auth_plugin) + + # Load default plugins along with user provided --plugins + default_plugins = [ + bytes_(p) + for p in FlagParser.get_default_plugins(args) + ] + requested_plugins = Plugins.resolve_plugin_flag( + args.plugins, opts.get('plugins', None), + ) + plugins = Plugins.load( + default_plugins + auth_plugins + requested_plugins, + ) + + # https://github.com/python/mypy/issues/5865 + # + # def option(t: object, key: str, default: Any) -> Any: + # return cast(t, opts.get(key, default)) + args.work_klass = work_klass + args.plugins = plugins + args.auth_code = cast( + Optional[bytes], + opts.get( + 'auth_code', + auth_code, + ), + ) + args.server_recvbuf_size = cast( + int, + opts.get( + 'server_recvbuf_size', + args.server_recvbuf_size, + ), + ) + args.client_recvbuf_size = cast( + int, + opts.get( + 'client_recvbuf_size', + args.client_recvbuf_size, + ), + ) + args.pac_file = cast( + Optional[str], opts.get( + 'pac_file', bytes_( + args.pac_file, + ), + ), + ) + args.pac_file_url_path = cast( + Optional[bytes], opts.get( + 'pac_file_url_path', bytes_( + args.pac_file_url_path, + ), + ), + ) + disabled_headers = cast( + Optional[List[bytes]], opts.get( + 'disable_headers', [ + header.lower() + for header in bytes_(args.disable_headers).split(COMMA) + if header.strip() != b'' + ], + ), + ) + args.disable_headers = disabled_headers if disabled_headers is not None else DEFAULT_DISABLE_HEADERS + args.certfile = cast( + Optional[str], opts.get( + 'cert_file', args.cert_file, + ), + ) + args.keyfile = cast(Optional[str], opts.get('key_file', args.key_file)) + args.ca_key_file = cast( + Optional[str], opts.get( + 'ca_key_file', args.ca_key_file, + ), + ) + args.ca_cert_file = cast( + Optional[str], opts.get( + 'ca_cert_file', args.ca_cert_file, + ), + ) + args.ca_signing_key_file = cast( + Optional[str], + opts.get( + 'ca_signing_key_file', + args.ca_signing_key_file, + ), + ) + args.ca_file = cast( + Optional[str], + opts.get( + 'ca_file', + args.ca_file, + ), + ) + args.hostname = cast( + IpAddress, + opts.get('hostname', ipaddress.ip_address(args.hostname)), + ) + args.unix_socket_path = opts.get( + 'unix_socket_path', args.unix_socket_path, + ) + # AF_UNIX is not available on Windows + # See https://bugs.python.org/issue33408 + if not IS_WINDOWS: + args.family = socket.AF_UNIX if args.unix_socket_path else ( + socket.AF_INET6 if args.hostname.version == 6 else socket.AF_INET + ) + else: + # FIXME: Not true for tests, as this value will be a mock. + # + # It's a problem only on Windows. Instead of a proper + # fix in the tests, simply commenting this line of assertion + # for now. + # + # assert args.unix_socket_path is None + args.family = socket.AF_INET6 if args.hostname.version == 6 else socket.AF_INET + args.port = cast(int, opts.get('port', args.port)) + args.backlog = cast(int, opts.get('backlog', args.backlog)) + num_workers = opts.get('num_workers', args.num_workers) + args.num_workers = cast( + int, num_workers if num_workers > 0 else multiprocessing.cpu_count(), + ) + num_acceptors = opts.get('num_acceptors', args.num_acceptors) + # See https://github.com/abhinavsingh/proxy.py/pull/714 description + # to understand rationale behind the following logic. + # + # --num-workers flag or option was found. We will use + # the same value for num_acceptors when --num-acceptors flag + # is absent. + if num_workers != DEFAULT_NUM_WORKERS and num_acceptors == DEFAULT_NUM_ACCEPTORS: + args.num_acceptors = args.num_workers + else: + args.num_acceptors = cast( + int, num_acceptors if num_acceptors > 0 else multiprocessing.cpu_count(), + ) + args.static_server_dir = cast( + str, + opts.get( + 'static_server_dir', + args.static_server_dir, + ), + ) + args.enable_static_server = cast( + bool, + opts.get( + 'enable_static_server', + args.enable_static_server, + ), + ) + args.min_compression_limit = cast( + bool, + opts.get( + 'min_compression_limit', + getattr( + args, 'min_compression_limit', + DEFAULT_MIN_COMPRESSION_LIMIT, + ), + ), + ) + args.devtools_ws_path = cast( + bytes, + opts.get( + 'devtools_ws_path', + getattr(args, 'devtools_ws_path', DEFAULT_DEVTOOLS_WS_PATH), + ), + ) + args.timeout = cast(int, opts.get('timeout', args.timeout)) + args.threadless = cast(bool, opts.get('threadless', args.threadless)) + args.threaded = cast(bool, opts.get('threaded', args.threaded)) + args.enable_events = cast( + bool, + opts.get( + 'enable_events', + args.enable_events, + ), + ) + args.pid_file = cast( + Optional[str], opts.get( + 'pid_file', + args.pid_file, + ), + ) + args.local_executor = cast( + bool, + opts.get( + 'local_executor', + args.local_executor, + ), + ) + + args.proxy_py_data_dir = DEFAULT_DATA_DIRECTORY_PATH + os.makedirs(args.proxy_py_data_dir, exist_ok=True) + + ca_cert_dir = opts.get('ca_cert_dir', args.ca_cert_dir) + args.ca_cert_dir = cast(Optional[str], ca_cert_dir) + if args.ca_cert_dir is None: + args.ca_cert_dir = os.path.join( + args.proxy_py_data_dir, 'certificates', + ) + os.makedirs(args.ca_cert_dir, exist_ok=True) + + return args + + @staticmethod + def get_default_plugins( + args: argparse.Namespace, + ) -> List[str]: + """Prepare list of plugins to load based upon + --enable-* and --disable-* flags. + """ + default_plugins: List[str] = [] + if hasattr(args, 'enable_dashboard') and args.enable_dashboard: + default_plugins.append(PLUGIN_WEB_SERVER) + args.enable_static_server = True + default_plugins.append(PLUGIN_DASHBOARD) + default_plugins.append(PLUGIN_INSPECT_TRAFFIC) + args.enable_events = True + args.enable_devtools = True + if hasattr(args, 'enable_devtools') and args.enable_devtools: + default_plugins.append(PLUGIN_DEVTOOLS_PROTOCOL) + default_plugins.append(PLUGIN_WEB_SERVER) + if not args.disable_http_proxy: + default_plugins.append(PLUGIN_HTTP_PROXY) + if args.enable_web_server or \ + args.pac_file is not None or \ + args.enable_static_server: + default_plugins.append(PLUGIN_WEB_SERVER) + if args.pac_file is not None: + default_plugins.append(PLUGIN_PAC_FILE) + return list(collections.OrderedDict.fromkeys(default_plugins).keys()) + flags = FlagParser() diff --git a/proxy/common/logger.py b/proxy/common/logger.py new file mode 100644 index 0000000000..675deeae98 --- /dev/null +++ b/proxy/common/logger.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import logging + +from typing import Optional, Any + +from .constants import DEFAULT_LOG_FILE, DEFAULT_LOG_FORMAT, DEFAULT_LOG_LEVEL + +SINGLE_CHAR_TO_LEVEL = { + 'D': 'DEBUG', + 'I': 'INFO', + 'W': 'WARNING', + 'E': 'ERROR', + 'C': 'CRITICAL', +} + + +def single_char_to_level(char: str) -> Any: + return getattr(logging, SINGLE_CHAR_TO_LEVEL[char.upper()[0]]) + + +class Logger: + """Common logging utilities and setup.""" + + @staticmethod + def setup( + log_file: Optional[str] = DEFAULT_LOG_FILE, + log_level: str = DEFAULT_LOG_LEVEL, + log_format: str = DEFAULT_LOG_FORMAT, + ) -> None: + if log_file: + logging.basicConfig( + filename=log_file, + filemode='a', + level=single_char_to_level(log_level), + format=log_format, + ) + else: + logging.basicConfig( + level=single_char_to_level(log_level), + format=log_format, + ) diff --git a/proxy/common/pki.py b/proxy/common/pki.py index 80711fd4c6..2a4395e904 100644 --- a/proxy/common/pki.py +++ b/proxy/common/pki.py @@ -7,16 +7,21 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + pki """ -import time +import os import sys +import uuid +import time +import logging +import tempfile import argparse import contextlib -import os -import uuid import subprocess -import tempfile -import logging + from typing import List, Generator, Optional, Tuple from .utils import bytes_ @@ -57,13 +62,14 @@ def remove_passphrase( key_in_path: str, password: str, key_out_path: str, - timeout: int = 10) -> bool: + timeout: int = 10, +) -> bool: """Remove passphrase from a private key.""" command = [ 'openssl', 'rsa', '-passin', 'pass:%s' % password, '-in', key_in_path, - '-out', key_out_path + '-out', key_out_path, ] return run_openssl_command(command, timeout) @@ -72,12 +78,13 @@ def gen_private_key( key_path: str, password: str, bits: int = 2048, - timeout: int = 10) -> bool: + timeout: int = 10, +) -> bool: """Generates a private key.""" command = [ 'openssl', 'genrsa', '-aes256', '-passout', 'pass:%s' % password, - '-out', key_path, str(bits) + '-out', key_path, str(bits), ] return run_openssl_command(command, timeout) @@ -90,7 +97,8 @@ def gen_public_key( alt_subj_names: Optional[List[str]] = None, extended_key_usage: Optional[str] = None, validity_in_days: int = 365, - timeout: int = 10) -> bool: + timeout: int = 10, +) -> bool: """For a given private key, generates a corresponding public key.""" with ssl_config(alt_subj_names, extended_key_usage) as (config_path, has_extension): command = [ @@ -98,7 +106,7 @@ def gen_public_key( '-days', str(validity_in_days), '-subj', subject, '-passin', 'pass:%s' % private_key_password, '-config', config_path, - '-key', private_key_path, '-out', public_key_path + '-key', private_key_path, '-out', public_key_path, ] if has_extension: command.extend([ @@ -112,13 +120,14 @@ def gen_csr( key_path: str, password: str, crt_path: str, - timeout: int = 10) -> bool: + timeout: int = 10, +) -> bool: """Generates a CSR based upon existing certificate and key file.""" command = [ 'openssl', 'x509', '-x509toreq', '-passin', 'pass:%s' % password, '-in', crt_path, '-signkey', key_path, - '-out', csr_path + '-out', csr_path, ] return run_openssl_command(command, timeout) @@ -133,7 +142,8 @@ def sign_csr( alt_subj_names: Optional[List[str]] = None, extended_key_usage: Optional[str] = None, validity_in_days: int = 365, - timeout: int = 10) -> bool: + timeout: int = 10, +) -> bool: """Sign a CSR using CA key and certificate.""" with ext_file(alt_subj_names, extended_key_usage) as extension_path: command = [ @@ -152,7 +162,8 @@ def sign_csr( def get_ext_config( alt_subj_names: Optional[List[str]] = None, - extended_key_usage: Optional[str] = None) -> bytes: + extended_key_usage: Optional[str] = None, +) -> bytes: config = b'' # Add SAN extension if alt_subj_names is not None and len(alt_subj_names) > 0: @@ -169,12 +180,14 @@ def get_ext_config( @contextlib.contextmanager def ext_file( alt_subj_names: Optional[List[str]] = None, - extended_key_usage: Optional[str] = None) -> Generator[str, None, None]: + extended_key_usage: Optional[str] = None, +) -> Generator[str, None, None]: # Write config to temp file config_path = os.path.join(tempfile.gettempdir(), uuid.uuid4().hex) with open(config_path, 'wb') as cnf: cnf.write( - get_ext_config(alt_subj_names, extended_key_usage)) + get_ext_config(alt_subj_names, extended_key_usage), + ) yield config_path @@ -185,7 +198,8 @@ def ext_file( @contextlib.contextmanager def ssl_config( alt_subj_names: Optional[List[str]] = None, - extended_key_usage: Optional[str] = None) -> Generator[Tuple[str, bool], None, None]: + extended_key_usage: Optional[str] = None, +) -> Generator[Tuple[str, bool], None, None]: config = DEFAULT_CONFIG has_extension = False @@ -212,7 +226,7 @@ def run_openssl_command(command: List[str], timeout: int) -> bool: cmd = subprocess.Popen( command, stdout=subprocess.PIPE, - stderr=subprocess.PIPE + stderr=subprocess.PIPE, ) cmd.communicate(timeout=timeout) return cmd.returncode == 0 @@ -221,7 +235,7 @@ def run_openssl_command(command: List[str], timeout: int) -> bool: if __name__ == '__main__': available_actions = ( 'remove_passphrase', 'gen_private_key', 'gen_public_key', - 'gen_csr', 'sign_csr' + 'gen_csr', 'sign_csr', ) parser = argparse.ArgumentParser( @@ -231,7 +245,7 @@ def run_openssl_command(command: List[str], timeout: int) -> bool: 'action', type=str, default=None, - help='Valid actions: ' + ', '.join(available_actions) + help='Valid actions: ' + ', '.join(available_actions), ) parser.add_argument( '--password', @@ -279,32 +293,44 @@ def run_openssl_command(command: List[str], timeout: int) -> bool: # Validation if args.action not in available_actions: - print('Invalid --action. Valid values ' + ', '.join(available_actions)) + logger.error( + 'Invalid --action. Valid values ' + + ', '.join(available_actions), + ) sys.exit(1) if args.action in ('gen_private_key', 'gen_public_key'): if args.private_key_path is None: - print('--private-key-path is required for ' + args.action) + logger.error('--private-key-path is required for ' + args.action) sys.exit(1) if args.action == 'gen_public_key': if args.public_key_path is None: - print('--public-key-file is required for private key generation') + logger.error( + '--public-key-file is required for private key generation', + ) sys.exit(1) # Execute if args.action == 'gen_private_key': gen_private_key(args.private_key_path, args.password) elif args.action == 'gen_public_key': - gen_public_key(args.public_key_path, args.private_key_path, - args.password, args.subject) + gen_public_key( + args.public_key_path, args.private_key_path, + args.password, args.subject, + ) elif args.action == 'remove_passphrase': - remove_passphrase(args.private_key_path, args.password, - args.private_key_path) + remove_passphrase( + args.private_key_path, args.password, + args.private_key_path, + ) elif args.action == 'gen_csr': gen_csr( args.csr_path, args.private_key_path, args.password, - args.public_key_path) + args.public_key_path, + ) elif args.action == 'sign_csr': - sign_csr(args.csr_path, args.crt_path, args.private_key_path, args.password, - args.public_key_path, str(int(time.time())), alt_subj_names=[args.hostname, ]) + sign_csr( + args.csr_path, args.crt_path, args.private_key_path, args.password, + args.public_key_path, str(int(time.time())), alt_subj_names=[args.hostname], + ) diff --git a/proxy/common/plugins.py b/proxy/common/plugins.py new file mode 100644 index 0000000000..b2236145a2 --- /dev/null +++ b/proxy/common/plugins.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import os +import abc +import logging +import inspect +import itertools +import importlib + +from typing import Any, List, Dict, Optional, Tuple, Union + +from .utils import bytes_, text_ +from .constants import DOT, DEFAULT_ABC_PLUGINS, COMMA + +logger = logging.getLogger(__name__) + + +class Plugins: + """Common utilities for plugin discovery.""" + + @staticmethod + def resolve_plugin_flag(flag_plugins: Any, opt_plugins: Optional[Any] = None) -> List[Union[bytes, type]]: + if isinstance(flag_plugins, list): + requested_plugins = list( + itertools.chain.from_iterable([ + p.split(text_(COMMA)) for p in list( + itertools.chain.from_iterable(flag_plugins), + ) + ]), + ) + else: + requested_plugins = flag_plugins.split(text_(COMMA)) + return [ + p if isinstance(p, type) else bytes_(p) + for p in (opt_plugins if opt_plugins is not None else requested_plugins) + if not (isinstance(p, str) and len(p) == 0) + ] + + @staticmethod + def discover(input_args: List[str]) -> None: + """Search for external plugin found in command line arguments, + then iterates over each value and discover/import the plugin. + """ + for i, f in enumerate(input_args): + if f in ('--plugin', '--plugins', '--auth-plugin'): + v = input_args[i + 1] + parts = v.split(',') + for part in parts: + Plugins.importer(bytes_(part)) + + @staticmethod + def load( + plugins: List[Union[bytes, type]], + abc_plugins: Optional[List[str]] = None, + ) -> Dict[bytes, List[type]]: + """Accepts a list Python modules, scans them to identify + if they are an implementation of abstract plugin classes and + returns a dictionary of matching plugins for each abstract class. + """ + p: Dict[bytes, List[type]] = {} + for abc_plugin in (abc_plugins or DEFAULT_ABC_PLUGINS): + p[bytes_(abc_plugin)] = [] + for plugin_ in plugins: + klass, module_name = Plugins.importer(plugin_) + assert klass and module_name + mro = list(inspect.getmro(klass)) + mro.reverse() + iterator = iter(mro) + while next(iterator) is not abc.ABC: + pass + base_klass = next(iterator) + if klass not in p[bytes_(base_klass.__name__)]: + p[bytes_(base_klass.__name__)].append(klass) + logger.info('Loaded plugin %s.%s', module_name, klass.__name__) + return p + + @staticmethod + def importer(plugin: Union[bytes, type]) -> Tuple[type, str]: + """Import and returns the plugin.""" + if isinstance(plugin, type): + return (plugin, '__main__') + plugin_ = text_(plugin.strip()) + assert plugin_ != '' + module_name, klass_name = plugin_.rsplit(text_(DOT), 1) + klass = getattr( + importlib.import_module( + module_name.replace( + os.path.sep, text_(DOT), + ), + ), + klass_name, + ) + return (klass, module_name) diff --git a/proxy/common/types.py b/proxy/common/types.py index 279211e422..a95a49e8d2 100644 --- a/proxy/common/types.py +++ b/proxy/common/types.py @@ -10,10 +10,17 @@ """ import queue import ipaddress +import sys from typing import TYPE_CHECKING, Dict, Any, List, Union -from typing_extensions import Protocol +# NOTE: Using try/except causes linting problems which is why it's necessary +# NOTE: to use this mypy/pylint idiom for py36-py38 compatibility +# Ref: https://github.com/python/typeshed/issues/3500#issuecomment-560958608 +if sys.version_info >= (3, 8): + from typing import Protocol +else: + from typing_extensions import Protocol if TYPE_CHECKING: DictQueueType = queue.Queue[Dict[str, Any]] # pragma: no cover diff --git a/proxy/common/utils.py b/proxy/common/utils.py index 203a17aee2..673b73d1ba 100644 --- a/proxy/common/utils.py +++ b/proxy/common/utils.py @@ -7,17 +7,46 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + utils + websocket + Websocket + WebSocket """ +import sys import ssl -import contextlib +import socket +import logging import functools import ipaddress -import socket +import contextlib from types import TracebackType from typing import Optional, Dict, Any, List, Tuple, Type, Callable -from .constants import HTTP_1_1, COLON, WHITESPACE, CRLF, DEFAULT_TIMEOUT +from ._compat import IS_WINDOWS # noqa: WPS436 +from .constants import HTTP_1_1, COLON, WHITESPACE, CRLF, DEFAULT_TIMEOUT, DEFAULT_THREADLESS + +if not IS_WINDOWS: + import resource + +logger = logging.getLogger(__name__) + + +def is_threadless(threadless: bool, threaded: bool) -> bool: + # if default is threadless then return true unless + # user has overridden mode using threaded flag. + # + # if default is not threadless then return true + # only if user has overridden using --threadless flag + return (DEFAULT_THREADLESS and not threaded) or (not DEFAULT_THREADLESS and threadless) + + +def is_py2() -> bool: + """Exists only to avoid mocking :data:`sys.version_info` in tests.""" + return sys.version_info.major == 2 def text_(s: Any, encoding: str = 'utf-8', errors: str = 'strict') -> Any: @@ -44,22 +73,27 @@ def bytes_(s: Any, encoding: str = 'utf-8', errors: str = 'strict') -> Any: return s -def build_http_request(method: bytes, url: bytes, - protocol_version: bytes = HTTP_1_1, - headers: Optional[Dict[bytes, bytes]] = None, - body: Optional[bytes] = None) -> bytes: +def build_http_request( + method: bytes, url: bytes, + protocol_version: bytes = HTTP_1_1, + headers: Optional[Dict[bytes, bytes]] = None, + body: Optional[bytes] = None, +) -> bytes: """Build and returns a HTTP request packet.""" if headers is None: headers = {} return build_http_pkt( - [method, url, protocol_version], headers, body) + [method, url, protocol_version], headers, body, + ) -def build_http_response(status_code: int, - protocol_version: bytes = HTTP_1_1, - reason: Optional[bytes] = None, - headers: Optional[Dict[bytes, bytes]] = None, - body: Optional[bytes] = None) -> bytes: +def build_http_response( + status_code: int, + protocol_version: bytes = HTTP_1_1, + reason: Optional[bytes] = None, + headers: Optional[Dict[bytes, bytes]] = None, + body: Optional[bytes] = None, +) -> bytes: """Build and returns a HTTP response packet.""" line = [protocol_version, bytes_(status_code)] if reason: @@ -85,25 +119,28 @@ def build_http_header(k: bytes, v: bytes) -> bytes: return k + COLON + WHITESPACE + v -def build_http_pkt(line: List[bytes], - headers: Optional[Dict[bytes, bytes]] = None, - body: Optional[bytes] = None) -> bytes: +def build_http_pkt( + line: List[bytes], + headers: Optional[Dict[bytes, bytes]] = None, + body: Optional[bytes] = None, +) -> bytes: """Build and returns a HTTP request or response packet.""" - req = WHITESPACE.join(line) + CRLF + pkt = WHITESPACE.join(line) + CRLF if headers is not None: for k in headers: - req += build_http_header(k, headers[k]) + CRLF - req += CRLF + pkt += build_http_header(k, headers[k]) + CRLF + pkt += CRLF if body: - req += body - return req + pkt += body + return pkt def build_websocket_handshake_request( key: bytes, method: bytes = b'GET', url: bytes = b'/', - host: bytes = b'localhost') -> bytes: + host: bytes = b'localhost', +) -> bytes: """ Build and returns a Websocket handshake request packet. @@ -119,7 +156,7 @@ def build_websocket_handshake_request( b'Upgrade': b'websocket', b'Sec-WebSocket-Key': key, b'Sec-WebSocket-Version': b'13', - } + }, ) @@ -134,8 +171,8 @@ def build_websocket_handshake_response(accept: bytes) -> bytes: headers={ b'Upgrade': b'websocket', b'Connection': b'Upgrade', - b'Sec-WebSocket-Accept': accept - } + b'Sec-WebSocket-Accept': accept, + }, ) @@ -143,23 +180,26 @@ def find_http_line(raw: bytes) -> Tuple[Optional[bytes], bytes]: """Find and returns first line ending in CRLF along with following buffer. If no ending CRLF is found, line is None.""" - pos = raw.find(CRLF) - if pos == -1: - return None, raw - line = raw[:pos] - rest = raw[pos + len(CRLF):] - return line, rest + parts = raw.split(CRLF, 1) + return (None, raw) \ + if len(parts) == 1 \ + else (parts[0], parts[1]) -def wrap_socket(conn: socket.socket, keyfile: str, - certfile: str) -> ssl.SSLSocket: +def wrap_socket( + conn: socket.socket, keyfile: str, + certfile: str, +) -> ssl.SSLSocket: + """Use this to upgrade server_side socket to TLS.""" ctx = ssl.create_default_context( - ssl.Purpose.CLIENT_AUTH) + ssl.Purpose.CLIENT_AUTH, + ) ctx.options |= ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1 ctx.verify_mode = ssl.CERT_NONE ctx.load_cert_chain( certfile=certfile, - keyfile=keyfile) + keyfile=keyfile, + ) return ctx.wrap_socket( conn, server_side=True, @@ -167,18 +207,23 @@ def wrap_socket(conn: socket.socket, keyfile: str, def new_socket_connection( - addr: Tuple[str, int], timeout: int = DEFAULT_TIMEOUT) -> socket.socket: + addr: Tuple[str, int], + timeout: float = DEFAULT_TIMEOUT, + source_address: Optional[Tuple[str, int]] = None, +) -> socket.socket: conn = None try: ip = ipaddress.ip_address(addr[0]) if ip.version == 4: conn = socket.socket( - socket.AF_INET, socket.SOCK_STREAM, 0) + socket.AF_INET, socket.SOCK_STREAM, 0, + ) conn.settimeout(timeout) conn.connect(addr) else: conn = socket.socket( - socket.AF_INET6, socket.SOCK_STREAM, 0) + socket.AF_INET6, socket.SOCK_STREAM, 0, + ) conn.settimeout(timeout) conn.connect((addr[0], addr[1], 0, 0)) except ValueError: @@ -188,7 +233,7 @@ def new_socket_connection( return conn # try to establish dual stack IPv4/IPv6 connection. - return socket.create_connection(addr, timeout=timeout) + return socket.create_connection(addr, timeout=timeout, source_address=source_address) class socket_connection(contextlib.ContextDecorator): @@ -207,12 +252,14 @@ def __exit__( self, exc_type: Optional[Type[BaseException]], exc_val: Optional[BaseException], - exc_tb: Optional[TracebackType]) -> None: + exc_tb: Optional[TracebackType], + ) -> None: if self.conn: self.conn.close() def __call__( # type: ignore - self, func: Callable[..., Any]) -> Callable[[Tuple[Any, ...], Dict[str, Any]], Any]: + self, func: Callable[..., Any], + ) -> Callable[[Tuple[Any, ...], Dict[str, Any]], Any]: @functools.wraps(func) def decorated(*args: Any, **kwargs: Any) -> Any: with self as conn: @@ -226,3 +273,20 @@ def get_available_port() -> int: sock.bind(('', 0)) _, port = sock.getsockname() return int(port) + + +def set_open_file_limit(soft_limit: int) -> None: + """Configure open file description soft limit on supported OS.""" + if IS_WINDOWS: # resource module not available on Windows OS + return + + curr_soft_limit, curr_hard_limit = resource.getrlimit( + resource.RLIMIT_NOFILE, + ) + if curr_soft_limit < soft_limit < curr_hard_limit: + resource.setrlimit( + resource.RLIMIT_NOFILE, (soft_limit, curr_hard_limit), + ) + logger.debug( + 'Open file soft limit set to %d', soft_limit, + ) diff --git a/proxy/common/version.py b/proxy/common/version.py index 585f155569..6940317403 100644 --- a/proxy/common/version.py +++ b/proxy/common/version.py @@ -8,5 +8,7 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -VERSION = (2, 3, 1) -__version__ = '.'.join(map(str, VERSION[0:3])) +from ._version import __version__, VERSION # noqa: WPS436 + + +__all__ = '__version__', 'VERSION' diff --git a/proxy/core/__init__.py b/proxy/core/__init__.py index 232621f0b5..ae3ea4267f 100644 --- a/proxy/core/__init__.py +++ b/proxy/core/__init__.py @@ -7,4 +7,8 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + Subpackages """ diff --git a/proxy/core/acceptor/__init__.py b/proxy/core/acceptor/__init__.py index cca3bcdb4d..577e2022f5 100644 --- a/proxy/core/acceptor/__init__.py +++ b/proxy/core/acceptor/__init__.py @@ -7,15 +7,30 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + acceptor + acceptors + pre + Submodules """ from .acceptor import Acceptor from .pool import AcceptorPool from .work import Work from .threadless import Threadless +from .remote import RemoteExecutor +from .local import LocalExecutor +from .executors import ThreadlessPool +from .listener import Listener __all__ = [ 'Acceptor', 'AcceptorPool', 'Work', 'Threadless', + 'RemoteExecutor', + 'LocalExecutor', + 'ThreadlessPool', + 'Listener', ] diff --git a/proxy/core/acceptor/acceptor.py b/proxy/core/acceptor/acceptor.py index 3a9ca61828..05cfa09c26 100644 --- a/proxy/core/acceptor/acceptor.py +++ b/proxy/core/acceptor/acceptor.py @@ -7,151 +7,221 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + acceptor + pre """ -import argparse +import socket import logging -import multiprocessing -import multiprocessing.synchronize +import argparse import selectors -import socket import threading +import multiprocessing +import multiprocessing.synchronize from multiprocessing import connection -from multiprocessing.reduction import send_handle, recv_handle -from typing import Optional, Type, Tuple +from multiprocessing.reduction import recv_handle -from .work import Work -from .threadless import Threadless +from typing import List, Optional, Tuple -from ..connection import TcpClientConnection -from ..event import EventQueue, eventNames -from ...common.constants import DEFAULT_THREADLESS from ...common.flag import flags +from ...common.utils import is_threadless +from ...common.logger import Logger +from ...common.backports import NonBlockingQueue +from ...common.constants import DEFAULT_LOCAL_EXECUTOR + +from ..event import EventQueue + +from .local import LocalExecutor +from .executors import ThreadlessPool logger = logging.getLogger(__name__) flags.add_argument( - '--threadless', + '--local-executor', action='store_true', - default=DEFAULT_THREADLESS, - help='Default: False. When disabled a new thread is spawned ' - 'to handle each client connection.' + default=DEFAULT_LOCAL_EXECUTOR, + help='Default: ' + ('True' if DEFAULT_LOCAL_EXECUTOR else 'False') + '. ' + + 'Disabled by default. When enabled acceptors will make use of ' + + 'local (same process) executor instead of distributing load across ' + + 'remote (other process) executors. Enable this option to achieve CPU affinity between ' + + 'acceptors and executors, instead of using underlying OS kernel scheduling algorithm.', ) class Acceptor(multiprocessing.Process): - """Socket server acceptor process. + """Work acceptor process. - Accepts a server socket fd over `work_queue` and start listening for client - connections over the passed server socket. By default, it spawns a separate thread - to handle each client request. + On start-up, `Acceptor` accepts a file descriptor which will be used to + accept new work. File descriptor is accepted over a `fd_queue`. - However, if `--threadless` option is enabled, Acceptor process will also pre-spawns a `Threadless` - process at startup. Accepted client connections are then passed to the `Threadless` process - which internally uses asyncio event loop to handle client connections. + `Acceptor` goes on to listen for new work over the received server socket. + By default, `Acceptor` will spawn a new thread to handle each work. - TODO(abhinavsingh): Instead of starting `Threadless` process, can we work with a `Threadless` thread? - What are the performance implications of sharing fds between threads vs processes? How much performance - degradation happen when processes are running on separate CPU cores? + However, when ``--threadless`` option is enabled without ``--local-executor``, + `Acceptor` process will also pre-spawns a + :class:`~proxy.core.acceptor.threadless.Threadless` process during start-up. + Accepted work is delegated to these :class:`~proxy.core.acceptor.threadless.Threadless` + processes. `Acceptor` process shares accepted work with a + :class:`~proxy.core.acceptor.threadless.Threadless` process over it's dedicated pipe. """ def __init__( self, idd: int, - work_queue: connection.Connection, + fd_queue: connection.Connection, flags: argparse.Namespace, - work_klass: Type[Work], lock: multiprocessing.synchronize.Lock, - event_queue: Optional[EventQueue] = None) -> None: + executor_queues: List[connection.Connection], + executor_pids: List[int], + executor_locks: List[multiprocessing.synchronize.Lock], + event_queue: Optional[EventQueue] = None, + ) -> None: super().__init__() - self.idd = idd - self.work_queue: connection.Connection = work_queue self.flags = flags - self.work_klass = work_klass - self.lock = lock + # Eventing core queue self.event_queue = event_queue - + # Index assigned by `AcceptorPool` + self.idd = idd + # Mutex used for synchronization with acceptors + self.lock = lock + # Queue over which server socket fd is received on start-up + self.fd_queue: connection.Connection = fd_queue + # Available executors + self.executor_queues = executor_queues + self.executor_pids = executor_pids + self.executor_locks = executor_locks + # Selector self.running = multiprocessing.Event() self.selector: Optional[selectors.DefaultSelector] = None + # File descriptor used to accept new work + # Currently, a socket fd is assumed. self.sock: Optional[socket.socket] = None - self.threadless_process: Optional[Threadless] = None - self.threadless_client_queue: Optional[connection.Connection] = None - - def start_threadless_process(self) -> None: - pipe = multiprocessing.Pipe() - self.threadless_client_queue = pipe[0] - self.threadless_process = Threadless( - client_queue=pipe[1], - flags=self.flags, - work_klass=self.work_klass, - event_queue=self.event_queue - ) - self.threadless_process.start() - logger.debug('Started process %d', self.threadless_process.pid) - - def shutdown_threadless_process(self) -> None: - assert self.threadless_process and self.threadless_client_queue - logger.debug('Stopped process %d', self.threadless_process.pid) - self.threadless_process.running.set() - self.threadless_process.join() - self.threadless_client_queue.close() - - def start_work(self, conn: socket.socket, addr: Tuple[str, int]) -> None: - if self.flags.threadless and \ - self.threadless_client_queue and \ - self.threadless_process: - self.threadless_client_queue.send(addr) - send_handle( - self.threadless_client_queue, - conn.fileno(), - self.threadless_process.pid - ) - conn.close() - else: - work = self.work_klass( - TcpClientConnection(conn, addr), - flags=self.flags, - event_queue=self.event_queue - ) - work_thread = threading.Thread(target=work.run) - work_thread.daemon = True - work.publish_event( - event_name=eventNames.WORK_STARTED, - event_payload={'fileno': conn.fileno(), 'addr': addr}, - publisher_id=self.__class__.__name__ - ) - work_thread.start() + # Internals + self._total: Optional[int] = None + self._local_work_queue: Optional['NonBlockingQueue'] = None + self._local: Optional[LocalExecutor] = None + self._lthread: Optional[threading.Thread] = None + + def accept(self, events: List[Tuple[selectors.SelectorKey, int]]) -> None: + for _, mask in events: + if mask & selectors.EVENT_READ: + if self.sock is not None: + conn, addr = self.sock.accept() + logging.debug( + 'Accepting new work#{0}'.format(conn.fileno()), + ) + work = (conn, addr or None) + if self.flags.local_executor: + assert self._local_work_queue + self._local_work_queue.put(work) + else: + self._work(*work) def run_once(self) -> None: - with self.lock: - assert self.selector and self.sock + if self.selector is not None: events = self.selector.select(timeout=1) if len(events) == 0: return - conn, addr = self.sock.accept() - self.start_work(conn, addr) + locked = False + try: + if self.lock.acquire(block=False): + locked = True + self.accept(events) + except BlockingIOError: + pass + finally: + if locked: + self.lock.release() def run(self) -> None: + Logger.setup( + self.flags.log_file, self.flags.log_level, + self.flags.log_format, + ) self.selector = selectors.DefaultSelector() - fileno = recv_handle(self.work_queue) - self.work_queue.close() + # TODO: Use selector on fd_queue so that we can + # dynamically accept from new fds. + fileno = recv_handle(self.fd_queue) + self.fd_queue.close() + # TODO: Convert to socks i.e. list of fds self.sock = socket.fromfd( fileno, family=self.flags.family, - type=socket.SOCK_STREAM + type=socket.SOCK_STREAM, ) try: + if self.flags.local_executor: + self._start_local() self.selector.register(self.sock, selectors.EVENT_READ) - if self.flags.threadless: - self.start_threadless_process() while not self.running.is_set(): self.run_once() except KeyboardInterrupt: pass finally: self.selector.unregister(self.sock) - if self.flags.threadless: - self.shutdown_threadless_process() + if self.flags.local_executor: + self._stop_local() self.sock.close() logger.debug('Acceptor#%d shutdown', self.idd) + + def _start_local(self) -> None: + assert self.sock + self._local_work_queue = NonBlockingQueue() + self._local = LocalExecutor( + work_queue=self._local_work_queue, + flags=self.flags, + event_queue=self.event_queue, + ) + self._lthread = threading.Thread(target=self._local.run) + self._lthread.daemon = True + self._lthread.start() + + def _stop_local(self) -> None: + if self._lthread is not None and self._local_work_queue is not None: + self._local_work_queue.put(False) + self._lthread.join() + + def _work(self, conn: socket.socket, addr: Optional[Tuple[str, int]]) -> None: + self._total = self._total or 0 + if is_threadless(self.flags.threadless, self.flags.threaded): + # Index of worker to which this work should be dispatched + # Use round-robin strategy by default. + # + # By default all acceptors will start sending work to + # 1st workers. To randomize, we offset index by idd. + index = (self._total + self.idd) % self.flags.num_workers + thread = threading.Thread( + target=ThreadlessPool.delegate, + args=( + self.executor_pids[index], + self.executor_queues[index], + self.executor_locks[index], + conn, + addr, + self.flags.unix_socket_path, + ), + ) + thread.start() + logger.debug( + 'Dispatched work#{0}.{1}.{2} to worker#{3}'.format( + conn.fileno(), self.idd, self._total, index, + ), + ) + else: + _, thread = ThreadlessPool.start_threaded_work( + self.flags, + conn, + addr, + event_queue=self.event_queue, + publisher_id=self.__class__.__name__, + ) + logger.debug( + 'Started work#{0}.{1}.{2} in thread#{3}'.format( + conn.fileno(), self.idd, self._total, thread.ident, + ), + ) + self._total += 1 diff --git a/proxy/core/acceptor/executors.py b/proxy/core/acceptor/executors.py new file mode 100644 index 0000000000..065e78bded --- /dev/null +++ b/proxy/core/acceptor/executors.py @@ -0,0 +1,206 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. + + .. spelling:: + + acceptor +""" +import socket +import logging +import argparse +import threading +import multiprocessing + +from multiprocessing import connection +from multiprocessing.reduction import send_handle + +from typing import Any, Optional, List, Tuple + +from .work import Work +from .remote import RemoteExecutor + +from ..connection import TcpClientConnection +from ..event import EventQueue, eventNames + +from ...common.flag import flags +from ...common.utils import is_threadless +from ...common.constants import DEFAULT_NUM_WORKERS, DEFAULT_THREADLESS + +logger = logging.getLogger(__name__) + + +flags.add_argument( + '--threadless', + action='store_true', + default=DEFAULT_THREADLESS, + help='Default: ' + ('True' if DEFAULT_THREADLESS else 'False') + '. ' + + 'Enabled by default on Python 3.8+ (mac, linux). ' + + 'When disabled a new thread is spawned ' + 'to handle each client connection.', +) + +flags.add_argument( + '--threaded', + action='store_true', + default=not DEFAULT_THREADLESS, + help='Default: ' + ('True' if not DEFAULT_THREADLESS else 'False') + '. ' + + 'Disabled by default on Python < 3.8 and windows. ' + + 'When enabled a new thread is spawned ' + 'to handle each client connection.', +) + +flags.add_argument( + '--num-workers', + type=int, + default=DEFAULT_NUM_WORKERS, + help='Defaults to number of CPU cores.', +) + + +class ThreadlessPool: + """Manages lifecycle of threadless pool and delegates work to them + using a round-robin strategy. + + Example usage:: + + with ThreadlessPool(flags=...) as pool: + while True: + time.sleep(1) + + If necessary, start multiple threadless pool with different + work classes. + """ + + def __init__( + self, + flags: argparse.Namespace, + event_queue: Optional[EventQueue] = None, + ) -> None: + self.flags = flags + self.event_queue = event_queue + # Threadless worker communication states + self.work_queues: List[connection.Connection] = [] + self.work_pids: List[int] = [] + self.work_locks: List[multiprocessing.synchronize.Lock] = [] + # List of threadless workers + self._workers: List[RemoteExecutor] = [] + self._processes: List[multiprocessing.Process] = [] + + def __enter__(self) -> 'ThreadlessPool': + self.setup() + return self + + def __exit__(self, *args: Any) -> None: + self.shutdown() + + @staticmethod + def delegate( + worker_pid: int, + work_queue: connection.Connection, + work_lock: multiprocessing.synchronize.Lock, + conn: socket.socket, + addr: Optional[Tuple[str, int]], + unix_socket_path: Optional[str] = None, + ) -> None: + """Utility method to delegate a work to threadless executor pool.""" + with work_lock: + # Accepted client address is empty string for + # unix socket domain, avoid sending empty string + # for optimization. + if not unix_socket_path: + work_queue.send(addr) + send_handle( + work_queue, + conn.fileno(), + worker_pid, + ) + conn.close() + + @staticmethod + def start_threaded_work( + flags: argparse.Namespace, + conn: socket.socket, + addr: Optional[Tuple[str, int]], + event_queue: Optional[EventQueue] = None, + publisher_id: Optional[str] = None, + ) -> Tuple[Work, threading.Thread]: + """Utility method to start a work in a new thread.""" + work = flags.work_klass( + TcpClientConnection(conn, addr), + flags=flags, + event_queue=event_queue, + ) + # TODO: Keep reference to threads and join during shutdown. + # This will ensure connections are not abruptly closed on shutdown + # for threaded execution mode. + thread = threading.Thread(target=work.run) + thread.daemon = True + thread.start() + work.publish_event( + event_name=eventNames.WORK_STARTED, + event_payload={'fileno': conn.fileno(), 'addr': addr}, + publisher_id=publisher_id or 'thread#{0}'.format( + thread.ident, + ), + ) + return (work, thread) + + def setup(self) -> None: + """Setup threadless processes.""" + if is_threadless(self.flags.threadless, self.flags.threaded): + for index in range(self.flags.num_workers): + self._start_worker(index) + logger.info( + 'Started {0} threadless workers'.format( + self.flags.num_workers, + ), + ) + + def shutdown(self) -> None: + """Shutdown threadless processes.""" + if is_threadless(self.flags.threadless, self.flags.threaded): + self._shutdown_workers() + logger.info( + 'Stopped {0} threadless workers'.format( + self.flags.num_workers, + ), + ) + + def _start_worker(self, index: int) -> None: + """Starts a threadless worker.""" + self.work_locks.append(multiprocessing.Lock()) + pipe = multiprocessing.Pipe() + self.work_queues.append(pipe[0]) + w = RemoteExecutor( + work_queue=pipe[1], + flags=self.flags, + event_queue=self.event_queue, + ) + self._workers.append(w) + p = multiprocessing.Process(target=w.run) + # p.daemon = True + self._processes.append(p) + p.start() + assert p.pid + self.work_pids.append(p.pid) + logger.debug('Started threadless#%d process#%d', index, p.pid) + + def _shutdown_workers(self) -> None: + """Pop a running threadless worker and clean it up.""" + for index in range(self.flags.num_workers): + self._workers[index].running.set() + for _ in range(self.flags.num_workers): + pid = self.work_pids[-1] + self._processes.pop().join() + self._workers.pop() + self.work_pids.pop() + self.work_queues.pop().close() + logger.debug('Stopped threadless process#%d', pid) + self.work_locks = [] diff --git a/proxy/core/acceptor/listener.py b/proxy/core/acceptor/listener.py new file mode 100644 index 0000000000..bef4b4461f --- /dev/null +++ b/proxy/core/acceptor/listener.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. + + .. spelling:: + + acceptor +""" +import os +import socket +import logging +import argparse + +from typing import Optional, Any + +from ...common.flag import flags +from ...common.constants import DEFAULT_BACKLOG, DEFAULT_IPV6_HOSTNAME, DEFAULT_PORT + + +flags.add_argument( + '--backlog', + type=int, + default=DEFAULT_BACKLOG, + help='Default: 100. Maximum number of pending connections to proxy server', +) + +flags.add_argument( + '--hostname', + type=str, + default=str(DEFAULT_IPV6_HOSTNAME), + help='Default: ::1. Server IP address.', +) + +flags.add_argument( + '--port', type=int, default=DEFAULT_PORT, + help='Default: 8899. Server port.', +) + +flags.add_argument( + '--unix-socket-path', + type=str, + default=None, + help='Default: None. Unix socket path to use. ' + + 'When provided --host and --port flags are ignored', +) + +logger = logging.getLogger(__name__) + + +class Listener: + + def __init__(self, flags: argparse.Namespace) -> None: + self.flags = flags + # Set after binding to a port. + # Stored here separately because ephemeral ports can be used. + self._port: Optional[int] = None + self._socket: Optional[socket.socket] = None + + def __enter__(self) -> 'Listener': + self.setup() + return self + + def __exit__(self, *args: Any) -> None: + self.shutdown() + + def fileno(self) -> Optional[int]: + if not self._socket: + return None + return self._socket.fileno() + + def setup(self) -> None: + if self.flags.unix_socket_path: + self._listen_unix_socket() + else: + self._listen_server_port() + if self.flags.unix_socket_path: + logger.info( + 'Listening on %s' % + self.flags.unix_socket_path, + ) + else: + logger.info( + 'Listening on %s:%s' % + (self.flags.hostname, self._port), + ) + + def shutdown(self) -> None: + assert self._socket + self._socket.close() + if self.flags.unix_socket_path: + os.remove(self.flags.unix_socket_path) + + def _listen_unix_socket(self) -> None: + self._socket = socket.socket(self.flags.family, socket.SOCK_STREAM) + self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self._socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + self._socket.bind(self.flags.unix_socket_path) + self._socket.listen(self.flags.backlog) + self._socket.setblocking(False) + + def _listen_server_port(self) -> None: + self._socket = socket.socket(self.flags.family, socket.SOCK_STREAM) + self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self._socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + self._socket.bind((str(self.flags.hostname), self.flags.port)) + self._socket.listen(self.flags.backlog) + self._socket.setblocking(False) + self._port = self._socket.getsockname()[1] diff --git a/proxy/core/acceptor/local.py b/proxy/core/acceptor/local.py new file mode 100644 index 0000000000..bb4909815f --- /dev/null +++ b/proxy/core/acceptor/local.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. + + .. spelling:: + + acceptor + teardown +""" +import queue +import logging +import asyncio +import contextlib + +from typing import Optional +from typing import Any + +from ...common.backports import NonBlockingQueue # noqa: W0611, F401 pylint: disable=unused-import + +from .threadless import Threadless + +logger = logging.getLogger(__name__) + + +class LocalExecutor(Threadless['NonBlockingQueue']): + """A threadless executor implementation which uses a queue to receive new work.""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self._loop: Optional[asyncio.AbstractEventLoop] = None + + @property + def loop(self) -> Optional[asyncio.AbstractEventLoop]: + if self._loop is None: + self._loop = asyncio.new_event_loop() + return self._loop + + def work_queue_fileno(self) -> Optional[int]: + return None + + def receive_from_work_queue(self) -> bool: + with contextlib.suppress(queue.Empty): + work = self.work_queue.get() + if isinstance(work, bool) and work is False: + return True + assert isinstance(work, tuple) + conn, addr = work + # NOTE: Here we are assuming to receive a connection object + # and not a fileno because we are a LocalExecutor. + fileno = conn.fileno() + self.work_on_tcp_conn(fileno=fileno, addr=addr, conn=conn) + return False diff --git a/proxy/core/acceptor/pool.py b/proxy/core/acceptor/pool.py index 65b98c72ac..dd110d3285 100644 --- a/proxy/core/acceptor/pool.py +++ b/proxy/core/acceptor/pool.py @@ -7,179 +7,134 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + acceptor + acceptors + pre """ -import argparse import logging +import argparse import multiprocessing -import socket -import threading + from multiprocessing import connection from multiprocessing.reduction import send_handle -from typing import List, Optional, Type +from typing import Any, List, Optional + +from .listener import Listener from .acceptor import Acceptor -from .work import Work -from ..event import EventQueue, EventDispatcher +from ..event import EventQueue + from ...common.flag import flags -from ...common.constants import DEFAULT_BACKLOG, DEFAULT_ENABLE_EVENTS -from ...common.constants import DEFAULT_IPV6_HOSTNAME, DEFAULT_NUM_WORKERS, DEFAULT_PORT +from ...common.constants import DEFAULT_NUM_ACCEPTORS logger = logging.getLogger(__name__) -# Lock shared by worker processes -LOCK = multiprocessing.Lock() - flags.add_argument( - '--backlog', + '--num-acceptors', type=int, - default=DEFAULT_BACKLOG, - help='Default: 100. Maximum number of pending connections to proxy server') - -flags.add_argument( - '--enable-events', - action='store_true', - default=DEFAULT_ENABLE_EVENTS, - help='Default: False. Enables core to dispatch lifecycle events. ' - 'Plugins can be used to subscribe for core events.' + default=DEFAULT_NUM_ACCEPTORS, + help='Defaults to number of CPU cores.', ) -flags.add_argument( - '--hostname', - type=str, - default=str(DEFAULT_IPV6_HOSTNAME), - help='Default: ::1. Server IP address.') - -flags.add_argument( - '--port', type=int, default=DEFAULT_PORT, - help='Default: 8899. Server port.') - -flags.add_argument( - '--num-workers', - type=int, - default=DEFAULT_NUM_WORKERS, - help='Defaults to number of CPU cores.') - class AcceptorPool: - """AcceptorPool. + """AcceptorPool is a helper class which pre-spawns + :py:class:`~proxy.core.acceptor.acceptor.Acceptor` processes to + utilize all available CPU cores for accepting new work. - Pre-spawns worker processes to utilize all cores available on the system. - A server socket is initialized and dispatched over a pipe to these workers. - Each worker process then accepts new client connection. + A file descriptor to consume work from is shared with + :py:class:`~proxy.core.acceptor.acceptor.Acceptor` processes over a + pipe. Each :py:class:`~proxy.core.acceptor.acceptor.Acceptor` + process then concurrently accepts new work over the shared file + descriptor. Example usage: - pool = AcceptorPool(flags=..., work_klass=...) - try: - pool.setup() + with AcceptorPool(flags=...) as pool: while True: time.sleep(1) - finally: - pool.shutdown() - - `work_klass` must implement `work.Work` class. - Optionally, AcceptorPool also initialize a global event queue. - It is a multiprocess safe queue which can be used to build pubsub patterns - for message sharing or signaling within proxy.py. + `flags.work_klass` must implement `work.Work` class. """ - def __init__(self, flags: argparse.Namespace, - work_klass: Type[Work]) -> None: + def __init__( + self, + flags: argparse.Namespace, + listener: Listener, + executor_queues: List[connection.Connection], + executor_pids: List[int], + executor_locks: List[multiprocessing.synchronize.Lock], + event_queue: Optional[EventQueue] = None, + ) -> None: self.flags = flags - self.socket: Optional[socket.socket] = None + # File descriptor to use for accepting new work + self.listener: Listener = listener + # Available executors + self.executor_queues: List[connection.Connection] = executor_queues + self.executor_pids: List[int] = executor_pids + self.executor_locks: List[multiprocessing.synchronize.Lock] = executor_locks + # Eventing core queue + self.event_queue: Optional[EventQueue] = event_queue + # Acceptor process instances self.acceptors: List[Acceptor] = [] - self.work_queues: List[connection.Connection] = [] - self.work_klass = work_klass - - self.event_queue: Optional[EventQueue] = None - self.event_dispatcher: Optional[EventDispatcher] = None - self.event_dispatcher_thread: Optional[threading.Thread] = None - self.event_dispatcher_shutdown: Optional[threading.Event] = None - self.manager: Optional[multiprocessing.managers.SyncManager] = None - - if self.flags.enable_events: - self.manager = multiprocessing.Manager() - self.event_queue = EventQueue(self.manager.Queue()) - - def listen(self) -> None: - self.socket = socket.socket(self.flags.family, socket.SOCK_STREAM) - self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - self.socket.bind((str(self.flags.hostname), self.flags.port)) - self.socket.listen(self.flags.backlog) - self.socket.setblocking(False) - logger.info( - 'Listening on %s:%d' % - (self.flags.hostname, self.flags.port)) - - def start_workers(self) -> None: - """Start worker processes.""" - for acceptor_id in range(self.flags.num_workers): + # Fd queues used to share file descriptor with acceptor processes + self.fd_queues: List[connection.Connection] = [] + # Internals + self.lock = multiprocessing.Lock() + + def __enter__(self) -> 'AcceptorPool': + self.setup() + return self + + def __exit__(self, *args: Any) -> None: + self.shutdown() + + def setup(self) -> None: + """Setup acceptors.""" + self._start() + logger.info('Started %d acceptors' % self.flags.num_acceptors) + # Send file descriptor to all acceptor processes. + fd = self.listener.fileno() + for index in range(self.flags.num_acceptors): + send_handle( + self.fd_queues[index], + fd, + self.acceptors[index].pid, + ) + self.fd_queues[index].close() + + def shutdown(self) -> None: + logger.info('Shutting down %d acceptors' % self.flags.num_acceptors) + for acceptor in self.acceptors: + acceptor.running.set() + for acceptor in self.acceptors: + acceptor.join() + logger.debug('Acceptors shutdown') + + def _start(self) -> None: + """Start acceptor processes.""" + for acceptor_id in range(self.flags.num_acceptors): work_queue = multiprocessing.Pipe() acceptor = Acceptor( idd=acceptor_id, - work_queue=work_queue[1], + fd_queue=work_queue[1], flags=self.flags, - work_klass=self.work_klass, - lock=LOCK, + lock=self.lock, event_queue=self.event_queue, + executor_queues=self.executor_queues, + executor_pids=self.executor_pids, + executor_locks=self.executor_locks, ) acceptor.start() logger.debug( 'Started acceptor#%d process %d', acceptor_id, - acceptor.pid) - self.acceptors.append(acceptor) - self.work_queues.append(work_queue[0]) - logger.info('Started %d workers' % self.flags.num_workers) - - def start_event_dispatcher(self) -> None: - self.event_dispatcher_shutdown = threading.Event() - assert self.event_dispatcher_shutdown - assert self.event_queue - self.event_dispatcher = EventDispatcher( - shutdown=self.event_dispatcher_shutdown, - event_queue=self.event_queue - ) - self.event_dispatcher_thread = threading.Thread( - target=self.event_dispatcher.run - ) - self.event_dispatcher_thread.start() - logger.debug('Thread ID: %d', self.event_dispatcher_thread.ident) - - def shutdown(self) -> None: - logger.info('Shutting down %d workers' % self.flags.num_workers) - for acceptor in self.acceptors: - acceptor.running.set() - if self.flags.enable_events: - assert self.event_dispatcher_shutdown - assert self.event_dispatcher_thread - self.event_dispatcher_shutdown.set() - self.event_dispatcher_thread.join() - logger.debug( - 'Shutdown of global event dispatcher thread %d successful', - self.event_dispatcher_thread.ident) - for acceptor in self.acceptors: - acceptor.join() - logger.debug('Acceptors shutdown') - - def setup(self) -> None: - """Listen on port, setup workers and pass server socket to workers.""" - self.listen() - if self.flags.enable_events: - logger.info('Core Event enabled') - self.start_event_dispatcher() - self.start_workers() - - # Send server socket to all acceptor processes. - assert self.socket is not None - for index in range(self.flags.num_workers): - send_handle( - self.work_queues[index], - self.socket.fileno(), - self.acceptors[index].pid + acceptor.pid, ) - self.work_queues[index].close() - self.socket.close() + self.acceptors.append(acceptor) + self.fd_queues.append(work_queue[0]) diff --git a/proxy/core/acceptor/remote.py b/proxy/core/acceptor/remote.py new file mode 100644 index 0000000000..76f8877d21 --- /dev/null +++ b/proxy/core/acceptor/remote.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. + + .. spelling:: + + acceptor +""" +import asyncio +import logging + +from typing import Optional, Any + +from multiprocessing import connection +from multiprocessing.reduction import recv_handle + +from .threadless import Threadless + +logger = logging.getLogger(__name__) + + +class RemoteExecutor(Threadless[connection.Connection]): + """A threadless executor implementation which receives work over a connection. + + NOTE: RemoteExecutor uses ``recv_handle`` to accept file descriptors. + + TODO: Refactor and abstract ``recv_handle`` part so that a threaded + remote executor can also accept work over a connection. Currently, + remote executors must be running in a process. + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self._loop: Optional[asyncio.AbstractEventLoop] = None + + @property + def loop(self) -> Optional[asyncio.AbstractEventLoop]: + if self._loop is None: + self._loop = asyncio.get_event_loop_policy().get_event_loop() + return self._loop + + def work_queue_fileno(self) -> Optional[int]: + return self.work_queue.fileno() + + def close_work_queue(self) -> None: + self.work_queue.close() + + def receive_from_work_queue(self) -> bool: + # Acceptor will not send address for + # unix socket domain environments. + addr = None + if not self.flags.unix_socket_path: + addr = self.work_queue.recv() + fileno = recv_handle(self.work_queue) + self.work_on_tcp_conn(fileno=fileno, addr=addr) + return False diff --git a/proxy/core/acceptor/threadless.py b/proxy/core/acceptor/threadless.py index 87ef4aba71..cc9292c2e6 100644 --- a/proxy/core/acceptor/threadless.py +++ b/proxy/core/acceptor/threadless.py @@ -7,177 +7,343 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + acceptor """ -import argparse import os +import ssl import socket import logging import asyncio +import argparse import selectors -import contextlib import multiprocessing -from multiprocessing import connection -from multiprocessing.reduction import recv_handle -from typing import Dict, Optional, Tuple, List, Generator, Any, Type +from abc import abstractmethod, ABC +from typing import Dict, Optional, Tuple, List, Set, Generic, TypeVar, Union -from .work import Work +from ...common.logger import Logger +from ...common.types import Readables, Writables +from ...common.constants import DEFAULT_INACTIVE_CONN_CLEANUP_TIMEOUT, DEFAULT_SELECTOR_SELECT_TIMEOUT from ..connection import TcpClientConnection -from ..event import EventQueue, eventNames +from ..event import eventNames, EventQueue -from ...common.types import Readables, Writables -from ...common.constants import DEFAULT_TIMEOUT +from .work import Work + +T = TypeVar('T') logger = logging.getLogger(__name__) -class Threadless(multiprocessing.Process): - """Threadless provides an event loop. Use it by implementing Threadless class. +class Threadless(ABC, Generic[T]): + """Work executor base class. + + Threadless provides an event loop, which is shared across + multiple :class:`~proxy.core.acceptor.work.Work` instances to handle + work. + + Threadless takes input a `work_klass` and an `event_queue`. `work_klass` + must conform to the :class:`~proxy.core.acceptor.work.Work` + protocol. Work is received over the `event_queue`. - When --threadless option is enabled, each Acceptor process also - spawns one Threadless process. And instead of spawning new thread - for each accepted client connection, Acceptor process sends - accepted client connection to Threadless process over a pipe. + When a work is accepted, threadless creates a new instance of `work_klass`. + Threadless will then invoke necessary lifecycle of the + :class:`~proxy.core.acceptor.work.Work` protocol, + allowing `work_klass` implementation to handle the assigned work. - Example, HttpProtocolHandler implements Work class to hooks into the - event loop provided by Threadless process. + Example, :class:`~proxy.core.base.tcp_server.BaseTcpServerHandler` + implements :class:`~proxy.core.acceptor.work.Work` protocol. It + expects a client connection as work payload and hooks into the + threadless event loop to handle the client connection. """ def __init__( self, - client_queue: connection.Connection, + work_queue: T, flags: argparse.Namespace, - work_klass: Type[Work], - event_queue: Optional[EventQueue] = None) -> None: + event_queue: Optional[EventQueue] = None, + ) -> None: super().__init__() - self.client_queue = client_queue + self.work_queue = work_queue self.flags = flags - self.work_klass = work_klass self.event_queue = event_queue self.running = multiprocessing.Event() self.works: Dict[int, Work] = {} self.selector: Optional[selectors.DefaultSelector] = None - self.loop: Optional[asyncio.AbstractEventLoop] = None - - @contextlib.contextmanager - def selected_events(self) -> Generator[Tuple[Readables, Writables], - None, None]: - events: Dict[socket.socket, int] = {} - for work in self.works.values(): - events.update(work.get_events()) - assert self.selector is not None - for fd in events: - self.selector.register(fd, events[fd]) - ev = self.selector.select(timeout=1) - readables = [] - writables = [] - for key, mask in ev: - if mask & selectors.EVENT_READ: - readables.append(key.fileobj) - if mask & selectors.EVENT_WRITE: - writables.append(key.fileobj) - yield (readables, writables) - for fd in events.keys(): - self.selector.unregister(fd) - - async def handle_events( - self, fileno: int, - readables: Readables, - writables: Writables) -> bool: - return self.works[fileno].handle_events(readables, writables) - - # TODO: Use correct future typing annotations - async def wait_for_tasks( - self, tasks: Dict[int, Any]) -> None: - for work_id in tasks: - # TODO: Resolving one handle_events here can block resolution of - # other tasks - try: - teardown = await asyncio.wait_for(tasks[work_id], DEFAULT_TIMEOUT) - if teardown: - self.cleanup(work_id) - except asyncio.TimeoutError: - self.cleanup(work_id) - - def fromfd(self, fileno: int) -> socket.socket: - return socket.fromfd( - fileno, family=socket.AF_INET if self.flags.hostname.version == 4 else socket.AF_INET6, - type=socket.SOCK_STREAM) + # If we remove single quotes for typing hint below, + # runtime exceptions will occur for < Python 3.9. + # + # Ref https://github.com/abhinavsingh/proxy.py/runs/4279055360?check_suite_focus=true + self.unfinished: Set['asyncio.Task[bool]'] = set() + self.registered_events_by_work_ids: Dict[ + # work_id + int, + # fileno, mask + Dict[int, int], + ] = {} + self.wait_timeout: float = DEFAULT_SELECTOR_SELECT_TIMEOUT + self.cleanup_inactive_timeout: float = DEFAULT_INACTIVE_CONN_CLEANUP_TIMEOUT + + @property + @abstractmethod + def loop(self) -> Optional[asyncio.AbstractEventLoop]: + raise NotImplementedError() + + @abstractmethod + def receive_from_work_queue(self) -> bool: + """Work queue is ready to receive new work. + + Receive it and call ``work_on_tcp_conn``. + + Return True to tear down the loop.""" + raise NotImplementedError() + + @abstractmethod + def work_queue_fileno(self) -> Optional[int]: + """If work queue must be selected before calling + ``receive_from_work_queue`` then implementation must + return work queue fd.""" + raise NotImplementedError() - def accept_client(self) -> None: - addr = self.client_queue.recv() - fileno = recv_handle(self.client_queue) - self.works[fileno] = self.work_klass( - TcpClientConnection(conn=self.fromfd(fileno), addr=addr), + def close_work_queue(self) -> None: + """Only called if ``work_queue_fileno`` returns an integer. + If an fd is select-able for work queue, make sure + to close the work queue fd now.""" + pass # pragma: no cover + + def work_on_tcp_conn( + self, + fileno: int, + addr: Optional[Tuple[str, int]] = None, + conn: Optional[Union[ssl.SSLSocket, socket.socket]] = None, + ) -> None: + self.works[fileno] = self.flags.work_klass( + TcpClientConnection( + conn=conn or self._fromfd(fileno), + addr=addr, + ), flags=self.flags, - event_queue=self.event_queue + event_queue=self.event_queue, ) self.works[fileno].publish_event( event_name=eventNames.WORK_STARTED, event_payload={'fileno': fileno, 'addr': addr}, - publisher_id=self.__class__.__name__ + publisher_id=self.__class__.__name__, ) try: self.works[fileno].initialize() except Exception as e: logger.exception( 'Exception occurred during initialization', - exc_info=e) - self.cleanup(fileno) + exc_info=e, + ) + self._cleanup(fileno) + + async def _selected_events(self) -> Tuple[ + Dict[int, Tuple[Readables, Writables]], + bool, + ]: + """For each work, collects events they are interested in. + Calls select for events of interest. """ + assert self.selector is not None + for work_id in self.works: + worker_events = await self.works[work_id].get_events() + # NOTE: Current assumption is that multiple works will not + # be interested in the same fd. Descriptors of interests + # returned by work must be unique. + # + # TODO: Ideally we must diff and unregister socks not + # returned of interest within this _select_events call + # but exists in registered_socks_by_work_ids + for fileno in worker_events: + if work_id not in self.registered_events_by_work_ids: + self.registered_events_by_work_ids[work_id] = {} + mask = worker_events[fileno] + if fileno in self.registered_events_by_work_ids[work_id]: + oldmask = self.registered_events_by_work_ids[work_id][fileno] + if mask != oldmask: + self.selector.modify( + fileno, events=mask, + data=work_id, + ) + self.registered_events_by_work_ids[work_id][fileno] = mask + logger.debug( + 'fd#{0} modified for mask#{1} by work#{2}'.format( + fileno, mask, work_id, + ), + ) + else: + # Can throw ValueError: Invalid file descriptor: -1 + # + # A guard within Work classes may not help here due to + # asynchronous nature. Hence, threadless will handle + # ValueError exceptions raised by selector.register + # for invalid fd. + self.selector.register( + fileno, events=mask, + data=work_id, + ) + self.registered_events_by_work_ids[work_id][fileno] = mask + logger.debug( + 'fd#{0} registered for mask#{1} by work#{2}'.format( + fileno, mask, work_id, + ), + ) + selected = self.selector.select( + timeout=DEFAULT_SELECTOR_SELECT_TIMEOUT, + ) + # Keys are work_id and values are 2-tuple indicating + # readables & writables that work_id is interested in + # and are ready for IO. + work_by_ids: Dict[int, Tuple[Readables, Writables]] = {} + new_work_available = False + wqfileno = self.work_queue_fileno() + if wqfileno is None: + new_work_available = True + for key, mask in selected: + if wqfileno is not None and key.fileobj == wqfileno: + assert mask & selectors.EVENT_READ + new_work_available = True + continue + if key.data not in work_by_ids: + work_by_ids[key.data] = ([], []) + if mask & selectors.EVENT_READ: + work_by_ids[key.data][0].append(key.fileobj) + if mask & selectors.EVENT_WRITE: + work_by_ids[key.data][1].append(key.fileobj) + return (work_by_ids, new_work_available) + + async def _wait_for_tasks(self) -> None: + finished, self.unfinished = await asyncio.wait( + self.unfinished, + timeout=self.wait_timeout, + return_when=asyncio.FIRST_COMPLETED, + ) + for task in finished: + if task.result(): + self._cleanup(task._work_id) # type: ignore + # self.cleanup(int(task.get_name())) + + def _fromfd(self, fileno: int) -> socket.socket: + return socket.fromfd( + fileno, family=socket.AF_INET if self.flags.hostname.version == 4 else socket.AF_INET6, + type=socket.SOCK_STREAM, + ) - def cleanup_inactive(self) -> None: + def _cleanup_inactive(self) -> None: inactive_works: List[int] = [] for work_id in self.works: if self.works[work_id].is_inactive(): inactive_works.append(work_id) for work_id in inactive_works: - self.cleanup(work_id) + self._cleanup(work_id) - def cleanup(self, work_id: int) -> None: - # TODO: HttpProtocolHandler.shutdown can call flush which may block + # TODO: HttpProtocolHandler.shutdown can call flush which may block + def _cleanup(self, work_id: int) -> None: + if work_id in self.registered_events_by_work_ids: + assert self.selector + for fileno in self.registered_events_by_work_ids[work_id]: + logger.debug( + 'fd#{0} unregistered by work#{1}'.format( + fileno, work_id, + ), + ) + self.selector.unregister(fileno) + self.registered_events_by_work_ids[work_id].clear() + del self.registered_events_by_work_ids[work_id] self.works[work_id].shutdown() del self.works[work_id] - os.close(work_id) + if self.work_queue_fileno() is not None: + os.close(work_id) - def run_once(self) -> None: + def _create_tasks( + self, + work_by_ids: Dict[int, Tuple[Readables, Writables]], + ) -> Set['asyncio.Task[bool]']: + assert self.loop + tasks: Set['asyncio.Task[bool]'] = set() + for work_id in work_by_ids: + task = self.loop.create_task( + self.works[work_id].handle_events(*work_by_ids[work_id]), + ) + task._work_id = work_id # type: ignore[attr-defined] + # task.set_name(work_id) + tasks.add(task) + return tasks + + async def _run_once(self) -> bool: assert self.loop is not None - with self.selected_events() as (readables, writables): - if len(readables) == 0 and len(writables) == 0: - # Remove and shutdown inactive connections - self.cleanup_inactive() - return - # Note that selector from now on is idle, - # until all the logic below completes. + work_by_ids, new_work_available = await self._selected_events() + # Accept new work if available # + # TODO: We must use a work klass to handle + # client_queue fd itself a.k.a. accept_client + # will become handle_readables. + if new_work_available: + teardown = self.receive_from_work_queue() + if teardown: + return teardown + if len(work_by_ids) == 0: + return False # Invoke Threadless.handle_events - # TODO: Only send readable / writables that client originally - # registered. - tasks = {} - for fileno in self.works: - tasks[fileno] = self.loop.create_task( - self.handle_events(fileno, readables, writables)) - # Accepted client connection from Acceptor - if self.client_queue in readables: - self.accept_client() - # Wait for Threadless.handle_events to complete - self.loop.run_until_complete(self.wait_for_tasks(tasks)) - # Remove and shutdown inactive connections - self.cleanup_inactive() + self.unfinished.update(self._create_tasks(work_by_ids)) + # logger.debug('Executing {0} works'.format(len(self.unfinished))) + await self._wait_for_tasks() + # logger.debug( + # 'Done executing works, {0} pending, {1} registered'.format( + # len(self.unfinished), len(self.registered_events_by_work_ids), + # ), + # ) + return False + + async def _run_forever(self) -> None: + tick = 0 + try: + while True: + if await self._run_once(): + break + # Check for inactive and shutdown signal only second + if (tick * DEFAULT_SELECTOR_SELECT_TIMEOUT) > self.cleanup_inactive_timeout: + self._cleanup_inactive() + if self.running.is_set(): + break + tick = 0 + tick += 1 + except KeyboardInterrupt: + pass + finally: + if self.loop: + self.loop.stop() def run(self) -> None: + Logger.setup( + self.flags.log_file, self.flags.log_level, + self.flags.log_format, + ) + wqfileno = self.work_queue_fileno() try: self.selector = selectors.DefaultSelector() - self.selector.register(self.client_queue, selectors.EVENT_READ) - self.loop = asyncio.get_event_loop() - while not self.running.is_set(): - self.run_once() + if wqfileno is not None: + self.selector.register( + wqfileno, + selectors.EVENT_READ, + data=wqfileno, + ) + assert self.loop + # logger.debug('Working on {0} works'.format(len(self.works))) + self.loop.create_task(self._run_forever()) + self.loop.run_forever() except KeyboardInterrupt: pass finally: assert self.selector is not None - self.selector.unregister(self.client_queue) - self.client_queue.close() + if wqfileno is not None: + self.selector.unregister(wqfileno) + self.close_work_queue() assert self.loop is not None + self.loop.run_until_complete(self.loop.shutdown_asyncgens()) self.loop.close() diff --git a/proxy/core/acceptor/work.py b/proxy/core/acceptor/work.py index 6bf3880ecf..11b5deecc6 100644 --- a/proxy/core/acceptor/work.py +++ b/proxy/core/acceptor/work.py @@ -7,9 +7,12 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + acceptor """ import argparse -import socket from abc import ABC, abstractmethod from uuid import uuid4, UUID @@ -25,25 +28,30 @@ class Work(ABC): def __init__( self, - client: TcpClientConnection, + work: TcpClientConnection, flags: argparse.Namespace, event_queue: Optional[EventQueue] = None, - uid: Optional[UUID] = None) -> None: - self.client = client + uid: Optional[UUID] = None, + ) -> None: + # Work uuid + self.uid: UUID = uid if uid is not None else uuid4() self.flags = flags + # Eventing core queue self.event_queue = event_queue - self.uid: UUID = uid if uid is not None else uuid4() + # Accept work + self.work = work @abstractmethod - def get_events(self) -> Dict[socket.socket, int]: + async def get_events(self) -> Dict[int, int]: """Return sockets and events (read or write) that we are interested in.""" return {} # pragma: no cover @abstractmethod - def handle_events( + async def handle_events( self, readables: Readables, - writables: Writables) -> bool: + writables: Writables, + ) -> bool: """Handle readable and writable sockets. Return True to shutdown work.""" @@ -63,7 +71,7 @@ def shutdown(self) -> None: self.publish_event( event_name=eventNames.WORK_FINISHED, event_payload={}, - publisher_id=self.__class__.__name__ + publisher_id=self.__class__.__name__, ) def run(self) -> None: @@ -71,13 +79,14 @@ def run(self) -> None: compatibility with threaded mode where work class is started as a separate thread. """ - pass + pass # pragma: no cover def publish_event( self, event_name: int, event_payload: Dict[str, Any], - publisher_id: Optional[str] = None) -> None: + publisher_id: Optional[str] = None, + ) -> None: """Convenience method provided to publish events into the global event queue.""" if not self.flags.enable_events: return @@ -86,5 +95,5 @@ def publish_event( self.uid.hex, event_name, event_payload, - publisher_id + publisher_id, ) diff --git a/proxy/core/base/__init__.py b/proxy/core/base/__init__.py index c60f14778e..8a307776d0 100644 --- a/proxy/core/base/__init__.py +++ b/proxy/core/base/__init__.py @@ -7,11 +7,17 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + Submodules """ from .tcp_server import BaseTcpServerHandler from .tcp_tunnel import BaseTcpTunnelHandler +from .tcp_upstream import TcpUpstreamConnectionHandler __all__ = [ 'BaseTcpServerHandler', 'BaseTcpTunnelHandler', + 'TcpUpstreamConnectionHandler', ] diff --git a/proxy/core/base/tcp_server.py b/proxy/core/base/tcp_server.py index bff043113b..ce1d476116 100644 --- a/proxy/core/base/tcp_server.py +++ b/proxy/core/base/tcp_server.py @@ -7,100 +7,130 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + tcp """ -from abc import abstractmethod -import socket +import logging import selectors +from abc import abstractmethod from typing import Dict, Any, Optional -from proxy.core.acceptor import Work -from proxy.common.types import Readables, Writables +from ...core.acceptor import Work +from ...common.types import Readables, Writables + +logger = logging.getLogger(__name__) class BaseTcpServerHandler(Work): """BaseTcpServerHandler implements Work interface. + BaseTcpServerHandler lifecycle is controlled by Threadless core + using asyncio. If you want to also support threaded mode, also + implement the optional run() method from Work class. + An instance of BaseTcpServerHandler is created for each client - connection. BaseServerHandler lifecycle is controlled by - Threadless core using asyncio. + connection. BaseTcpServerHandler ensures that server is always + ready to accept new data from the client. It also ensures, client + is ready to accept new data before flushing data to it. - BaseServerHandler ensures that pending buffers are flushed - before client connection is closed. + Most importantly, BaseTcpServerHandler ensures that pending buffers + to the client are flushed before connection is closed. - Implementations must provide: - a) handle_data(data: memoryview) - c) (optionally) intialize, is_inactive and shutdown methods + Implementations must provide:: + + a. handle_data(data: memoryview) implementation + b. Optionally, also implement other Work method + e.g. initialize, is_inactive, shutdown """ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) self.must_flush_before_shutdown = False - print('Connection accepted from {0}'.format(self.client.addr)) + logger.debug( + 'Work#%d accepted from %s', + self.work.connection.fileno(), + self.work.address, + ) @abstractmethod def handle_data(self, data: memoryview) -> Optional[bool]: """Optionally return True to close client connection.""" pass # pragma: no cover - def get_events(self) -> Dict[socket.socket, int]: + async def get_events(self) -> Dict[int, int]: events = {} # We always want to read from client # Register for EVENT_READ events if self.must_flush_before_shutdown is False: - events[self.client.connection] = selectors.EVENT_READ + events[self.work.connection.fileno()] = selectors.EVENT_READ # If there is pending buffer for client # also register for EVENT_WRITE events - if self.client.has_buffer(): - if self.client.connection in events: - events[self.client.connection] |= selectors.EVENT_WRITE + if self.work.has_buffer(): + if self.work.connection.fileno() in events: + events[self.work.connection.fileno()] |= selectors.EVENT_WRITE else: - events[self.client.connection] = selectors.EVENT_WRITE + events[self.work.connection.fileno()] = selectors.EVENT_WRITE return events - def handle_events( + async def handle_events( self, readables: Readables, - writables: Writables) -> bool: + writables: Writables, + ) -> bool: """Return True to shutdown work.""" - do_shutdown = False - if self.client.connection in readables: - try: - data = self.client.recv() - if data is None: - # Client closed connection, signal shutdown - print( - 'Connection closed by client {0}'.format( - self.client.addr)) - do_shutdown = True - else: - r = self.handle_data(data) - if isinstance(r, bool) and r is True: - print( - 'Implementation signaled shutdown for client {0}'.format( - self.client.addr)) - if self.client.has_buffer(): - print( - 'Client {0} has pending buffer, will be flushed before shutting down'.format( - self.client.addr)) - self.must_flush_before_shutdown = True - else: - do_shutdown = True - except ConnectionResetError: - print( - 'Connection reset by client {0}'.format( - self.client.addr)) - do_shutdown = True - - if self.client.connection in writables: - print('Flushing buffer to client {0}'.format(self.client.addr)) - self.client.flush() + teardown = await self.handle_writables( + writables, + ) or await self.handle_readables(readables) + if teardown: + logger.debug( + 'Shutting down client {0} connection'.format( + self.work.address, + ), + ) + return teardown + + async def handle_writables(self, writables: Writables) -> bool: + teardown = False + if self.work.connection.fileno() in writables and self.work.has_buffer(): + logger.debug( + 'Flushing buffer to client {0}'.format(self.work.address), + ) + self.work.flush() if self.must_flush_before_shutdown is True: - do_shutdown = True - self.must_flush_before_shutdown = False + if not self.work.has_buffer(): + teardown = True + self.must_flush_before_shutdown = False + return teardown - if do_shutdown: - print( - 'Shutting down client {0} connection'.format( - self.client.addr)) - return do_shutdown + async def handle_readables(self, readables: Readables) -> bool: + teardown = False + if self.work.connection.fileno() in readables: + data = self.work.recv(self.flags.client_recvbuf_size) + if data is None: + logger.debug( + 'Connection closed by client {0}'.format( + self.work.address, + ), + ) + teardown = True + else: + r = self.handle_data(data) + if isinstance(r, bool) and r is True: + logger.debug( + 'Implementation signaled shutdown for client {0}'.format( + self.work.address, + ), + ) + if self.work.has_buffer(): + logger.debug( + 'Client {0} has pending buffer, will be flushed before shutting down'.format( + self.work.address, + ), + ) + self.must_flush_before_shutdown = True + else: + teardown = True + return teardown diff --git a/proxy/core/base/tcp_tunnel.py b/proxy/core/base/tcp_tunnel.py index d83053ae11..3c17ce8e31 100644 --- a/proxy/core/base/tcp_tunnel.py +++ b/proxy/core/base/tcp_tunnel.py @@ -7,10 +7,15 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + tcp """ -from abc import abstractmethod -import socket +import logging import selectors + +from abc import abstractmethod from typing import Any, Optional, Dict from ...http.parser import HttpParser, httpParserTypes @@ -20,13 +25,26 @@ from ..connection import TcpServerConnection from .tcp_server import BaseTcpServerHandler +logger = logging.getLogger(__name__) + class BaseTcpTunnelHandler(BaseTcpServerHandler): - """Base TCP tunnel interface.""" + """BaseTcpTunnelHandler build on-top of BaseTcpServerHandler work class. + + On-top of BaseTcpServerHandler implementation, + BaseTcpTunnelHandler introduces an upstream TcpServerConnection + and adds it to the core event loop when needed. + + Currently, implementations must call connect_upstream from within + handle_data. See HttpsConnectTunnelHandler for example usage. + """ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) - self.request = HttpParser(httpParserTypes.REQUEST_PARSER) + self.request = HttpParser( + httpParserTypes.REQUEST_PARSER, + enable_proxy_protocol=self.flags.enable_proxy_protocol, + ) self.upstream: Optional[TcpServerConnection] = None @abstractmethod @@ -34,55 +52,63 @@ def handle_data(self, data: memoryview) -> Optional[bool]: pass # pragma: no cover def initialize(self) -> None: - self.client.connection.setblocking(False) + self.work.connection.setblocking(False) def shutdown(self) -> None: if self.upstream: - print('Connection closed with upstream {0}:{1}'.format( - text_(self.request.host), self.request.port)) + logger.debug( + 'Connection closed with upstream {0}:{1}'.format( + text_(self.request.host), self.request.port, + ), + ) self.upstream.close() super().shutdown() - def get_events(self) -> Dict[socket.socket, int]: + async def get_events(self) -> Dict[int, int]: # Get default client events - ev: Dict[socket.socket, int] = super().get_events() + ev: Dict[int, int] = await super().get_events() # Read from server if we are connected if self.upstream and self.upstream._conn is not None: - ev[self.upstream.connection] = selectors.EVENT_READ + ev[self.upstream.connection.fileno()] = selectors.EVENT_READ # If there is pending buffer for server # also register for EVENT_WRITE events if self.upstream and self.upstream.has_buffer(): - if self.upstream.connection in ev: - ev[self.upstream.connection] |= selectors.EVENT_WRITE + if self.upstream.connection.fileno() in ev: + ev[self.upstream.connection.fileno()] |= selectors.EVENT_WRITE else: - ev[self.upstream.connection] = selectors.EVENT_WRITE + ev[self.upstream.connection.fileno()] = selectors.EVENT_WRITE return ev - def handle_events( + async def handle_events( self, readables: Readables, - writables: Writables) -> bool: + writables: Writables, + ) -> bool: # Handle client events - do_shutdown: bool = super().handle_events(readables, writables) + do_shutdown: bool = await super().handle_events(readables, writables) if do_shutdown: return do_shutdown # Handle server events - if self.upstream and self.upstream.connection in readables: + if self.upstream and self.upstream.connection.fileno() in readables: data = self.upstream.recv() if data is None: # Server closed connection - print('Connection closed by server') + logger.debug('Connection closed by server') return True # tunnel data to client - self.client.queue(data) - if self.upstream and self.upstream.connection in writables: + self.work.queue(data) + if self.upstream and self.upstream.connection.fileno() in writables: self.upstream.flush() return False def connect_upstream(self) -> None: assert self.request.host and self.request.port self.upstream = TcpServerConnection( - text_(self.request.host), self.request.port) + text_(self.request.host), self.request.port, + ) self.upstream.connect() - print('Connection established with upstream {0}:{1}'.format( - text_(self.request.host), self.request.port)) + logger.debug( + 'Connection established with upstream {0}:{1}'.format( + text_(self.request.host), self.request.port, + ), + ) diff --git a/proxy/core/base/tcp_upstream.py b/proxy/core/base/tcp_upstream.py new file mode 100644 index 0000000000..1de1e4a910 --- /dev/null +++ b/proxy/core/base/tcp_upstream.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import ssl +import logging + +from abc import ABC, abstractmethod +from typing import Tuple, List, Optional, Any + +from ...common.types import Readables, Writables +from ...core.connection import TcpServerConnection + +logger = logging.getLogger(__name__) + + +class TcpUpstreamConnectionHandler(ABC): + """:class:`~proxy.core.base.TcpUpstreamConnectionHandler` can + be used to insert an upstream server connection lifecycle. + + Call `initialize_upstream` to initialize the upstream connection object. + Then, directly use ``self.upstream`` object within your class. + + See :class:`~proxy.plugin.proxy_pool.ProxyPoolPlugin` for example usage. + + .. spelling:: + + tcp + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + # This is currently a hack, see comments below for rationale, + # will be fixed later. + super().__init__(*args, **kwargs) # type: ignore + self.upstream: Optional[TcpServerConnection] = None + # TODO: Currently, :class:`~proxy.core.base.TcpUpstreamConnectionHandler` + # is used within :class:`~proxy.plugin.ReverseProxyPlugin` and + # :class:`~proxy.plugin.ProxyPoolPlugin`. + # + # For both of which we expect a 4-tuple as arguments + # containing (uuid, flags, client, event_queue). + # We really don't need the rest of the args here. + # May be uuid? May be event_queue in the future. + # But certainly we don't not client here. + # A separate tunnel class must be created which handles + # client connection too. + # + # Both :class:`~proxy.plugin.ReverseProxyPlugin` and + # :class:`~proxy.plugin.ProxyPoolPlugin` are currently + # calling client queue within `handle_upstream_data` callback. + # + # This can be abstracted out too. + self.server_recvbuf_size = args[1].server_recvbuf_size + self.total_size = 0 + + @abstractmethod + def handle_upstream_data(self, raw: memoryview) -> None: + raise NotImplementedError() # pragma: no cover + + def initialize_upstream(self, addr: str, port: int) -> None: + self.upstream = TcpServerConnection(addr, port) + + def get_descriptors(self) -> Tuple[List[int], List[int]]: + if not self.upstream: + return [], [] + return [self.upstream.connection.fileno()], \ + [self.upstream.connection.fileno()] \ + if self.upstream.has_buffer() \ + else [] + + def read_from_descriptors(self, r: Readables) -> bool: + if self.upstream and \ + self.upstream.connection.fileno() in r: + try: + raw = self.upstream.recv(self.server_recvbuf_size) + if raw is not None: + self.total_size += len(raw) + self.handle_upstream_data(raw) + else: + # Tear down because upstream proxy closed the connection + return True + except ssl.SSLWantReadError: + logger.info('Upstream SSLWantReadError, will retry') + return False + except ConnectionResetError: + logger.debug('Connection reset by upstream') + return True + return False + + def write_to_descriptors(self, w: Writables) -> bool: + if self.upstream and \ + self.upstream.connection.fileno() in w and \ + self.upstream.has_buffer(): + try: + self.upstream.flush() + except ssl.SSLWantWriteError: + logger.info('Upstream SSLWantWriteError, will retry') + return False + except BrokenPipeError: + logger.debug('BrokenPipeError when flushing to upstream') + return True + return False diff --git a/proxy/core/connection/__init__.py b/proxy/core/connection/__init__.py index ee44bc14a6..952ee08f9e 100644 --- a/proxy/core/connection/__init__.py +++ b/proxy/core/connection/__init__.py @@ -7,10 +7,17 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + reusability + Submodules """ -from .connection import TcpConnection, TcpConnectionUninitializedException, tcpConnectionTypes +from .connection import TcpConnection, TcpConnectionUninitializedException from .client import TcpClientConnection from .server import TcpServerConnection +from .pool import ConnectionPool +from .types import tcpConnectionTypes __all__ = [ 'TcpConnection', @@ -18,4 +25,5 @@ 'TcpServerConnection', 'TcpClientConnection', 'tcpConnectionTypes', + 'ConnectionPool', ] diff --git a/proxy/core/connection/client.py b/proxy/core/connection/client.py index 62597a10d4..a6d91896cf 100644 --- a/proxy/core/connection/client.py +++ b/proxy/core/connection/client.py @@ -8,22 +8,31 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -import socket import ssl +import socket + from typing import Union, Tuple, Optional -from .connection import TcpConnection, tcpConnectionTypes, TcpConnectionUninitializedException +from .connection import TcpConnection, TcpConnectionUninitializedException +from .types import tcpConnectionTypes class TcpClientConnection(TcpConnection): - """An accepted client connection request.""" + """A buffered client connection object.""" - def __init__(self, - conn: Union[ssl.SSLSocket, socket.socket], - addr: Tuple[str, int]): + def __init__( + self, + conn: Union[ssl.SSLSocket, socket.socket], + # optional for unix socket servers + addr: Optional[Tuple[str, int]] = None, + ) -> None: super().__init__(tcpConnectionTypes.CLIENT) self._conn: Optional[Union[ssl.SSLSocket, socket.socket]] = conn - self.addr: Tuple[str, int] = addr + self.addr: Optional[Tuple[str, int]] = addr + + @property + def address(self) -> str: + return 'unix:client' if not self.addr else '{0}:{1}'.format(self.addr[0], self.addr[1]) @property def connection(self) -> Union[ssl.SSLSocket, socket.socket]: @@ -40,5 +49,6 @@ def wrap(self, keyfile: str, certfile: str) -> None: # ca_certs=self.flags.ca_cert_file, certfile=certfile, keyfile=keyfile, - ssl_version=ssl.PROTOCOL_TLS) + ssl_version=ssl.PROTOCOL_TLS, + ) self.connection.setblocking(False) diff --git a/proxy/core/connection/connection.py b/proxy/core/connection/connection.py index 73eabe208b..69036755aa 100644 --- a/proxy/core/connection/connection.py +++ b/proxy/core/connection/connection.py @@ -8,22 +8,18 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -import socket import ssl +import socket import logging + from abc import ABC, abstractmethod -from typing import NamedTuple, Optional, Union, List +from typing import Optional, Union, List from ...common.constants import DEFAULT_BUFFER_SIZE, DEFAULT_MAX_SEND_SIZE -logger = logging.getLogger(__name__) - +from .types import tcpConnectionTypes -TcpConnectionTypes = NamedTuple('TcpConnectionTypes', [ - ('SERVER', int), - ('CLIENT', int), -]) -tcpConnectionTypes = TcpConnectionTypes(1, 2) +logger = logging.getLogger(__name__) class TcpConnectionUninitializedException(Exception): @@ -37,12 +33,15 @@ class TcpConnection(ABC): when reading and writing into the socket. Implement the connection property abstract method to return - a socket connection object.""" + a socket connection object. + """ - def __init__(self, tag: int): + def __init__(self, tag: int) -> None: + self.tag: str = 'server' if tag == tcpConnectionTypes.SERVER else 'client' self.buffer: List[memoryview] = [] self.closed: bool = False - self.tag: str = 'server' if tag == tcpConnectionTypes.SERVER else 'client' + self._reusable: bool = False + self._num_buffer = 0 @property @abstractmethod @@ -52,17 +51,20 @@ def connection(self) -> Union[ssl.SSLSocket, socket.socket]: def send(self, data: bytes) -> int: """Users must handle BrokenPipeError exceptions""" + # logger.info(data) return self.connection.send(data) def recv( - self, buffer_size: int = DEFAULT_BUFFER_SIZE) -> Optional[memoryview]: + self, buffer_size: int = DEFAULT_BUFFER_SIZE, + ) -> Optional[memoryview]: """Users must handle socket.error exceptions""" data: bytes = self.connection.recv(buffer_size) if len(data) == 0: return None logger.debug( 'received %d bytes from %s' % - (len(data), self.tag)) + (len(data), self.tag), + ) # logger.info(data) return memoryview(data) @@ -73,10 +75,11 @@ def close(self) -> bool: return self.closed def has_buffer(self) -> bool: - return len(self.buffer) > 0 + return self._num_buffer > 0 def queue(self, mv: memoryview) -> None: self.buffer.append(mv) + self._num_buffer += 1 def flush(self) -> int: """Users must handle BrokenPipeError exceptions""" @@ -86,8 +89,21 @@ def flush(self) -> int: sent: int = self.send(mv[:DEFAULT_MAX_SEND_SIZE]) if sent == len(mv): self.buffer.pop(0) + self._num_buffer -= 1 else: self.buffer[0] = memoryview(mv[sent:]) del mv logger.debug('flushed %d bytes to %s' % (sent, self.tag)) return sent + + def is_reusable(self) -> bool: + return self._reusable + + def mark_inuse(self) -> None: + self._reusable = False + + def reset(self) -> None: + assert not self.closed + self._reusable = True + self.buffer = [] + self._num_buffer = 0 diff --git a/proxy/core/connection/pool.py b/proxy/core/connection/pool.py new file mode 100644 index 0000000000..16cd5096b1 --- /dev/null +++ b/proxy/core/connection/pool.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. + + .. spelling:: + + reusability +""" +import logging + +from typing import Set, Dict, Tuple + +from ...common.flag import flags + +from .server import TcpServerConnection + +logger = logging.getLogger(__name__) + + +flags.add_argument( + '--enable-conn-pool', + action='store_true', + default=False, + help='Default: False. (WIP) Enable upstream connection pooling.', +) + + +class ConnectionPool: + """Manages connection pool to upstream servers. + + `ConnectionPool` avoids need to reconnect with the upstream + servers repeatedly when a reusable connection is available + in the pool. + + A separate pool is maintained for each upstream server. + So internally, it's a pool of pools. + + TODO: Listen for read events from the connections + to remove them from the pool when peer closes the + connection. This can also be achieved lazily by + the pool users. Example, if acquired connection + is stale, reacquire. + + TODO: Ideally, ConnectionPool must be shared across + all cores to make SSL session cache to also work + without additional out-of-bound synchronizations. + + TODO: ConnectionPool currently WON'T work for + HTTPS connection. This is because of missing support for + session cache, session ticket, abbr TLS handshake + and other necessary features to make it work. + + NOTE: However, for all HTTP only connections, ConnectionPool + can be used to save upon connection setup time and + speed-up performance of requests. + """ + + def __init__(self) -> None: + # Pools of connection per upstream server + self.pools: Dict[Tuple[str, int], Set[TcpServerConnection]] = {} + + def acquire(self, host: str, port: int) -> Tuple[bool, TcpServerConnection]: + """Returns a connection for use with the server.""" + addr = (host, port) + # Return a reusable connection if available + if addr in self.pools: + for old_conn in self.pools[addr]: + if old_conn.is_reusable(): + old_conn.mark_inuse() + logger.debug( + 'Reusing connection#{2} for upstream {0}:{1}'.format( + host, port, id(old_conn), + ), + ) + return False, old_conn + # Create new connection + new_conn = TcpServerConnection(*addr) + if addr not in self.pools: + self.pools[addr] = set() + self.pools[addr].add(new_conn) + logger.debug( + 'Created new connection#{2} for upstream {0}:{1}'.format( + host, port, id(new_conn), + ), + ) + return True, new_conn + + def release(self, conn: TcpServerConnection) -> None: + """Release the connection. + + If the connection has not been closed, + then it will be retained in the pool for reusability. + """ + if conn.closed: + logger.debug( + 'Removing connection#{2} from pool from upstream {0}:{1}'.format( + conn.addr[0], conn.addr[1], id(conn), + ), + ) + self.pools[conn.addr].remove(conn) + else: + logger.debug( + 'Retaining connection#{2} to upstream {0}:{1}'.format( + conn.addr[0], conn.addr[1], id(conn), + ), + ) + assert not conn.is_reusable() + # Reset for reusability + conn.reset() diff --git a/proxy/core/connection/server.py b/proxy/core/connection/server.py index c5636e6a92..7aae5371cc 100644 --- a/proxy/core/connection/server.py +++ b/proxy/core/connection/server.py @@ -10,19 +10,23 @@ """ import ssl import socket + from typing import Optional, Union, Tuple -from .connection import TcpConnection, tcpConnectionTypes, TcpConnectionUninitializedException from ...common.utils import new_socket_connection +from .connection import TcpConnection, TcpConnectionUninitializedException +from .types import tcpConnectionTypes + class TcpServerConnection(TcpConnection): - """Establishes connection to upstream server.""" + """A buffered server connection object.""" - def __init__(self, host: str, port: int): + def __init__(self, host: str, port: int) -> None: super().__init__(tcpConnectionTypes.SERVER) self._conn: Optional[Union[ssl.SSLSocket, socket.socket]] = None self.addr: Tuple[str, int] = (host, int(port)) + self.closed = True @property def connection(self) -> Union[ssl.SSLSocket, socket.socket]: @@ -30,18 +34,26 @@ def connection(self) -> Union[ssl.SSLSocket, socket.socket]: raise TcpConnectionUninitializedException() return self._conn - def connect(self) -> None: - if self._conn is not None: - return - self._conn = new_socket_connection(self.addr) + def connect( + self, + addr: Optional[Tuple[str, int]] = None, + source_address: Optional[Tuple[str, int]] = None, + ) -> None: + if self._conn is None: + self._conn = new_socket_connection( + addr or self.addr, source_address=source_address, + ) + self.closed = False def wrap(self, hostname: str, ca_file: Optional[str]) -> None: ctx = ssl.create_default_context( - ssl.Purpose.SERVER_AUTH, cafile=ca_file) - ctx.options |= ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1 + ssl.Purpose.SERVER_AUTH, cafile=ca_file, + ) + ctx.options |= ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1 ctx.check_hostname = True self.connection.setblocking(True) self._conn = ctx.wrap_socket( self.connection, - server_hostname=hostname) + server_hostname=hostname, + ) self.connection.setblocking(False) diff --git a/proxy/core/connection/types.py b/proxy/core/connection/types.py new file mode 100644 index 0000000000..663c0ae4d4 --- /dev/null +++ b/proxy/core/connection/types.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. + + .. spelling:: + + iterable +""" +from typing import NamedTuple + + +TcpConnectionTypes = NamedTuple( + 'TcpConnectionTypes', [ + ('SERVER', int), + ('CLIENT', int), + ], +) + +tcpConnectionTypes = TcpConnectionTypes(1, 2) diff --git a/proxy/core/event/__init__.py b/proxy/core/event/__init__.py index 6907dcd55b..08b6f5be49 100644 --- a/proxy/core/event/__init__.py +++ b/proxy/core/event/__init__.py @@ -7,11 +7,18 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + eventing + iterable + Submodules """ from .queue import EventQueue from .names import EventNames, eventNames from .dispatcher import EventDispatcher from .subscriber import EventSubscriber +from .manager import EventManager __all__ = [ 'eventNames', @@ -19,4 +26,5 @@ 'EventQueue', 'EventDispatcher', 'EventSubscriber', + 'EventManager', ] diff --git a/proxy/core/event/dispatcher.py b/proxy/core/event/dispatcher.py index fb7c527533..1bcb40201d 100644 --- a/proxy/core/event/dispatcher.py +++ b/proxy/core/event/dispatcher.py @@ -9,12 +9,12 @@ :license: BSD, see LICENSE for more details. """ import queue -import threading import logging +import threading -from typing import Dict, Any, List +from multiprocessing import connection -from ...common.types import DictQueueType +from typing import Dict, Any, List from .queue import EventQueue from .names import eventNames @@ -25,52 +25,52 @@ class EventDispatcher: """Core EventDispatcher. - Provides: - 1. A dispatcher module which consumes core events and dispatches - them to EventQueueBasePlugin - 2. A publish utility for publishing core events into - global events queue. - Direct consuming from global events queue outside of dispatcher module is not-recommended. Python native multiprocessing queue doesn't provide a fanout functionality which core dispatcher module - implements so that several plugins can consume same published - event at a time. + implements so that several plugins can consume the same published + event concurrently (when necessary). When --enable-events is used, a multiprocessing.Queue is created and - attached to global argparse. This queue can then be used for + attached to global flags. This queue can then be used for dispatching an Event dict object into the queue. When --enable-events is used, dispatcher module is automatically - started. Dispatcher module also ensures that queue is not full and - doesn't utilize too much memory in case there are no event plugins - enabled. + started. Most importantly, dispatcher module ensures that queue is + not flooded and doesn't utilize too much memory in case there are no + event subscriber is enabled. """ def __init__( self, shutdown: threading.Event, - event_queue: EventQueue) -> None: + event_queue: EventQueue, + ) -> None: self.shutdown: threading.Event = shutdown self.event_queue: EventQueue = event_queue - self.subscribers: Dict[str, DictQueueType] = {} + # subscriber connection objects + self.subscribers: Dict[str, connection.Connection] = {} def handle_event(self, ev: Dict[str, Any]) -> None: if ev['event_name'] == eventNames.SUBSCRIBE: self.subscribers[ev['event_payload']['sub_id']] = \ - ev['event_payload']['channel'] + ev['event_payload']['conn'] + # send ack + ev['event_payload']['conn'].send({ + 'event_name': eventNames.SUBSCRIBED, + }) elif ev['event_name'] == eventNames.UNSUBSCRIBE: + # send ack + print('unsubscription request ack sent') + self.subscribers[ev['event_payload']['sub_id']].send({ + 'event_name': eventNames.UNSUBSCRIBED, + }) + # close conn and delete subscriber + self.subscribers[ev['event_payload']['sub_id']].close() del self.subscribers[ev['event_payload']['sub_id']] else: # logger.info(ev) - unsub_ids: List[str] = [] - for sub_id in self.subscribers: - try: - self.subscribers[sub_id].put(ev) - except BrokenPipeError: - unsub_ids.append(sub_id) - for sub_id in unsub_ids: - del self.subscribers[sub_id] + self._broadcast(ev) def run_once(self) -> None: ev: Dict[str, Any] = self.event_queue.queue.get(timeout=1) @@ -90,4 +90,23 @@ def run(self) -> None: except KeyboardInterrupt: pass except Exception as e: - logger.exception('Event dispatcher exception', exc_info=e) + logger.exception('Dispatcher exception', exc_info=e) + finally: + # Send shutdown message to all active subscribers + self._broadcast({ + 'event_name': eventNames.DISPATCHER_SHUTDOWN, + }) + + def _broadcast(self, ev: Dict[str, Any]) -> None: + broken_pipes: List[str] = [] + for sub_id in self.subscribers: + try: + self.subscribers[sub_id].send(ev) + except BrokenPipeError: + logger.warning( + 'Subscriber#%s broken pipe', sub_id, + ) + self.subscribers[sub_id].close() + broken_pipes.append(sub_id) + for sub_id in broken_pipes: + del self.subscribers[sub_id] diff --git a/proxy/core/event/manager.py b/proxy/core/event/manager.py new file mode 100644 index 0000000000..3923c6c12e --- /dev/null +++ b/proxy/core/event/manager.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. + + .. spelling:: + + eventing +""" +import logging +import threading +import multiprocessing + +from typing import Any, Optional + +from .queue import EventQueue +from .dispatcher import EventDispatcher + +from ...common.flag import flags +from ...common.constants import DEFAULT_ENABLE_EVENTS + +logger = logging.getLogger(__name__) + + +flags.add_argument( + '--enable-events', + action='store_true', + default=DEFAULT_ENABLE_EVENTS, + help='Default: False. Enables core to dispatch lifecycle events. ' + 'Plugins can be used to subscribe for core events.', +) + + +class EventManager: + """Event manager is a context manager which provides + encapsulation around various setup and shutdown steps + to start the eventing core. + """ + + def __init__(self) -> None: + self.queue: Optional[EventQueue] = None + self.dispatcher: Optional[EventDispatcher] = None + self.dispatcher_thread: Optional[threading.Thread] = None + self.dispatcher_shutdown: Optional[threading.Event] = None + + def __enter__(self) -> 'EventManager': + self.setup() + return self + + def __exit__(self, *args: Any) -> None: + self.shutdown() + + def setup(self) -> None: + self.queue = EventQueue(multiprocessing.Queue()) + self.dispatcher_shutdown = threading.Event() + assert self.dispatcher_shutdown + assert self.queue + self.dispatcher = EventDispatcher( + shutdown=self.dispatcher_shutdown, + event_queue=self.queue, + ) + self.dispatcher_thread = threading.Thread( + target=self.dispatcher.run, + ) + self.dispatcher_thread.start() + logger.debug('Dispatcher#%d started', self.dispatcher_thread.ident) + + def shutdown(self) -> None: + assert self.dispatcher_shutdown and self.dispatcher_thread + self.dispatcher_shutdown.set() + self.dispatcher_thread.join() + logger.debug( + 'Dispatcher#%d shutdown', + self.dispatcher_thread.ident, + ) diff --git a/proxy/core/event/names.py b/proxy/core/event/names.py index b45a70b2d5..9a58926c6c 100644 --- a/proxy/core/event/names.py +++ b/proxy/core/event/names.py @@ -7,17 +7,31 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + eventing + iterable """ from typing import NamedTuple -EventNames = NamedTuple('EventNames', [ - ('SUBSCRIBE', int), - ('UNSUBSCRIBE', int), - ('WORK_STARTED', int), - ('WORK_FINISHED', int), - ('REQUEST_COMPLETE', int), - ('RESPONSE_HEADERS_COMPLETE', int), - ('RESPONSE_CHUNK_RECEIVED', int), - ('RESPONSE_COMPLETE', int), -]) -eventNames = EventNames(1, 2, 3, 4, 5, 6, 7, 8) +# Name of the events that eventing framework supports. +# +# Ideally this must be configurable via command line or +# at-least extendable via plugins. +EventNames = NamedTuple( + 'EventNames', [ + ('SUBSCRIBE', int), + ('SUBSCRIBED', int), + ('UNSUBSCRIBE', int), + ('UNSUBSCRIBED', int), + ('DISPATCHER_SHUTDOWN', int), + ('WORK_STARTED', int), + ('WORK_FINISHED', int), + ('REQUEST_COMPLETE', int), + ('RESPONSE_HEADERS_COMPLETE', int), + ('RESPONSE_CHUNK_RECEIVED', int), + ('RESPONSE_COMPLETE', int), + ], +) +eventNames = EventNames(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11) diff --git a/proxy/core/event/queue.py b/proxy/core/event/queue.py index 36b246648d..f0110e0401 100644 --- a/proxy/core/event/queue.py +++ b/proxy/core/event/queue.py @@ -9,8 +9,11 @@ :license: BSD, see LICENSE for more details. """ import os -import threading import time +import threading + +from multiprocessing import connection + from typing import Dict, Optional, Any from ...common.types import DictQueueType @@ -19,23 +22,25 @@ class EventQueue: - """Global event queue. + """Global event queue. Must be a multiprocessing.Manager queue because + subscribers need to dispatch their subscription queue over this global + queue. + + Each published event contains following schema:: - Each event contains: + { + 'request_id': 'Globally unique request ID', + 'process_id': 'Process ID of event publisher. This ' + 'will be the process ID of acceptor workers.', + 'thread_id': 'Thread ID of event publisher. ' + 'When --threadless is enabled, this value ' + 'will be same for all the requests.' + 'event_timestamp': 'Time when this event occured', + 'event_name': 'one of the pre-defined or custom event name', + 'event_payload': 'Optional data associated with the event', + 'publisher_id': 'Optional publisher entity unique name', + } - 1. Request ID - Globally unique - 2. Process ID - Process ID of event publisher. - This will be process id of acceptor workers. - 3. Thread ID - Thread ID of event publisher. - When --threadless is enabled, this value will - be same for all the requests - received by a single acceptor worker. - When --threadless is disabled, this value will be - Thread ID of the thread handling the client request. - 4. Event Timestamp - Time when this event occur - 5. Event Name - One of the defined or custom event name - 6. Event Payload - Optional data associated with the event - 7. Publisher ID (optional) - Optionally, publishing entity unique name / ID """ def __init__(self, queue: DictQueueType) -> None: @@ -46,13 +51,13 @@ def publish( request_id: str, event_name: int, event_payload: Dict[str, Any], - publisher_id: Optional[str] = None + publisher_id: Optional[str] = None, ) -> None: self.queue.put({ - 'request_id': request_id, 'process_id': os.getpid(), 'thread_id': threading.get_ident(), 'event_timestamp': time.time(), + 'request_id': request_id, 'event_name': event_name, 'event_payload': event_payload, 'publisher_id': publisher_id, @@ -61,16 +66,22 @@ def publish( def subscribe( self, sub_id: str, - channel: DictQueueType) -> None: - """Subscribe to global events.""" + channel: connection.Connection, + ) -> None: + """Subscribe to global events. + + sub_id is a subscription identifier which must be globally + unique. channel MUST be a multiprocessing connection. + """ self.queue.put({ 'event_name': eventNames.SUBSCRIBE, - 'event_payload': {'sub_id': sub_id, 'channel': channel}, + 'event_payload': {'sub_id': sub_id, 'conn': channel}, }) def unsubscribe( self, - sub_id: str) -> None: + sub_id: str, + ) -> None: """Unsubscribe by subscriber id.""" self.queue.put({ 'event_name': eventNames.UNSUBSCRIBE, diff --git a/proxy/core/event/subscriber.py b/proxy/core/event/subscriber.py index 90648e0d87..c92803074f 100644 --- a/proxy/core/event/subscriber.py +++ b/proxy/core/event/subscriber.py @@ -8,85 +8,175 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ +import uuid import queue +import logging import threading import multiprocessing -import logging -import uuid -from typing import Dict, Optional, Any, Callable +from multiprocessing import connection -from ...common.types import DictQueueType +from typing import Dict, Optional, Any, Callable from .queue import EventQueue +from .names import eventNames logger = logging.getLogger(__name__) class EventSubscriber: - """Core event subscriber.""" + """Core event subscriber. + + Usage: Initialize one instance per CPU core for optimum performance. + + EventSubscriber can run within various context. E.g. main thread, + another thread or a different process. EventSubscriber context + can be different from publishers. Publishers can even be processes + outside of the proxy.py core. + + Note that, EventSubscriber cannot share the `multiprocessing.Manager` + with the EventManager. Because EventSubscriber can be started + in a different process than EventManager. + + `multiprocessing.Manager` is used to initialize + a new Queue which is used for subscriptions. EventDispatcher + might be running in a separate process and hence + subscription queue must be multiprocess safe. + + When `subscribe` method is called, EventManager will + start a relay thread which consumes using the multiprocess + safe queue passed to the relay thread. + """ - def __init__(self, event_queue: EventQueue) -> None: - self.manager = multiprocessing.Manager() + def __init__(self, event_queue: EventQueue, callback: Callable[[Dict[str, Any]], None]) -> None: self.event_queue = event_queue + self.callback = callback self.relay_thread: Optional[threading.Thread] = None self.relay_shutdown: Optional[threading.Event] = None - self.relay_channel: Optional[DictQueueType] = None + self.relay_recv: Optional[connection.Connection] = None + self.relay_send: Optional[connection.Connection] = None self.relay_sub_id: Optional[str] = None - def subscribe(self, callback: Callable[[Dict[str, Any]], None]) -> None: - self.relay_shutdown = threading.Event() - self.relay_channel = self.manager.Queue() - self.relay_thread = threading.Thread( - target=self.relay, - args=(self.relay_shutdown, self.relay_channel, callback)) - self.relay_thread.start() - self.relay_sub_id = uuid.uuid4().hex - self.event_queue.subscribe(self.relay_sub_id, self.relay_channel) + def __enter__(self) -> 'EventSubscriber': + self.setup() + return self + + def __exit__(self, *args: Any) -> None: + self.shutdown() + + def setup(self, do_subscribe: bool = True) -> None: + """Setup subscription thread. + + Call subscribe() to actually start subscription. + """ + self._start_relay_thread() + assert self.relay_sub_id and self.relay_recv + logger.debug( + 'Subscriber#%s relay setup done', + self.relay_sub_id, + ) + if do_subscribe: + self.subscribe() + + def shutdown(self, do_unsubscribe: bool = True) -> None: + """Tear down subscription thread. + + Call unsubscribe() to actually stop subscription. + """ + self._stop_relay_thread() logger.debug( - 'Subscribed relay sub id %s from core events', - self.relay_sub_id) + 'Subscriber#%s relay shutdown done', + self.relay_sub_id, + ) + if do_unsubscribe: + self.unsubscribe() + + def subscribe(self) -> None: + assert self.relay_sub_id and self.relay_send + self.event_queue.subscribe(self.relay_sub_id, self.relay_send) def unsubscribe(self) -> None: if self.relay_sub_id is None: - logger.warning('Unsubscribe called without existing subscription') + logger.warning( + 'Relay called unsubscribe without an active subscription', + ) return - - assert self.relay_thread - assert self.relay_shutdown - assert self.relay_channel - assert self.relay_sub_id - try: self.event_queue.unsubscribe(self.relay_sub_id) except BrokenPipeError: pass except EOFError: pass - - self.relay_shutdown.set() - self.relay_thread.join() - logger.debug( - 'Un-subscribed relay sub id %s from core events', - self.relay_sub_id) - - self.relay_thread = None - self.relay_shutdown = None - self.relay_channel = None - self.relay_sub_id = None + finally: + # self.relay_sub_id = None + pass @staticmethod def relay( + sub_id: str, shutdown: threading.Event, - channel: DictQueueType, - callback: Callable[[Dict[str, Any]], None]) -> None: + channel: connection.Connection, + callback: Callable[[Dict[str, Any]], None], + ) -> None: while not shutdown.is_set(): try: - ev = channel.get(timeout=1) - callback(ev) + if channel.poll(timeout=1): + ev = channel.recv() + if ev['event_name'] == eventNames.SUBSCRIBED: + logger.info( + 'Subscriber#{0} subscribe ack received'.format( + sub_id, + ), + ) + elif ev['event_name'] == eventNames.UNSUBSCRIBED: + logger.info( + 'Subscriber#{0} unsubscribe ack received'.format( + sub_id, + ), + ) + break + elif ev['event_name'] == eventNames.DISPATCHER_SHUTDOWN: + logger.info( + 'Subscriber#{0} received dispatcher shutdown event'.format( + sub_id, + ), + ) + break + else: + callback(ev) except queue.Empty: pass except EOFError: break except KeyboardInterrupt: break + + def _start_relay_thread(self) -> None: + self.relay_sub_id = uuid.uuid4().hex + self.relay_shutdown = threading.Event() + self.relay_recv, self.relay_send = multiprocessing.Pipe() + self.relay_thread = threading.Thread( + target=EventSubscriber.relay, + args=( + self.relay_sub_id, self.relay_shutdown, + self.relay_recv, self.callback, + ), + ) + self.relay_thread.daemon = True + self.relay_thread.start() + + def _stop_relay_thread(self) -> None: + assert self.relay_thread and self.relay_shutdown and self.relay_recv and self.relay_send + self.relay_shutdown.set() + self.relay_thread.join() + self.relay_recv.close() + # Currently relay_send instance here in + # subscriber is not the same as one received + # by dispatcher. This may cause file + # descriptor leakage. So we make a close + # here explicit on our side of relay_send too. + self.relay_send.close() + self.relay_thread = None + self.relay_shutdown = None + self.relay_recv = None + self.relay_send = None diff --git a/proxy/core/ssh/__init__.py b/proxy/core/ssh/__init__.py index 232621f0b5..e37310801c 100644 --- a/proxy/core/ssh/__init__.py +++ b/proxy/core/ssh/__init__.py @@ -7,4 +7,15 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + Submodules """ +from .client import SshClient +from .tunnel import Tunnel + +__all__ = [ + 'SshClient', + 'Tunnel', +] diff --git a/proxy/core/ssh/client.py b/proxy/core/ssh/client.py index 650d894809..4657b1a3c1 100644 --- a/proxy/core/ssh/client.py +++ b/proxy/core/ssh/client.py @@ -18,7 +18,7 @@ class SshClient(TcpClientConnection): """Overrides TcpClientConnection. - This is necessary because paramiko fileno() can be used for polling + This is necessary because paramiko ``fileno()`` can be used for polling but not for send / recv. """ diff --git a/proxy/core/ssh/tunnel.py b/proxy/core/ssh/tunnel.py index e3a61b54df..4a899543ae 100644 --- a/proxy/core/ssh/tunnel.py +++ b/proxy/core/ssh/tunnel.py @@ -8,15 +8,18 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from typing import Tuple, Callable - +import logging import paramiko +from typing import Optional, Tuple, Callable + +logger = logging.getLogger(__name__) + class Tunnel: """Establishes a tunnel between local (machine where Tunnel is running) and remote host. Once a tunnel has been established, remote host can route HTTP(s) traffic to - localhost over tunnel. + ``localhost`` over tunnel. """ def __init__( @@ -25,7 +28,8 @@ def __init__( remote_addr: Tuple[str, int], private_pem_key: str, remote_proxy_port: int, - conn_handler: Callable[[paramiko.channel.Channel], None]) -> None: + conn_handler: Callable[[paramiko.channel.Channel], None], + ) -> None: self.remote_addr = remote_addr self.ssh_username = ssh_username self.private_pem_key = private_pem_key @@ -41,14 +45,19 @@ def run(self) -> None: hostname=self.remote_addr[0], port=self.remote_addr[1], username=self.ssh_username, - key_filename=self.private_pem_key + key_filename=self.private_pem_key, + ) + logger.info('SSH connection established...') + transport: Optional[paramiko.transport.Transport] = ssh.get_transport( ) - print('SSH connection established...') - transport: paramiko.transport.Transport = ssh.get_transport() + assert transport is not None transport.request_port_forward('', self.remote_proxy_port) - print('Tunnel port forward setup successful...') + logger.info('Tunnel port forward setup successful...') while True: - conn: paramiko.channel.Channel = transport.accept(timeout=1) + conn: Optional[paramiko.channel.Channel] = transport.accept( + timeout=1, + ) + assert conn is not None e = transport.get_exception() if e: raise e diff --git a/proxy/dashboard/__init__.py b/proxy/dashboard/__init__.py index 232621f0b5..61f5ec0232 100644 --- a/proxy/dashboard/__init__.py +++ b/proxy/dashboard/__init__.py @@ -7,4 +7,19 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + Submodules + websocket + Websocket """ +from .dashboard import ProxyDashboard +from .inspect_traffic import InspectTrafficPlugin +from .plugin import ProxyDashboardWebsocketPlugin + +__all__ = [ + 'ProxyDashboard', + 'InspectTrafficPlugin', + 'ProxyDashboardWebsocketPlugin', +] diff --git a/proxy/dashboard/dashboard.py b/proxy/dashboard/dashboard.py index dda3ae0ed3..9bb2a3eef3 100644 --- a/proxy/dashboard/dashboard.py +++ b/proxy/dashboard/dashboard.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ proxy.py ~~~~~~~~ @@ -15,10 +16,11 @@ from .plugin import ProxyDashboardWebsocketPlugin from ..common.utils import build_http_response, bytes_ -from ..http.server import HttpWebServerPlugin, HttpWebServerBasePlugin, httpProtocolTypes + +from ..http import httpStatusCodes from ..http.parser import HttpParser from ..http.websocket import WebsocketFrame -from ..http.codes import httpStatusCodes +from ..http.server import HttpWebServerPlugin, HttpWebServerBasePlugin, httpProtocolTypes logger = logging.getLogger(__name__) @@ -63,18 +65,29 @@ def handle_request(self, request: HttpParser) -> None: if request.path == b'/dashboard/': self.client.queue( HttpWebServerPlugin.read_and_build_static_file_response( - os.path.join(self.flags.static_server_dir, 'dashboard', 'proxy.html'))) + os.path.join( + self.flags.static_server_dir, + 'dashboard', 'proxy.html', + ), + self.flags.min_compression_limit, + ), + ) elif request.path in ( b'/dashboard', - b'/dashboard/proxy.html'): - self.client.queue(memoryview(build_http_response( - httpStatusCodes.PERMANENT_REDIRECT, reason=b'Permanent Redirect', - headers={ - b'Location': b'/dashboard/', - b'Content-Length': b'0', - b'Connection': b'close', - } - ))) + b'/dashboard/proxy.html', + ): + self.client.queue( + memoryview( + build_http_response( + httpStatusCodes.PERMANENT_REDIRECT, reason=b'Permanent Redirect', + headers={ + b'Location': b'/dashboard/', + b'Content-Length': b'0', + b'Connection': b'close', + }, + ), + ), + ) def on_websocket_open(self) -> None: logger.info('app ws opened') @@ -98,12 +111,17 @@ def on_websocket_message(self, frame: WebsocketFrame) -> None: logger.info(frame.opcode) self.reply({'id': message['id'], 'response': 'not_implemented'}) - def on_websocket_close(self) -> None: + def on_client_connection_close(self) -> None: logger.info('app ws closed') # TODO(abhinavsingh): unsubscribe def reply(self, data: Dict[str, Any]) -> None: self.client.queue( - memoryview(WebsocketFrame.text( - bytes_( - json.dumps(data))))) + memoryview( + WebsocketFrame.text( + bytes_( + json.dumps(data), + ), + ), + ), + ) diff --git a/proxy/dashboard/inspect_traffic.py b/proxy/dashboard/inspect_traffic.py index df82a37f90..15edc0d687 100644 --- a/proxy/dashboard/inspect_traffic.py +++ b/proxy/dashboard/inspect_traffic.py @@ -7,6 +7,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + websocket + Websocket """ import json from typing import List, Dict, Any @@ -24,7 +29,12 @@ class InspectTrafficPlugin(ProxyDashboardWebsocketPlugin): def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) - self.subscriber = EventSubscriber(self.event_queue) + self.subscriber = EventSubscriber( + self.event_queue, + callback=lambda event: InspectTrafficPlugin.callback( + self.client, event, + ), + ) def methods(self) -> List[str]: return [ @@ -37,23 +47,30 @@ def handle_message(self, message: Dict[str, Any]) -> None: # inspection can only be enabled if --enable-events is used if not self.flags.enable_events: self.client.queue( - memoryview(WebsocketFrame.text( - bytes_( - json.dumps( - {'id': message['id'], 'response': 'not enabled'}) - ) - )) + memoryview( + WebsocketFrame.text( + bytes_( + json.dumps( + { + 'id': message['id'], + 'response': 'not enabled', + }, + ), + ), + ), + ), ) else: - self.subscriber.subscribe( - lambda event: InspectTrafficPlugin.callback( - self.client, event)) + self.subscriber.setup() self.reply( - {'id': message['id'], 'response': 'inspection_enabled'}) + {'id': message['id'], 'response': 'inspection_enabled'}, + ) elif message['method'] == 'disable_inspection': - self.subscriber.unsubscribe() - self.reply({'id': message['id'], - 'response': 'inspection_disabled'}) + self.subscriber.shutdown() + self.reply({ + 'id': message['id'], + 'response': 'inspection_disabled', + }) else: raise NotImplementedError() @@ -61,6 +78,11 @@ def handle_message(self, message: Dict[str, Any]) -> None: def callback(client: TcpClientConnection, event: Dict[str, Any]) -> None: event['push'] = 'inspect_traffic' client.queue( - memoryview(WebsocketFrame.text( - bytes_( - json.dumps(event))))) + memoryview( + WebsocketFrame.text( + bytes_( + json.dumps(event), + ), + ), + ), + ) diff --git a/proxy/dashboard/plugin.py b/proxy/dashboard/plugin.py index b3787ac2df..86032c69e0 100644 --- a/proxy/dashboard/plugin.py +++ b/proxy/dashboard/plugin.py @@ -26,7 +26,8 @@ def __init__( self, flags: argparse.Namespace, client: TcpClientConnection, - event_queue: EventQueue) -> None: + event_queue: EventQueue, + ) -> None: self.flags = flags self.client = client self.event_queue = event_queue @@ -34,23 +35,28 @@ def __init__( @abstractmethod def methods(self) -> List[str]: """Return list of methods that this plugin will handle.""" - pass + pass # pragma: no cover def connected(self) -> None: """Invoked when client websocket handshake finishes.""" - pass + pass # pragma: no cover @abstractmethod def handle_message(self, message: Dict[str, Any]) -> None: """Handle messages for registered methods.""" - pass + pass # pragma: no cover def disconnected(self) -> None: """Invoked when client websocket connection gets closed.""" - pass + pass # pragma: no cover def reply(self, data: Dict[str, Any]) -> None: self.client.queue( - memoryview(WebsocketFrame.text( - bytes_( - json.dumps(data))))) + memoryview( + WebsocketFrame.text( + bytes_( + json.dumps(data), + ), + ), + ), + ) diff --git a/proxy/http/__init__.py b/proxy/http/__init__.py index 232621f0b5..b1d6877d15 100644 --- a/proxy/http/__init__.py +++ b/proxy/http/__init__.py @@ -7,4 +7,23 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + Subpackages + Submodules """ +from .handler import HttpProtocolHandler +from .plugin import HttpProtocolHandlerPlugin +from .codes import httpStatusCodes +from .methods import httpMethods +from .url import Url + +__all__ = [ + 'HttpProtocolHandler', + 'HttpProtocolHandlerPlugin', + 'httpStatusCodes', + 'httpMethods', + 'Url', +] diff --git a/proxy/http/codes.py b/proxy/http/codes.py index 042d27e4f9..ad6716090d 100644 --- a/proxy/http/codes.py +++ b/proxy/http/codes.py @@ -7,41 +7,49 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + iterable """ from typing import NamedTuple -HttpStatusCodes = NamedTuple('HttpStatusCodes', [ - # 1xx - ('CONTINUE', int), - ('SWITCHING_PROTOCOLS', int), - # 2xx - ('OK', int), - # 3xx - ('MOVED_PERMANENTLY', int), - ('SEE_OTHER', int), - ('TEMPORARY_REDIRECT', int), - ('PERMANENT_REDIRECT', int), - # 4xx - ('BAD_REQUEST', int), - ('UNAUTHORIZED', int), - ('FORBIDDEN', int), - ('NOT_FOUND', int), - ('PROXY_AUTH_REQUIRED', int), - ('REQUEST_TIMEOUT', int), - ('I_AM_A_TEAPOT', int), - # 5xx - ('INTERNAL_SERVER_ERROR', int), - ('NOT_IMPLEMENTED', int), - ('BAD_GATEWAY', int), - ('GATEWAY_TIMEOUT', int), - ('NETWORK_READ_TIMEOUT_ERROR', int), - ('NETWORK_CONNECT_TIMEOUT_ERROR', int), -]) +HttpStatusCodes = NamedTuple( + 'HttpStatusCodes', [ + # 1xx + ('CONTINUE', int), + ('SWITCHING_PROTOCOLS', int), + # 2xx + ('OK', int), + # 3xx + ('MOVED_PERMANENTLY', int), + ('SEE_OTHER', int), + ('TEMPORARY_REDIRECT', int), + ('PERMANENT_REDIRECT', int), + # 4xx + ('BAD_REQUEST', int), + ('UNAUTHORIZED', int), + ('FORBIDDEN', int), + ('NOT_FOUND', int), + ('PROXY_AUTH_REQUIRED', int), + ('REQUEST_TIMEOUT', int), + ('I_AM_A_TEAPOT', int), + # 5xx + ('INTERNAL_SERVER_ERROR', int), + ('NOT_IMPLEMENTED', int), + ('BAD_GATEWAY', int), + ('GATEWAY_TIMEOUT', int), + ('NETWORK_READ_TIMEOUT_ERROR', int), + ('NETWORK_CONNECT_TIMEOUT_ERROR', int), + ], +) + httpStatusCodes = HttpStatusCodes( 100, 101, 200, 301, 303, 307, 308, 400, 401, 403, 404, 407, 408, 418, - 500, 501, 502, 504, 598, 599 + 500, 501, 502, 504, 598, 599, ) diff --git a/proxy/http/exception/__init__.py b/proxy/http/exception/__init__.py index 513d2bd510..bb0bf7b1e8 100644 --- a/proxy/http/exception/__init__.py +++ b/proxy/http/exception/__init__.py @@ -7,6 +7,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + Submodules """ from .base import HttpProtocolException from .http_request_rejected import HttpRequestRejected diff --git a/proxy/http/exception/base.py b/proxy/http/exception/base.py index 65138e87b7..37817c9265 100644 --- a/proxy/http/exception/base.py +++ b/proxy/http/exception/base.py @@ -7,6 +7,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http """ from typing import Optional @@ -14,11 +18,12 @@ class HttpProtocolException(Exception): - """Top level HttpProtocolException exception class. + """Top level :exc:`HttpProtocolException` exception class. - All exceptions raised during execution of Http request lifecycle MUST - inherit HttpProtocolException base class. Implement response() method - to optionally return custom response to client.""" + All exceptions raised during execution of HTTP request lifecycle MUST + inherit :exc:`HttpProtocolException` base class. Implement + ``response()`` method to optionally return custom response to client. + """ def response(self, request: HttpParser) -> Optional[memoryview]: return None # pragma: no cover diff --git a/proxy/http/exception/http_request_rejected.py b/proxy/http/exception/http_request_rejected.py index 46fd9b04a0..d17571bd26 100644 --- a/proxy/http/exception/http_request_rejected.py +++ b/proxy/http/exception/http_request_rejected.py @@ -7,6 +7,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http """ from typing import Optional, Dict @@ -21,11 +25,13 @@ class HttpRequestRejected(HttpProtocolException): Connections can either be dropped/closed or optionally an HTTP status code can be returned.""" - def __init__(self, - status_code: Optional[int] = None, - reason: Optional[bytes] = None, - headers: Optional[Dict[bytes, bytes]] = None, - body: Optional[bytes] = None): + def __init__( + self, + status_code: Optional[int] = None, + reason: Optional[bytes] = None, + headers: Optional[Dict[bytes, bytes]] = None, + body: Optional[bytes] = None, + ): self.status_code: Optional[int] = status_code self.reason: Optional[bytes] = reason self.headers: Optional[Dict[bytes, bytes]] = headers @@ -33,10 +39,12 @@ def __init__(self, def response(self, _request: HttpParser) -> Optional[memoryview]: if self.status_code: - return memoryview(build_http_response( - status_code=self.status_code, - reason=self.reason, - headers=self.headers, - body=self.body - )) + return memoryview( + build_http_response( + status_code=self.status_code, + reason=self.reason, + headers=self.headers, + body=self.body, + ), + ) return None diff --git a/proxy/http/exception/proxy_auth_failed.py b/proxy/http/exception/proxy_auth_failed.py index ae1c6a4443..d82211d092 100644 --- a/proxy/http/exception/proxy_auth_failed.py +++ b/proxy/http/exception/proxy_auth_failed.py @@ -7,28 +7,37 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + auth + http """ from .base import HttpProtocolException -from ..parser import HttpParser + from ..codes import httpStatusCodes +from ..parser import HttpParser from ...common.constants import PROXY_AGENT_HEADER_VALUE, PROXY_AGENT_HEADER_KEY from ...common.utils import build_http_response class ProxyAuthenticationFailed(HttpProtocolException): - """Exception raised when Http Proxy auth is enabled and + """Exception raised when HTTP Proxy auth is enabled and incoming request doesn't present necessary credentials.""" - RESPONSE_PKT = memoryview(build_http_response( - httpStatusCodes.PROXY_AUTH_REQUIRED, - reason=b'Proxy Authentication Required', - headers={ - PROXY_AGENT_HEADER_KEY: PROXY_AGENT_HEADER_VALUE, - b'Proxy-Authenticate': b'Basic', - b'Connection': b'close', - }, - body=b'Proxy Authentication Required')) + RESPONSE_PKT = memoryview( + build_http_response( + httpStatusCodes.PROXY_AUTH_REQUIRED, + reason=b'Proxy Authentication Required', + headers={ + PROXY_AGENT_HEADER_KEY: PROXY_AGENT_HEADER_VALUE, + b'Proxy-Authenticate': b'Basic', + b'Connection': b'close', + }, + body=b'Proxy Authentication Required', + ), + ) def response(self, _request: HttpParser) -> memoryview: return self.RESPONSE_PKT diff --git a/proxy/http/exception/proxy_conn_failed.py b/proxy/http/exception/proxy_conn_failed.py index 0cec224277..86d13cc880 100644 --- a/proxy/http/exception/proxy_conn_failed.py +++ b/proxy/http/exception/proxy_conn_failed.py @@ -7,27 +7,35 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + conn + http """ from .base import HttpProtocolException -from ..parser import HttpParser + from ..codes import httpStatusCodes +from ..parser import HttpParser from ...common.constants import PROXY_AGENT_HEADER_VALUE, PROXY_AGENT_HEADER_KEY from ...common.utils import build_http_response class ProxyConnectionFailed(HttpProtocolException): - """Exception raised when HttpProxyPlugin is unable to establish connection to upstream server.""" - - RESPONSE_PKT = memoryview(build_http_response( - httpStatusCodes.BAD_GATEWAY, - reason=b'Bad Gateway', - headers={ - PROXY_AGENT_HEADER_KEY: PROXY_AGENT_HEADER_VALUE, - b'Connection': b'close' - }, - body=b'Bad Gateway' - )) + """Exception raised when ``HttpProxyPlugin`` is unable to establish connection to upstream server.""" + + RESPONSE_PKT = memoryview( + build_http_response( + httpStatusCodes.BAD_GATEWAY, + reason=b'Bad Gateway', + headers={ + PROXY_AGENT_HEADER_KEY: PROXY_AGENT_HEADER_VALUE, + b'Connection': b'close', + }, + body=b'Bad Gateway', + ), + ) def __init__(self, host: str, port: int, reason: str): self.host: str = host diff --git a/proxy/http/handler.py b/proxy/http/handler.py index 474ef62260..91c96624ca 100644 --- a/proxy/http/handler.py +++ b/proxy/http/handler.py @@ -7,30 +7,32 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http """ -import argparse -import socket -import selectors import ssl import time -import contextlib import errno +import socket +import asyncio import logging +import selectors -from typing import Tuple, List, Union, Optional, Generator, Dict -from uuid import UUID +from typing import Tuple, List, Union, Optional, Dict, Any from .plugin import HttpProtocolHandlerPlugin from .parser import HttpParser, httpParserStates, httpParserTypes from .exception import HttpProtocolException from ..common.types import Readables, Writables -from ..common.utils import wrap_socket -from ..core.acceptor.work import Work -from ..core.event import EventQueue +from ..common.utils import wrap_socket, is_threadless +from ..core.base import BaseTcpServerHandler from ..core.connection import TcpClientConnection from ..common.flag import flags -from ..common.constants import DEFAULT_CLIENT_RECVBUF_SIZE, DEFAULT_KEY_FILE, DEFAULT_TIMEOUT +from ..common.constants import DEFAULT_CLIENT_RECVBUF_SIZE, DEFAULT_KEY_FILE +from ..common.constants import DEFAULT_SELECTOR_SELECT_TIMEOUT, DEFAULT_TIMEOUT logger = logging.getLogger(__name__) @@ -43,13 +45,14 @@ help='Default: 1 MB. Maximum amount of data received from the ' 'client in a single recv() operation. Bump this ' 'value for faster uploads at the expense of ' - 'increased RAM.') + 'increased RAM.', +) flags.add_argument( '--key-file', type=str, default=DEFAULT_KEY_FILE, help='Default: None. Server key file to enable end-to-end TLS encryption with clients. ' - 'If used, must also pass --cert-file.' + 'If used, must also pass --cert-file.', ) flags.add_argument( '--timeout', @@ -58,300 +61,285 @@ help='Default: ' + str(DEFAULT_TIMEOUT) + '. Number of seconds after which ' 'an inactive connection must be dropped. Inactivity is defined by no ' - 'data sent or received by the client.' + 'data sent or received by the client.', ) -class HttpProtocolHandler(Work): +class HttpProtocolHandler(BaseTcpServerHandler): """HTTP, HTTPS, HTTP2, WebSockets protocol handler. - Accepts `Client` connection object and manages HttpProtocolHandlerPlugin invocations. + Accepts `Client` connection and delegates to HttpProtocolHandlerPlugin. """ - def __init__(self, client: TcpClientConnection, - flags: argparse.Namespace, - event_queue: Optional[EventQueue] = None, - uid: Optional[UUID] = None): - super().__init__(client, flags, event_queue, uid) - + def __init__(self, *args: Any, **kwargs: Any): + super().__init__(*args, **kwargs) self.start_time: float = time.time() self.last_activity: float = self.start_time - self.request: HttpParser = HttpParser(httpParserTypes.REQUEST_PARSER) - self.response: HttpParser = HttpParser(httpParserTypes.RESPONSE_PARSER) - self.selector = selectors.DefaultSelector() - self.client: TcpClientConnection = client + self.request: HttpParser = HttpParser( + httpParserTypes.REQUEST_PARSER, + enable_proxy_protocol=self.flags.enable_proxy_protocol, + ) + self.selector: Optional[selectors.DefaultSelector] = None + if not is_threadless(self.flags.threadless, self.flags.threaded): + self.selector = selectors.DefaultSelector() self.plugins: Dict[str, HttpProtocolHandlerPlugin] = {} - def encryption_enabled(self) -> bool: - return self.flags.keyfile is not None and \ - self.flags.certfile is not None + ## + # initialize, is_inactive, shutdown, get_events, handle_events + # overrides Work class definitions. + ## def initialize(self) -> None: - """Optionally upgrades connection to HTTPS, set conn in non-blocking mode and initializes plugins.""" - conn = self.optionally_wrap_socket(self.client.connection) + """Optionally upgrades connection to HTTPS, set ``conn`` in non-blocking mode and initializes plugins.""" + conn = self._optionally_wrap_socket(self.work.connection) conn.setblocking(False) - if self.encryption_enabled(): - self.client = TcpClientConnection(conn=conn, addr=self.client.addr) + # Update client connection reference if connection was wrapped + if self._encryption_enabled(): + self.work = TcpClientConnection(conn=conn, addr=self.work.addr) if b'HttpProtocolHandlerPlugin' in self.flags.plugins: for klass in self.flags.plugins[b'HttpProtocolHandlerPlugin']: - instance = klass( + instance: HttpProtocolHandlerPlugin = klass( self.uid, self.flags, - self.client, + self.work, self.request, - self.event_queue) + self.event_queue, + ) self.plugins[instance.name()] = instance - logger.debug('Handling connection %r' % self.client.connection) + logger.debug('Handling connection %r' % self.work.connection) def is_inactive(self) -> bool: - if not self.client.has_buffer() and \ - self.connection_inactive_for() > self.flags.timeout: + if not self.work.has_buffer() and \ + self._connection_inactive_for() > self.flags.timeout: return True return False - def get_events(self) -> Dict[socket.socket, int]: - events: Dict[socket.socket, int] = { - self.client.connection: selectors.EVENT_READ - } - if self.client.has_buffer(): - events[self.client.connection] |= selectors.EVENT_WRITE + def shutdown(self) -> None: + try: + # Flush pending buffer in threaded mode only. + # For threadless mode, BaseTcpServerHandler implements + # the must_flush_before_shutdown logic automagically. + if self.selector and self.work.has_buffer(): + self._flush() + # Invoke plugin.on_client_connection_close + for plugin in self.plugins.values(): + plugin.on_client_connection_close() + logger.debug( + 'Closing client connection %r ' + 'at address %s has buffer %s' % + (self.work.connection, self.work.address, self.work.has_buffer()), + ) + conn = self.work.connection + # Unwrap if wrapped before shutdown. + if self._encryption_enabled() and \ + isinstance(self.work.connection, ssl.SSLSocket): + conn = self.work.connection.unwrap() + conn.shutdown(socket.SHUT_WR) + logger.debug('Client connection shutdown successful') + except OSError: + pass + finally: + self.work.connection.close() + logger.debug('Client connection closed') + super().shutdown() + async def get_events(self) -> Dict[int, int]: + # Get default client events + events: Dict[int, int] = await super().get_events() # HttpProtocolHandlerPlugin.get_descriptors for plugin in self.plugins.values(): plugin_read_desc, plugin_write_desc = plugin.get_descriptors() - for r in plugin_read_desc: - if r not in events: - events[r] = selectors.EVENT_READ + for rfileno in plugin_read_desc: + if rfileno not in events: + events[rfileno] = selectors.EVENT_READ else: - events[r] |= selectors.EVENT_READ - for w in plugin_write_desc: - if w not in events: - events[w] = selectors.EVENT_WRITE + events[rfileno] |= selectors.EVENT_READ + for wfileno in plugin_write_desc: + if wfileno not in events: + events[wfileno] = selectors.EVENT_WRITE else: - events[w] |= selectors.EVENT_WRITE - + events[wfileno] |= selectors.EVENT_WRITE return events - def handle_events( + # We override super().handle_events and never call it + async def handle_events( self, readables: Readables, - writables: Writables) -> bool: - """Returns True if proxy must teardown.""" + writables: Writables, + ) -> bool: + """Returns True if proxy must tear down.""" # Flush buffer for ready to write sockets - teardown = self.handle_writables(writables) + teardown = await self.handle_writables(writables) if teardown: return True - # Invoke plugin.write_to_descriptors for plugin in self.plugins.values(): - teardown = plugin.write_to_descriptors(writables) + teardown = await plugin.write_to_descriptors(writables) if teardown: return True - # Read from ready to read sockets - teardown = self.handle_readables(readables) + teardown = await self.handle_readables(readables) if teardown: return True - # Invoke plugin.read_from_descriptors for plugin in self.plugins.values(): - teardown = plugin.read_from_descriptors(readables) + teardown = await plugin.read_from_descriptors(readables) if teardown: return True - return False - def shutdown(self) -> None: - try: - # Flush pending buffer if any - self.flush() - - # Invoke plugin.on_client_connection_close - for plugin in self.plugins.values(): - plugin.on_client_connection_close() - - logger.debug( - 'Closing client connection %r ' - 'at address %r has buffer %s' % - (self.client.connection, self.client.addr, self.client.has_buffer())) - - conn = self.client.connection - # Unwrap if wrapped before shutdown. - if self.encryption_enabled() and \ - isinstance(self.client.connection, ssl.SSLSocket): - conn = self.client.connection.unwrap() - conn.shutdown(socket.SHUT_WR) - logger.debug('Client connection shutdown successful') - except OSError: - pass - finally: - self.client.connection.close() - logger.debug('Client connection closed') - super().shutdown() - - def optionally_wrap_socket( - self, conn: socket.socket) -> Union[ssl.SSLSocket, socket.socket]: - """Attempts to wrap accepted client connection using provided certificates. - - Shutdown and closes client connection upon error. - """ - if self.encryption_enabled(): - assert self.flags.keyfile and self.flags.certfile - conn = wrap_socket(conn, self.flags.keyfile, self.flags.certfile) - return conn - - def connection_inactive_for(self) -> float: - return time.time() - self.last_activity + def handle_data(self, data: memoryview) -> Optional[bool]: + if data is None: + logger.debug('Client closed connection, tearing down...') + self.work.closed = True + return True - def flush(self) -> None: - if not self.client.has_buffer(): - return try: - self.selector.register( - self.client.connection, - selectors.EVENT_WRITE) - while self.client.has_buffer(): - ev: List[Tuple[selectors.SelectorKey, int] - ] = self.selector.select(timeout=1) - if len(ev) == 0: - continue - self.client.flush() - except BrokenPipeError: - pass - finally: - self.selector.unregister(self.client.connection) + # HttpProtocolHandlerPlugin.on_client_data + # Can raise HttpProtocolException to tear down the connection + for plugin in self.plugins.values(): + optional_data = plugin.on_client_data(data) + if optional_data is None: + break + data = optional_data + # Don't parse incoming data any further after 1st request has completed. + # + # This specially does happen for pipeline requests. + # + # Plugins can utilize on_client_data for such cases and + # apply custom logic to handle request data sent after 1st + # valid request. + if data and self.request.state != httpParserStates.COMPLETE: + # Parse http request + # TODO(abhinavsingh): Remove .tobytes after parser is + # memoryview compliant + self.request.parse(data.tobytes()) + if self.request.state == httpParserStates.COMPLETE: + # Invoke plugin.on_request_complete + for plugin in self.plugins.values(): + upgraded_sock = plugin.on_request_complete() + if isinstance(upgraded_sock, ssl.SSLSocket): + logger.debug( + 'Updated client conn to %s', upgraded_sock, + ) + self.work._conn = upgraded_sock + for plugin_ in self.plugins.values(): + if plugin_ != plugin: + plugin_.client._conn = upgraded_sock + elif isinstance(upgraded_sock, bool) and upgraded_sock is True: + return True + except HttpProtocolException as e: + logger.debug('HttpProtocolException raised') + response: Optional[memoryview] = e.response(self.request) + if response: + self.work.queue(response) + return True + return False - def handle_writables(self, writables: Writables) -> bool: - if self.client.has_buffer() and self.client.connection in writables: + async def handle_writables(self, writables: Writables) -> bool: + if self.work.connection.fileno() in writables and self.work.has_buffer(): logger.debug('Client is ready for writes, flushing buffer') self.last_activity = time.time() # TODO(abhinavsingh): This hook could just reside within server recv block # instead of invoking when flushed to client. + # # Invoke plugin.on_response_chunk - chunk = self.client.buffer + chunk = self.work.buffer for plugin in self.plugins.values(): chunk = plugin.on_response_chunk(chunk) if chunk is None: break try: - self.client.flush() + # Call super() for client flush + teardown = await super().handle_writables(writables) + if teardown: + return True except BrokenPipeError: logger.error( - 'BrokenPipeError when flushing buffer for client') + 'BrokenPipeError when flushing buffer for client', + ) return True except OSError: logger.error('OSError when flushing buffer to client') return True return False - def handle_readables(self, readables: Readables) -> bool: - if self.client.connection in readables: + async def handle_readables(self, readables: Readables) -> bool: + if self.work.connection.fileno() in readables: logger.debug('Client is ready for reads, reading') self.last_activity = time.time() try: - client_data = self.client.recv(self.flags.client_recvbuf_size) + teardown = await super().handle_readables(readables) + if teardown: + return teardown except ssl.SSLWantReadError: # Try again later logger.warning( - 'SSLWantReadError encountered while reading from client, will retry ...') + 'SSLWantReadError encountered while reading from client, will retry ...', + ) return False except socket.error as e: if e.errno == errno.ECONNRESET: - logger.warning('%r' % e) + # Most requests for mobile devices will end up + # with client closed connection. Using `debug` + # here to avoid flooding the logs. + logger.debug('%r' % e) else: - logger.exception( - 'Exception while receiving from %s connection %r with reason %r' % - (self.client.tag, self.client.connection, e)) + logger.warning( + 'Exception when receiving from %s connection#%d with reason %r' % + (self.work.tag, self.work.connection.fileno(), e), + ) return True + return False - if client_data is None: - logger.debug('Client closed connection, tearing down...') - self.client.closed = True - return True + ## + # Internal methods + ## - try: - # HttpProtocolHandlerPlugin.on_client_data - # Can raise HttpProtocolException to teardown the connection - plugin_index = 0 - plugins = list(self.plugins.values()) - while plugin_index < len(plugins) and client_data: - client_data = plugins[plugin_index].on_client_data( - client_data) - if client_data is None: - break - plugin_index += 1 - - # Don't parse request any further after 1st request has completed. - # This specially does happen for pipeline requests. - # Plugins can utilize on_client_data for such cases and - # apply custom logic to handle request data sent after 1st - # valid request. - if client_data and self.request.state != httpParserStates.COMPLETE: - # Parse http request - # TODO(abhinavsingh): Remove .tobytes after parser is - # memoryview compliant - self.request.parse(client_data.tobytes()) - if self.request.state == httpParserStates.COMPLETE: - # Invoke plugin.on_request_complete - for plugin in self.plugins.values(): - upgraded_sock = plugin.on_request_complete() - if isinstance(upgraded_sock, ssl.SSLSocket): - logger.debug( - 'Updated client conn to %s', upgraded_sock) - self.client._conn = upgraded_sock - for plugin_ in self.plugins.values(): - if plugin_ != plugin: - plugin_.client._conn = upgraded_sock - elif isinstance(upgraded_sock, bool) and upgraded_sock is True: - return True - except HttpProtocolException as e: - logger.debug( - 'HttpProtocolException type raised') - response: Optional[memoryview] = e.response(self.request) - if response: - self.client.queue(response) - return True - return False + def _encryption_enabled(self) -> bool: + return self.flags.keyfile is not None and \ + self.flags.certfile is not None - @contextlib.contextmanager - def selected_events(self) -> \ - Generator[Tuple[Readables, Writables], - None, None]: - events = self.get_events() - for fd in events: - self.selector.register(fd, events[fd]) - ev = self.selector.select(timeout=1) - readables = [] - writables = [] - for key, mask in ev: - if mask & selectors.EVENT_READ: - readables.append(key.fileobj) - if mask & selectors.EVENT_WRITE: - writables.append(key.fileobj) - yield (readables, writables) - for fd in events.keys(): - self.selector.unregister(fd) + def _optionally_wrap_socket( + self, conn: socket.socket, + ) -> Union[ssl.SSLSocket, socket.socket]: + """Attempts to wrap accepted client connection using provided certificates. - def run_once(self) -> bool: - with self.selected_events() as (readables, writables): - teardown = self.handle_events(readables, writables) - if teardown: - return True - return False + Shutdown and closes client connection upon error. + """ + if self._encryption_enabled(): + assert self.flags.keyfile and self.flags.certfile + # TODO(abhinavsingh): Insecure TLS versions must not be accepted by default + conn = wrap_socket(conn, self.flags.keyfile, self.flags.certfile) + return conn + + def _connection_inactive_for(self) -> float: + return time.time() - self.last_activity + + ## + # run() and _run_once() are here to maintain backward compatibility + # with threaded mode. These methods are only called when running + # in threaded mode. + ## def run(self) -> None: + """run() method is not used when in --threadless mode. + + This is here just to maintain backward compatibility with threaded mode. + """ + loop = asyncio.new_event_loop() try: self.initialize() while True: - # Teardown if client buffer is empty and connection is inactive + # Tear down if client buffer is empty and connection is inactive if self.is_inactive(): logger.debug( 'Client buffer is empty and maximum inactivity has reached ' - 'between client and server connection, tearing down...') + 'between client and server connection, tearing down...', + ) break - teardown = self.run_once() - if teardown: + if loop.run_until_complete(self._run_once()): break except KeyboardInterrupt: # pragma: no cover pass @@ -360,6 +348,58 @@ def run(self) -> None: except Exception as e: logger.exception( 'Exception while handling connection %r' % - self.client.connection, exc_info=e) + self.work.connection, exc_info=e, + ) finally: self.shutdown() + loop.close() + + async def _run_once(self) -> bool: + events, readables, writables = await self._selected_events() + try: + return await self.handle_events(readables, writables) + finally: + assert self.selector + # TODO: Like Threadless we should not unregister + # work fds repeatedly. + for fd in events: + self.selector.unregister(fd) + + # FIXME: Returning events is only necessary because we cannot use async context manager + # for < Python 3.8. As a reason, this method is no longer a context manager and caller + # is responsible for unregistering the descriptors. + async def _selected_events(self) -> Tuple[Dict[int, int], Readables, Writables]: + assert self.selector + events = await self.get_events() + for fd in events: + self.selector.register(fd, events[fd]) + ev = self.selector.select(timeout=DEFAULT_SELECTOR_SELECT_TIMEOUT) + readables = [] + writables = [] + for key, mask in ev: + if mask & selectors.EVENT_READ: + readables.append(key.fileobj) + if mask & selectors.EVENT_WRITE: + writables.append(key.fileobj) + return (events, readables, writables) + + def _flush(self) -> None: + assert self.selector + logger.debug('Flushing pending data') + try: + self.selector.register( + self.work.connection, + selectors.EVENT_WRITE, + ) + while self.work.has_buffer(): + logging.debug('Waiting for client read ready') + ev: List[ + Tuple[selectors.SelectorKey, int] + ] = self.selector.select(timeout=DEFAULT_SELECTOR_SELECT_TIMEOUT) + if len(ev) == 0: + continue + self.work.flush() + except BrokenPipeError: + pass + finally: + self.selector.unregister(self.work.connection) diff --git a/proxy/http/inspector/__init__.py b/proxy/http/inspector/__init__.py index 8e5d4aa017..6e968ddebe 100644 --- a/proxy/http/inspector/__init__.py +++ b/proxy/http/inspector/__init__.py @@ -7,6 +7,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + Submodules """ from .devtools import DevtoolsProtocolPlugin diff --git a/proxy/http/inspector/devtools.py b/proxy/http/inspector/devtools.py index 50b34192a4..ee2a4d8e67 100644 --- a/proxy/http/inspector/devtools.py +++ b/proxy/http/inspector/devtools.py @@ -7,6 +7,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + devtools + http """ import json import logging @@ -20,17 +25,25 @@ from ...common.utils import bytes_, text_ from ...core.event import EventSubscriber from ...common.flag import flags -from ...common.constants import DEFAULT_DEVTOOLS_WS_PATH +from ...common.constants import DEFAULT_DEVTOOLS_WS_PATH, DEFAULT_DEVTOOLS_DOC_URL +from ...common.constants import DEFAULT_ENABLE_DEVTOOLS logger = logging.getLogger(__name__) +flags.add_argument( + '--enable-devtools', + action='store_true', + default=DEFAULT_ENABLE_DEVTOOLS, + help='Default: False. Enables integration with Chrome Devtool Frontend. Also see --devtools-ws-path.', +) + flags.add_argument( '--devtools-ws-path', type=str, default=DEFAULT_DEVTOOLS_WS_PATH, help='Default: /devtools. Only applicable ' - 'if --enable-devtools is used.' + 'if --enable-devtools is used.', ) @@ -44,23 +57,25 @@ class DevtoolsProtocolPlugin(HttpWebServerBasePlugin): - Core events unrelated to DevTools protocol are dropped. """ - DOC_URL = 'http://dashboard.proxy.py' - def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) - self.subscriber = EventSubscriber(self.event_queue) + self.subscriber = EventSubscriber( + self.event_queue, + callback=lambda event: CoreEventsToDevtoolsProtocol.transformer( + self.client, event, + ), + ) def routes(self) -> List[Tuple[int, str]]: return [ - (httpProtocolTypes.WEBSOCKET, text_(self.flags.devtools_ws_path)) + (httpProtocolTypes.WEBSOCKET, text_(self.flags.devtools_ws_path)), ] def handle_request(self, request: HttpParser) -> None: raise NotImplementedError('This should have never been called') def on_websocket_open(self) -> None: - self.subscriber.subscribe( - lambda event: CoreEventsToDevtoolsProtocol.transformer(self.client, event)) + self.subscriber.setup() def on_websocket_message(self, frame: WebsocketFrame) -> None: try: @@ -72,14 +87,13 @@ def on_websocket_message(self, frame: WebsocketFrame) -> None: return self.handle_devtools_message(message) - def on_websocket_close(self) -> None: - self.subscriber.unsubscribe() + def on_client_connection_close(self) -> None: + self.subscriber.shutdown() def handle_devtools_message(self, message: Dict[str, Any]) -> None: frame = WebsocketFrame() frame.fin = True frame.opcode = websocketOpcodes.TEXT_FRAME - # logger.info(message) method = message['method'] if method in ( @@ -88,7 +102,7 @@ def handle_devtools_message(self, message: Dict[str, Any]) -> None: 'Emulation.canEmulate', ): data: Dict[str, Any] = { - 'result': False + 'result': False, } elif method == 'Page.getResourceTree': data = { @@ -96,13 +110,13 @@ def handle_devtools_message(self, message: Dict[str, Any]) -> None: 'frameTree': { 'frame': { 'id': 1, - 'url': DevtoolsProtocolPlugin.DOC_URL, + 'url': DEFAULT_DEVTOOLS_DOC_URL, 'mimeType': 'other', }, 'childFrames': [], - 'resources': [] - } - } + 'resources': [], + }, + }, } elif method == 'Network.getResponseBody': connection_id = message['params']['requestId'] @@ -110,12 +124,11 @@ def handle_devtools_message(self, message: Dict[str, Any]) -> None: 'result': { 'body': text_(CoreEventsToDevtoolsProtocol.RESPONSES[connection_id]), 'base64Encoded': False, - } + }, } else: logging.warning('Unhandled devtools method %s', method) data = {} - data['id'] = message['id'] frame.data = bytes_(json.dumps(data)) self.client.queue(memoryview(frame.build())) diff --git a/proxy/http/inspector/transformer.py b/proxy/http/inspector/transformer.py index 450ce7bba1..97fec409b3 100644 --- a/proxy/http/inspector/transformer.py +++ b/proxy/http/inspector/transformer.py @@ -7,36 +7,43 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http """ import json -import secrets import time from typing import Any, Dict from ..websocket import WebsocketFrame -from ...common.constants import PROXY_PY_START_TIME +from ...common.constants import PROXY_PY_START_TIME, DEFAULT_DEVTOOLS_DOC_URL +from ...common.constants import DEFAULT_DEVTOOLS_FRAME_ID, DEFAULT_DEVTOOLS_LOADER_ID from ...common.utils import bytes_ from ...core.connection import TcpClientConnection from ...core.event import eventNames class CoreEventsToDevtoolsProtocol: + """Open in Chrome - DOC_URL = 'http://dashboard.proxy.py' - FRAME_ID = secrets.token_hex(8) - LOADER_ID = secrets.token_hex(8) + ``devtools://devtools/bundled/inspector.html?ws=localhost:8899/devtools`` + """ RESPONSES: Dict[str, bytes] = {} @staticmethod - def transformer(client: TcpClientConnection, - event: Dict[str, Any]) -> None: + def transformer( + client: TcpClientConnection, + event: Dict[str, Any], + ) -> None: event_name = event['event_name'] if event_name == eventNames.REQUEST_COMPLETE: data = CoreEventsToDevtoolsProtocol.request_complete(event) elif event_name == eventNames.RESPONSE_HEADERS_COMPLETE: data = CoreEventsToDevtoolsProtocol.response_headers_complete( - event) + event, + ) elif event_name == eventNames.RESPONSE_CHUNK_RECEIVED: data = CoreEventsToDevtoolsProtocol.response_chunk_received(event) elif event_name == eventNames.RESPONSE_COMPLETE: @@ -45,98 +52,113 @@ def transformer(client: TcpClientConnection, # drop core events unrelated to Devtools return client.queue( - memoryview(WebsocketFrame.text( - bytes_( - json.dumps(data))))) + memoryview( + WebsocketFrame.text( + bytes_( + json.dumps(data), + ), + ), + ), + ) @staticmethod def request_complete(event: Dict[str, Any]) -> Dict[str, Any]: now = time.time() return { - 'requestId': event['request_id'], - 'frameId': CoreEventsToDevtoolsProtocol.FRAME_ID, - 'loaderId': CoreEventsToDevtoolsProtocol.LOADER_ID, - 'documentURL': CoreEventsToDevtoolsProtocol.DOC_URL, - 'timestamp': now - PROXY_PY_START_TIME, - 'wallTime': now, - 'hasUserGesture': False, - 'type': event['event_payload']['headers']['content-type'] - if 'content-type' in event['event_payload']['headers'] - else 'Other', - # TODO(abhinavsingh): Bring this inline with devtools protocol 'method': 'Network.requestWillBeSent', - 'request': { - 'url': event['event_payload']['url'], - 'method': event['event_payload']['method'], - 'headers': event['event_payload']['headers'], - 'postData': event['event_payload']['body'], - 'initialPriority': 'High', - 'urlFragment': '', - 'mixedContentType': 'none', - }, - 'initiator': { - 'type': 'other' + 'params': { + 'requestId': event['request_id'], + 'frameId': DEFAULT_DEVTOOLS_FRAME_ID, + 'loaderId': DEFAULT_DEVTOOLS_LOADER_ID, + 'documentURL': DEFAULT_DEVTOOLS_DOC_URL, + 'timestamp': now - PROXY_PY_START_TIME, + 'wallTime': now, + 'hasUserGesture': False, + 'type': event['event_payload']['headers']['content-type'] + if 'content-type' in event['event_payload']['headers'] + else 'Other', + 'request': { + 'url': event['event_payload']['url'], + 'method': event['event_payload']['method'], + 'headers': event['event_payload']['headers'], + 'postData': event['event_payload']['body'], + 'initialPriority': 'High', + 'urlFragment': '', + 'mixedContentType': 'none', + }, + 'initiator': { + 'type': 'other', + }, }, } @staticmethod def response_headers_complete(event: Dict[str, Any]) -> Dict[str, Any]: return { - 'requestId': event['request_id'], - 'frameId': CoreEventsToDevtoolsProtocol.FRAME_ID, - 'loaderId': CoreEventsToDevtoolsProtocol.LOADER_ID, - 'timestamp': time.time(), - 'type': event['event_payload']['headers']['content-type'] - if event['event_payload']['headers'].has_header('content-type') - else 'Other', - 'response': { - 'url': '', - 'status': '', - 'statusText': '', - 'headers': '', - 'headersText': '', - 'mimeType': '', - 'connectionReused': True, - 'connectionId': '', - 'encodedDataLength': '', - 'fromDiskCache': False, - 'fromServiceWorker': False, - 'timing': { - 'requestTime': '', - 'proxyStart': -1, - 'proxyEnd': -1, - 'dnsStart': -1, - 'dnsEnd': -1, - 'connectStart': -1, - 'connectEnd': -1, - 'sslStart': -1, - 'sslEnd': -1, - 'workerStart': -1, - 'workerReady': -1, - 'sendStart': 0, - 'sendEnd': 0, - 'receiveHeadersEnd': 0, + 'method': 'Network.responseReceived', + 'params': { + 'requestId': event['request_id'], + 'frameId': DEFAULT_DEVTOOLS_FRAME_ID, + 'loaderId': DEFAULT_DEVTOOLS_LOADER_ID, + 'timestamp': time.time(), + 'type': event['event_payload']['headers']['content-type'] + if event['event_payload']['headers'].has_header('content-type') + else 'Other', + 'response': { + 'url': '', + 'status': '', + 'statusText': '', + 'headers': '', + 'headersText': '', + 'mimeType': '', + 'connectionReused': True, + 'connectionId': '', + 'encodedDataLength': '', + 'fromDiskCache': False, + 'fromServiceWorker': False, + 'timing': { + 'requestTime': '', + 'proxyStart': -1, + 'proxyEnd': -1, + 'dnsStart': -1, + 'dnsEnd': -1, + 'connectStart': -1, + 'connectEnd': -1, + 'sslStart': -1, + 'sslEnd': -1, + 'workerStart': -1, + 'workerReady': -1, + 'sendStart': 0, + 'sendEnd': 0, + 'receiveHeadersEnd': 0, + }, + 'requestHeaders': '', + 'remoteIPAddress': '', + 'remotePort': '', }, - 'requestHeaders': '', - 'remoteIPAddress': '', - 'remotePort': '', - } + }, } @staticmethod def response_chunk_received(event: Dict[str, Any]) -> Dict[str, Any]: return { - 'requestId': event['request_id'], - 'timestamp': time.time(), - 'dataLength': event['event_payload']['chunk_size'], - 'encodedDataLength': event['event_payload']['encoded_chunk_size'], + 'method': 'Network.dataReceived', + 'params': { + 'requestId': event['request_id'], + 'timestamp': time.time(), + 'dataLength': event['event_payload']['chunk_size'], + 'encodedDataLength': event['event_payload']['encoded_chunk_size'], + }, } @staticmethod def response_complete(event: Dict[str, Any]) -> Dict[str, Any]: return { - 'requestId': event['request_id'], - 'timestamp': time.time(), - 'encodedDataLength': event['event_payload']['encoded_response_size'], - 'shouldReportCorbBlocking': False, + 'method': 'Network.loadingFinished', + 'params': { + 'requestId': event['request_id'], + 'timestamp': time.time(), + 'encodedDataLength': event['event_payload']['encoded_response_size'], + 'shouldReportCorbBlocking': False, + }, } diff --git a/proxy/http/methods.py b/proxy/http/methods.py index 63b8cd1e98..30c9b71a71 100644 --- a/proxy/http/methods.py +++ b/proxy/http/methods.py @@ -7,29 +7,100 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + iterable """ from typing import NamedTuple -HttpMethods = NamedTuple('HttpMethods', [ - ('GET', bytes), - ('HEAD', bytes), - ('POST', bytes), - ('PUT', bytes), - ('DELETE', bytes), - ('CONNECT', bytes), - ('OPTIONS', bytes), - ('TRACE', bytes), - ('PATCH', bytes), -]) +# Ref: https://www.iana.org/assignments/http-methods/http-methods.xhtml +HttpMethods = NamedTuple( + 'HttpMethods', [ + ('ACL', bytes), + ('BASELINE_CONTROL', bytes), + ('BIND', bytes), + ('CHECKIN', bytes), + ('CHECKOUT', bytes), + ('CONNECT', bytes), + ('COPY', bytes), + ('DELETE', bytes), + ('GET', bytes), + ('HEAD', bytes), + ('LABEL', bytes), + ('LINK', bytes), + ('LOCK', bytes), + ('MERGE', bytes), + ('MKACTIVITY', bytes), + ('MKCALENDAR', bytes), + ('MKCOL', bytes), + ('MKREDIRECTREF', bytes), + ('MKWORKSPACE', bytes), + ('MOVE', bytes), + ('OPTIONS', bytes), + ('ORDERPATCH', bytes), + ('PATCH', bytes), + ('POST', bytes), + ('PRI', bytes), + ('PROPFIND', bytes), + ('PROPPATCH', bytes), + ('PUT', bytes), + ('REBIND', bytes), + ('REPORT', bytes), + ('SEARCH', bytes), + ('TRACE', bytes), + ('UNBIND', bytes), + ('UNCHECKOUT', bytes), + ('UNLINK', bytes), + ('UNLOCK', bytes), + ('UPDATE', bytes), + ('UPDATEREDIRECTREF', bytes), + ('VERSION_CONTROL', bytes), + ('STAR', bytes), + ], +) + httpMethods = HttpMethods( + b'ACL', + b'BASELINE-CONTROL', + b'BIND', + b'CHECKIN', + b'CHECKOUT', + b'CONNECT', + b'COPY', + b'DELETE', b'GET', b'HEAD', + b'LABEL', + b'LINK', + b'LOCK', + b'MERGE', + b'MKACTIVITY', + b'MKCALENDAR', + b'MKCOL', + b'MKREDIRECTREF', + b'MKWORKSPACE', + b'MOVE', + b'OPTIONS', + b'ORDERPATCH', + b'PATCH', b'POST', + b'PRI', + b'PROPFIND', + b'PROPPATCH', b'PUT', - b'DELETE', - b'CONNECT', - b'OPTIONS', + b'REBIND', + b'REPORT', + b'SEARCH', b'TRACE', - b'PATCH', + b'UNBIND', + b'UNCHECKOUT', + b'UNLINK', + b'UNLOCK', + b'UPDATE', + b'UPDATEREDIRECTREF', + b'VERSION-CONTROL', + b'*', ) diff --git a/proxy/http/parser.py b/proxy/http/parser.py deleted file mode 100644 index 638959ac97..0000000000 --- a/proxy/http/parser.py +++ /dev/null @@ -1,278 +0,0 @@ -# -*- coding: utf-8 -*- -""" - proxy.py - ~~~~~~~~ - โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on - Network monitoring, controls & Application development, testing, debugging. - - :copyright: (c) 2013-present by Abhinav Singh and contributors. - :license: BSD, see LICENSE for more details. -""" -from urllib import parse as urlparse -from typing import TypeVar, NamedTuple, Optional, Dict, Type, Tuple, List - -from .methods import httpMethods -from .chunk_parser import ChunkParser, chunkParserStates - -from ..common.constants import DEFAULT_DISABLE_HEADERS, COLON, CRLF, WHITESPACE, HTTP_1_1, DEFAULT_HTTP_PORT -from ..common.utils import build_http_request, build_http_response, find_http_line, text_ - - -HttpParserStates = NamedTuple('HttpParserStates', [ - ('INITIALIZED', int), - ('LINE_RCVD', int), - ('RCVING_HEADERS', int), - ('HEADERS_COMPLETE', int), - ('RCVING_BODY', int), - ('COMPLETE', int), -]) -httpParserStates = HttpParserStates(1, 2, 3, 4, 5, 6) - -HttpParserTypes = NamedTuple('HttpParserTypes', [ - ('REQUEST_PARSER', int), - ('RESPONSE_PARSER', int), -]) -httpParserTypes = HttpParserTypes(1, 2) - - -T = TypeVar('T', bound='HttpParser') - - -class HttpParser: - """HTTP request/response parser.""" - - def __init__(self, parser_type: int) -> None: - self.type: int = parser_type - self.state: int = httpParserStates.INITIALIZED - - # Total size of raw bytes passed for parsing - self.total_size: int = 0 - - # Buffer to hold unprocessed bytes - self.buffer: bytes = b'' - - self.headers: Dict[bytes, Tuple[bytes, bytes]] = dict() - self.body: Optional[bytes] = None - - self.method: Optional[bytes] = None - self.url: Optional[urlparse.SplitResultBytes] = None - self.code: Optional[bytes] = None - self.reason: Optional[bytes] = None - self.version: Optional[bytes] = None - - self.chunk_parser: Optional[ChunkParser] = None - - # This cleans up developer APIs as Python urlparse.urlsplit behaves differently - # for incoming proxy request and incoming web request. Web request is the one - # which is broken. - self.host: Optional[bytes] = None - self.port: Optional[int] = None - self.path: Optional[bytes] = None - - @classmethod - def request(cls: Type[T], raw: bytes) -> T: - parser = cls(httpParserTypes.REQUEST_PARSER) - parser.parse(raw) - return parser - - @classmethod - def response(cls: Type[T], raw: bytes) -> T: - parser = cls(httpParserTypes.RESPONSE_PARSER) - parser.parse(raw) - return parser - - def header(self, key: bytes) -> bytes: - if key.lower() not in self.headers: - raise KeyError('%s not found in headers', text_(key)) - return self.headers[key.lower()][1] - - def has_header(self, key: bytes) -> bool: - return key.lower() in self.headers - - def add_header(self, key: bytes, value: bytes) -> None: - self.headers[key.lower()] = (key, value) - - def add_headers(self, headers: List[Tuple[bytes, bytes]]) -> None: - for (key, value) in headers: - self.add_header(key, value) - - def del_header(self, header: bytes) -> None: - if header.lower() in self.headers: - del self.headers[header.lower()] - - def del_headers(self, headers: List[bytes]) -> None: - for key in headers: - self.del_header(key.lower()) - - def set_url(self, url: bytes) -> None: - # Work around with urlsplit semantics. - # - # For CONNECT requests, request line contains - # upstream_host:upstream_port which is not complaint - # with urlsplit, which expects a fully qualified url. - if self.method == httpMethods.CONNECT: - url = b'https://' + url - self.url = urlparse.urlsplit(url) - self.set_line_attributes() - - def set_line_attributes(self) -> None: - if self.type == httpParserTypes.REQUEST_PARSER: - if self.method == httpMethods.CONNECT and self.url: - self.host = self.url.hostname - self.port = 443 if self.url.port is None else self.url.port - elif self.url: - self.host, self.port = self.url.hostname, self.url.port \ - if self.url.port else DEFAULT_HTTP_PORT - else: - raise KeyError( - 'Invalid request. Method: %r, Url: %r' % - (self.method, self.url)) - self.path = self.build_path() - - def is_chunked_encoded(self) -> bool: - return b'transfer-encoding' in self.headers and \ - self.headers[b'transfer-encoding'][1].lower() == b'chunked' - - def body_expected(self) -> bool: - return (b'content-length' in self.headers and - int(self.header(b'content-length')) > 0) or \ - self.is_chunked_encoded() - - def parse(self, raw: bytes) -> None: - """Parses Http request out of raw bytes. - - Check HttpParser state after parse has successfully returned.""" - self.total_size += len(raw) - raw = self.buffer + raw - self.buffer = b'' - - more = True if len(raw) > 0 else False - while more and self.state != httpParserStates.COMPLETE: - if self.state in ( - httpParserStates.HEADERS_COMPLETE, - httpParserStates.RCVING_BODY): - if b'content-length' in self.headers: - self.state = httpParserStates.RCVING_BODY - if self.body is None: - self.body = b'' - total_size = int(self.header(b'content-length')) - received_size = len(self.body) - self.body += raw[:total_size - received_size] - if self.body and \ - len(self.body) == int(self.header(b'content-length')): - self.state = httpParserStates.COMPLETE - more, raw = len(raw) > 0, raw[total_size - received_size:] - elif self.is_chunked_encoded(): - if not self.chunk_parser: - self.chunk_parser = ChunkParser() - raw = self.chunk_parser.parse(raw) - if self.chunk_parser.state == chunkParserStates.COMPLETE: - self.body = self.chunk_parser.body - self.state = httpParserStates.COMPLETE - more = False - else: - raise NotImplementedError( - 'Parser shouldn\'t have reached here') - else: - more, raw = self.process(raw) - self.buffer = raw - - def process(self, raw: bytes) -> Tuple[bool, bytes]: - """Returns False when no CRLF could be found in received bytes.""" - line, raw = find_http_line(raw) - if line is None: - return False, raw - - if self.state == httpParserStates.INITIALIZED: - self.process_line(line) - self.state = httpParserStates.LINE_RCVD - elif self.state in (httpParserStates.LINE_RCVD, httpParserStates.RCVING_HEADERS): - if self.state == httpParserStates.LINE_RCVD: - # LINE_RCVD state is equivalent to RCVING_HEADERS - self.state = httpParserStates.RCVING_HEADERS - if line.strip() == b'': # Blank line received. - self.state = httpParserStates.HEADERS_COMPLETE - else: - self.process_header(line) - - # When server sends a response line without any header or body e.g. - # HTTP/1.1 200 Connection established\r\n\r\n - if self.state == httpParserStates.LINE_RCVD and \ - self.type == httpParserTypes.RESPONSE_PARSER and \ - raw == CRLF: - self.state = httpParserStates.COMPLETE - elif self.state == httpParserStates.HEADERS_COMPLETE and \ - not self.body_expected() and \ - raw == b'': - self.state = httpParserStates.COMPLETE - - return len(raw) > 0, raw - - def process_line(self, raw: bytes) -> None: - line = raw.split(WHITESPACE) - if self.type == httpParserTypes.REQUEST_PARSER: - self.method = line[0].upper() - self.set_url(line[1]) - self.version = line[2] - else: - self.version = line[0] - self.code = line[1] - self.reason = WHITESPACE.join(line[2:]) - - def process_header(self, raw: bytes) -> None: - parts = raw.split(COLON) - key = parts[0].strip() - value = COLON.join(parts[1:]).strip() - self.add_headers([(key, value)]) - - def build_path(self) -> bytes: - if not self.url: - return b'/None' - url = self.url.path - if url == b'': - url = b'/' - if not self.url.query == b'': - url += b'?' + self.url.query - if not self.url.fragment == b'': - url += b'#' + self.url.fragment - return url - - def build(self, disable_headers: Optional[List[bytes]] = None) -> bytes: - """Rebuild the request object.""" - assert self.method and self.version and self.path and self.type == httpParserTypes.REQUEST_PARSER - if disable_headers is None: - disable_headers = DEFAULT_DISABLE_HEADERS - body: Optional[bytes] = ChunkParser.to_chunks(self.body) \ - if self.is_chunked_encoded() and self.body else \ - self.body - return build_http_request( - self.method, self.path, self.version, - headers={} if not self.headers else {self.headers[k][0]: self.headers[k][1] for k in self.headers if - k.lower() not in disable_headers}, - body=body - ) - - def build_response(self) -> bytes: - """Rebuild the response object.""" - assert self.code and self.version and self.body and self.type == httpParserTypes.RESPONSE_PARSER - return build_http_response( - status_code=int(self.code), - protocol_version=self.version, - reason=self.reason, - headers={} if not self.headers else { - self.headers[k][0]: self.headers[k][1] for k in self.headers}, - body=self.body if not self.is_chunked_encoded() else ChunkParser.to_chunks(self.body)) - - def has_upstream_server(self) -> bool: - """Host field SHOULD be None for incoming local WebServer requests.""" - return True if self.host is not None else False - - def is_http_1_1_keep_alive(self) -> bool: - return self.version == HTTP_1_1 and \ - (not self.has_header(b'Connection') or - self.header(b'Connection').lower() == b'keep-alive') - - def is_connection_upgrade(self) -> bool: - return self.version == HTTP_1_1 and \ - self.has_header(b'Connection') and \ - self.has_header(b'Upgrade') diff --git a/proxy/http/parser/__init__.py b/proxy/http/parser/__init__.py new file mode 100644 index 0000000000..be633d4873 --- /dev/null +++ b/proxy/http/parser/__init__.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + Submodules +""" +from .parser import HttpParser +from .chunk import ChunkParser, chunkParserStates +from .types import httpParserStates, httpParserTypes +from .protocol import ProxyProtocol, PROXY_PROTOCOL_V2_SIGNATURE + +__all__ = [ + 'HttpParser', + 'httpParserTypes', + 'httpParserStates', + 'ChunkParser', + 'chunkParserStates', + 'ProxyProtocol', + 'PROXY_PROTOCOL_V2_SIGNATURE', +] diff --git a/proxy/http/chunk_parser.py b/proxy/http/parser/chunk.py similarity index 88% rename from proxy/http/chunk_parser.py rename to proxy/http/parser/chunk.py index 2b9b72c42f..fa60d437df 100644 --- a/proxy/http/chunk_parser.py +++ b/proxy/http/parser/chunk.py @@ -7,18 +7,25 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + iterable """ from typing import NamedTuple, Tuple, List, Optional -from ..common.utils import bytes_, find_http_line -from ..common.constants import CRLF, DEFAULT_BUFFER_SIZE +from ...common.utils import bytes_, find_http_line +from ...common.constants import CRLF, DEFAULT_BUFFER_SIZE -ChunkParserStates = NamedTuple('ChunkParserStates', [ - ('WAITING_FOR_SIZE', int), - ('WAITING_FOR_DATA', int), - ('COMPLETE', int), -]) +ChunkParserStates = NamedTuple( + 'ChunkParserStates', [ + ('WAITING_FOR_SIZE', int), + ('WAITING_FOR_DATA', int), + ('COMPLETE', int), + ], +) chunkParserStates = ChunkParserStates(1, 2, 3) @@ -33,7 +40,7 @@ def __init__(self) -> None: self.size: Optional[int] = None def parse(self, raw: bytes) -> bytes: - more = True if len(raw) > 0 else False + more = len(raw) > 0 while more and self.state != chunkParserStates.COMPLETE: more, raw = self.process(raw) return raw diff --git a/proxy/http/parser/parser.py b/proxy/http/parser/parser.py new file mode 100644 index 0000000000..cff81abd1a --- /dev/null +++ b/proxy/http/parser/parser.py @@ -0,0 +1,357 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. + + .. spelling:: + + http +""" +from typing import TypeVar, Optional, Dict, Type, Tuple, List + +from ...common.constants import DEFAULT_DISABLE_HEADERS, COLON, DEFAULT_ENABLE_PROXY_PROTOCOL +from ...common.constants import HTTP_1_1, HTTP_1_0, SLASH, CRLF +from ...common.constants import WHITESPACE, DEFAULT_HTTP_PORT +from ...common.utils import build_http_request, build_http_response, find_http_line, text_ +from ...common.flag import flags + +from ..url import Url +from ..methods import httpMethods + +from .protocol import ProxyProtocol +from .chunk import ChunkParser, chunkParserStates +from .types import httpParserTypes, httpParserStates + +flags.add_argument( + '--enable-proxy-protocol', + action='store_true', + default=DEFAULT_ENABLE_PROXY_PROTOCOL, + help='Default: ' + str(DEFAULT_ENABLE_PROXY_PROTOCOL) + '. ' + + 'If used, will enable proxy protocol. ' + + 'Only version 1 is currently supported.', +) + + +T = TypeVar('T', bound='HttpParser') + + +class HttpParser: + """HTTP request/response parser. + + TODO: Make me zero-copy by using :class:`memoryview`. + Currently due to chunk/buffer handling we + are not able to utilize :class:`memoryview` + efficiently. + + For this to happen we must store ``buffer`` + as ``List[memoryview]`` instead of raw bytes and + update parser to work accordingly. + """ + + def __init__( + self, parser_type: int, + enable_proxy_protocol: int = DEFAULT_ENABLE_PROXY_PROTOCOL, + ) -> None: + self.state: int = httpParserStates.INITIALIZED + self.type: int = parser_type + self.protocol: Optional[ProxyProtocol] = None + if enable_proxy_protocol: + assert self.type == httpParserTypes.REQUEST_PARSER + self.protocol = ProxyProtocol() + self.host: Optional[bytes] = None + self.port: Optional[int] = None + self.path: Optional[bytes] = None + self.method: Optional[bytes] = None + self.code: Optional[bytes] = None + self.reason: Optional[bytes] = None + self.version: Optional[bytes] = None + # Total size of raw bytes passed for parsing + self.total_size: int = 0 + # Buffer to hold unprocessed bytes + self.buffer: bytes = b'' + # Internal headers datastructure: + # - Keys are lower case header names. + # - Values are 2-tuple containing original + # header and it's value as received. + self.headers: Dict[bytes, Tuple[bytes, bytes]] = {} + self.body: Optional[bytes] = None + self.chunk: Optional[ChunkParser] = None + # Internal request line as a url structure + self._url: Optional[Url] = None + + @classmethod + def request( + cls: Type[T], + raw: bytes, + enable_proxy_protocol: int = DEFAULT_ENABLE_PROXY_PROTOCOL, + ) -> T: + parser = cls( + httpParserTypes.REQUEST_PARSER, + enable_proxy_protocol=enable_proxy_protocol, + ) + parser.parse(raw) + return parser + + @classmethod + def response(cls: Type[T], raw: bytes) -> T: + parser = cls(httpParserTypes.RESPONSE_PARSER) + parser.parse(raw) + return parser + + def header(self, key: bytes) -> bytes: + """Convenient method to return original header value from internal data structure.""" + if key.lower() not in self.headers: + raise KeyError('%s not found in headers', text_(key)) + return self.headers[key.lower()][1] + + def has_header(self, key: bytes) -> bool: + """Returns true if header key was found in payload.""" + return key.lower() in self.headers + + def add_header(self, key: bytes, value: bytes) -> None: + """Add/Update a header to internal data structure.""" + self.headers[key.lower()] = (key, value) + + def add_headers(self, headers: List[Tuple[bytes, bytes]]) -> None: + """Add/Update multiple headers to internal data structure""" + for (key, value) in headers: + self.add_header(key, value) + + def del_header(self, header: bytes) -> None: + """Delete a header from internal data structure.""" + if header.lower() in self.headers: + del self.headers[header.lower()] + + def del_headers(self, headers: List[bytes]) -> None: + """Delete headers from internal data structure.""" + for key in headers: + self.del_header(key.lower()) + + def set_url(self, url: bytes) -> None: + """Given a request line, parses it and sets line attributes a.k.a. host, port, path.""" + self._url = Url.from_bytes(url) + self._set_line_attributes() + + def has_host(self) -> bool: + """Returns whether host line attribute was parsed or set. + + NOTE: Host field WILL be None for incoming local WebServer requests.""" + return self.host is not None + + def is_http_1_1_keep_alive(self) -> bool: + """Returns true for HTTP/1.1 keep-alive connections.""" + return self.version == HTTP_1_1 and \ + ( + not self.has_header(b'Connection') or + self.header(b'Connection').lower() == b'keep-alive' + ) + + def is_connection_upgrade(self) -> bool: + """Returns true for websocket upgrade requests.""" + return self.version == HTTP_1_1 and \ + self.has_header(b'Connection') and \ + self.has_header(b'Upgrade') + + def is_https_tunnel(self) -> bool: + """Returns true for HTTPS CONNECT tunnel request.""" + return self.method == httpMethods.CONNECT + + def is_chunked_encoded(self) -> bool: + """Returns true if transfer-encoding chunked is used.""" + return b'transfer-encoding' in self.headers and \ + self.headers[b'transfer-encoding'][1].lower() == b'chunked' + + def content_expected(self) -> bool: + """Returns true if content-length is present and not 0.""" + return b'content-length' in self.headers and int(self.header(b'content-length')) > 0 + + def body_expected(self) -> bool: + """Returns true if content or chunked response is expected.""" + return self.content_expected() or self.is_chunked_encoded() + + def parse(self, raw: bytes) -> None: + """Parses HTTP request out of raw bytes. + + Check for `HttpParser.state` after `parse` has successfully returned.""" + self.total_size += len(raw) + raw = self.buffer + raw + self.buffer, more = b'', len(raw) > 0 + while more and self.state != httpParserStates.COMPLETE: + # gte with HEADERS_COMPLETE also encapsulated RCVING_BODY state + more, raw = self._process_body(raw) \ + if self.state >= httpParserStates.HEADERS_COMPLETE else \ + self._process_line_and_headers(raw) + self.buffer = raw + + def build(self, disable_headers: Optional[List[bytes]] = None, for_proxy: bool = False) -> bytes: + """Rebuild the request object.""" + assert self.method and self.version and self.type == httpParserTypes.REQUEST_PARSER + if disable_headers is None: + disable_headers = DEFAULT_DISABLE_HEADERS + body: Optional[bytes] = self._get_body_or_chunks() + path = self.path or b'/' + if for_proxy: + assert self.host and self.port and self._url + path = ( + b'http' if not self._url.scheme else self._url.scheme + + COLON + SLASH + SLASH + + self.host + + COLON + + str(self.port).encode() + + path + ) if not self.is_https_tunnel() else (self.host + COLON + str(self.port).encode()) + return build_http_request( + self.method, path, self.version, + headers={} if not self.headers else { + self.headers[k][0]: self.headers[k][1] for k in self.headers if + k.lower() not in disable_headers + }, + body=body, + ) + + def build_response(self) -> bytes: + """Rebuild the response object.""" + assert self.code and self.version and self.type == httpParserTypes.RESPONSE_PARSER + return build_http_response( + status_code=int(self.code), + protocol_version=self.version, + reason=self.reason, + headers={} if not self.headers else { + self.headers[k][0]: self.headers[k][1] for k in self.headers + }, + body=self._get_body_or_chunks(), + ) + + def _process_body(self, raw: bytes) -> Tuple[bool, bytes]: + # Ref: http://www.ietf.org/rfc/rfc2616.txt + # 3.If a Content-Length header field (section 14.13) is present, its + # decimal value in OCTETs represents both the entity-length and the + # transfer-length. The Content-Length header field MUST NOT be sent + # if these two lengths are different (i.e., if a Transfer-Encoding + # header field is present). If a message is received with both a + # Transfer-Encoding header field and a Content-Length header field, + # the latter MUST be ignored. + # + # TL;DR -- Give transfer-encoding header preference over content-length. + if self.is_chunked_encoded(): + if not self.chunk: + self.chunk = ChunkParser() + raw = self.chunk.parse(raw) + if self.chunk.state == chunkParserStates.COMPLETE: + self.body = self.chunk.body + self.state = httpParserStates.COMPLETE + more = False + elif b'content-length' in self.headers: + self.state = httpParserStates.RCVING_BODY + if self.body is None: + self.body = b'' + total_size = int(self.header(b'content-length')) + received_size = len(self.body) + self.body += raw[:total_size - received_size] + if self.body and \ + len(self.body) == int(self.header(b'content-length')): + self.state = httpParserStates.COMPLETE + more, raw = len(raw) > 0, raw[total_size - received_size:] + else: + # HTTP/1.0 scenario only + assert self.version == HTTP_1_0 + self.state = httpParserStates.RCVING_BODY + # Received a packet without content-length header + # and no transfer-encoding specified. + # + # Ref https://github.com/abhinavsingh/proxy.py/issues/398 + # See TestHttpParser.test_issue_398 scenario + self.body = raw + more, raw = False, b'' + return more, raw + + def _process_line_and_headers(self, raw: bytes) -> Tuple[bool, bytes]: + """Returns False when no CRLF could be found in received bytes.""" + line, raw = find_http_line(raw) + if line is None: + return False, raw + + if self.state == httpParserStates.INITIALIZED: + self._process_line(line) + if self.state == httpParserStates.INITIALIZED: + return len(raw) > 0, raw + elif self.state in (httpParserStates.LINE_RCVD, httpParserStates.RCVING_HEADERS): + if self.state == httpParserStates.LINE_RCVD: + # LINE_RCVD state is equivalent to RCVING_HEADERS + self.state = httpParserStates.RCVING_HEADERS + if line.strip() == b'': # Blank line received. + self.state = httpParserStates.HEADERS_COMPLETE + else: + self._process_header(line) + + # When server sends a response line without any header or body e.g. + # HTTP/1.1 200 Connection established\r\n\r\n + if self.state == httpParserStates.LINE_RCVD and \ + self.type == httpParserTypes.RESPONSE_PARSER and \ + raw == CRLF: + self.state = httpParserStates.COMPLETE + elif self.state == httpParserStates.HEADERS_COMPLETE and \ + not self.body_expected() and \ + raw == b'': + self.state = httpParserStates.COMPLETE + + return len(raw) > 0, raw + + def _process_line(self, raw: bytes) -> None: + if self.type == httpParserTypes.REQUEST_PARSER: + if self.protocol is not None and self.protocol.version is None: + # We expect to receive entire proxy protocol v1 line + # in one network read and don't expect partial packets + self.protocol.parse(raw) + else: + # Ref: https://datatracker.ietf.org/doc/html/rfc2616#section-5.1 + line = raw.split(WHITESPACE) + if len(line) == 3: + self.method = line[0].upper() + self.set_url(line[1]) + self.version = line[2] + self.state = httpParserStates.LINE_RCVD + else: + # To avoid a possible attack vector, we raise exception + # if parser receives an invalid request line. + # + # TODO: Better to use raise HttpProtocolException, + # but we should solve circular import problem first. + raise ValueError('Invalid request line') + else: + line = raw.split(WHITESPACE) + self.version = line[0] + self.code = line[1] + self.reason = WHITESPACE.join(line[2:]) + self.state = httpParserStates.LINE_RCVD + + def _process_header(self, raw: bytes) -> None: + parts = raw.split(COLON) + key = parts[0].strip() + value = COLON.join(parts[1:]).strip() + self.add_headers([(key, value)]) + + def _get_body_or_chunks(self) -> Optional[bytes]: + return ChunkParser.to_chunks(self.body) \ + if self.body and self.is_chunked_encoded() else \ + self.body + + def _set_line_attributes(self) -> None: + if self.type == httpParserTypes.REQUEST_PARSER: + if self.is_https_tunnel() and self._url: + self.host = self._url.hostname + self.port = 443 if self._url.port is None else self._url.port + elif self._url: + self.host, self.port = self._url.hostname, self._url.port \ + if self._url.port else DEFAULT_HTTP_PORT + else: + raise KeyError( + 'Invalid request. Method: %r, Url: %r' % + (self.method, self._url), + ) + self.path = self._url.remainder diff --git a/proxy/http/parser/protocol.py b/proxy/http/parser/protocol.py new file mode 100644 index 0000000000..84982d1022 --- /dev/null +++ b/proxy/http/parser/protocol.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. + + .. spelling:: + + http +""" +from typing import Optional, Tuple + +from ...common.constants import WHITESPACE + +PROXY_PROTOCOL_V2_SIGNATURE = b'\x0D\x0A\x0D\x0A\x00\x0D\x0A\x51\x55\x49\x54\x0A' + + +class ProxyProtocol: + """Reference https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt""" + + def __init__(self) -> None: + self.version: Optional[int] = None + self.family: Optional[bytes] = None + self.source: Optional[Tuple[bytes, int]] = None + self.destination: Optional[Tuple[bytes, int]] = None + + def parse(self, raw: bytes) -> None: + if raw.startswith(b'PROXY'): + self.version = 1 + # Per spec, v1 line cannot exceed this limit + assert len(raw) <= 57 + line = raw.split(WHITESPACE) + assert line[0] == b'PROXY' and line[1] in ( + b'TCP4', b'TCP6', b'UNKNOWN', + ) + self.family = line[1] + if len(line) == 6: + self.source = (line[2], int(line[4])) + self.destination = (line[3], int(line[5])) + else: + assert self.family == b'UNKNOWN' + elif raw.startswith(PROXY_PROTOCOL_V2_SIGNATURE): + self.version = 2 + raise NotImplementedError() + else: + raise ValueError('Neither a v1 or v2 proxy protocol packet') diff --git a/proxy/http/parser/types.py b/proxy/http/parser/types.py new file mode 100644 index 0000000000..c5e019c6e3 --- /dev/null +++ b/proxy/http/parser/types.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + iterable +""" +from typing import NamedTuple + + +HttpParserStates = NamedTuple( + 'HttpParserStates', [ + ('INITIALIZED', int), + ('LINE_RCVD', int), + ('RCVING_HEADERS', int), + ('HEADERS_COMPLETE', int), + ('RCVING_BODY', int), + ('COMPLETE', int), + ], +) +httpParserStates = HttpParserStates(1, 2, 3, 4, 5, 6) + +HttpParserTypes = NamedTuple( + 'HttpParserTypes', [ + ('REQUEST_PARSER', int), + ('RESPONSE_PARSER', int), + ], +) +httpParserTypes = HttpParserTypes(1, 2) diff --git a/proxy/http/plugin.py b/proxy/http/plugin.py index 2e7fed454f..d5510b5c2b 100644 --- a/proxy/http/plugin.py +++ b/proxy/http/plugin.py @@ -7,12 +7,16 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http """ -import argparse import socket +import argparse -from abc import ABC, abstractmethod from uuid import UUID +from abc import ABC, abstractmethod from typing import Tuple, List, Union, Optional from .parser import HttpParser @@ -41,7 +45,7 @@ class HttpProtocolHandlerPlugin(ABC): 4. Server Response Chunk Received on_response_chunk is called for every chunk received from the server. 5. Client Connection Closed - Add your logic within `on_client_connection_close` for any per connection teardown. + Add your logic within `on_client_connection_close` for any per connection tear-down. """ def __init__( @@ -50,7 +54,8 @@ def __init__( flags: argparse.Namespace, client: TcpClientConnection, request: HttpParser, - event_queue: EventQueue): + event_queue: EventQueue, + ): self.uid: UUID = uid self.flags: argparse.Namespace = flags self.client: TcpClientConnection = client @@ -66,16 +71,25 @@ def name(self) -> str: return self.__class__.__name__ @abstractmethod - def get_descriptors( - self) -> Tuple[List[socket.socket], List[socket.socket]]: + def get_descriptors(self) -> Tuple[List[int], List[int]]: + """Implementations must return a list of descriptions that they wish to + read from and write into.""" return [], [] # pragma: no cover @abstractmethod - def write_to_descriptors(self, w: Writables) -> bool: + async def write_to_descriptors(self, w: Writables) -> bool: + """Implementations must now write/flush data over the socket. + + Note that buffer management is in-build into the connection classes. + Hence implementations MUST call + :meth:`~proxy.core.connection.TcpConnection.flush` here, to send + any buffered data over the socket. + """ return False # pragma: no cover @abstractmethod - def read_from_descriptors(self, r: Readables) -> bool: + async def read_from_descriptors(self, r: Readables) -> bool: + """Implementations must now read data over the socket.""" return False # pragma: no cover @abstractmethod @@ -96,4 +110,7 @@ def on_response_chunk(self, chunk: List[memoryview]) -> List[memoryview]: @abstractmethod def on_client_connection_close(self) -> None: + """Client connection shutdown has been received, flush has been called, + perform any cleanup work here. + """ pass # pragma: no cover diff --git a/proxy/http/proxy/__init__.py b/proxy/http/proxy/__init__.py index 4e18002c0d..69568bd96f 100644 --- a/proxy/http/proxy/__init__.py +++ b/proxy/http/proxy/__init__.py @@ -7,6 +7,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + Submodules """ from .plugin import HttpProxyBasePlugin from .server import HttpProxyPlugin diff --git a/proxy/http/proxy/auth.py b/proxy/http/proxy/auth.py index d1ac9a8611..5653a9528c 100644 --- a/proxy/http/proxy/auth.py +++ b/proxy/http/proxy/auth.py @@ -7,10 +7,16 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + auth + http """ from typing import Optional from ..exception import ProxyAuthenticationFailed + from ...common.flag import flags from ...common.constants import DEFAULT_BASIC_AUTH from ...http.parser import HttpParser @@ -22,14 +28,16 @@ type=str, default=DEFAULT_BASIC_AUTH, help='Default: No authentication. Specify colon separated user:password ' - 'to enable basic authentication.') + 'to enable basic authentication.', +) class AuthPlugin(HttpProxyBasePlugin): """Performs proxy authentication.""" def before_upstream_connection( - self, request: HttpParser) -> Optional[HttpParser]: + self, request: HttpParser, + ) -> Optional[HttpParser]: if self.flags.auth_code: if b'proxy-authorization' not in request.headers: raise ProxyAuthenticationFailed() @@ -39,13 +47,3 @@ def before_upstream_connection( or parts[1] != self.flags.auth_code: raise ProxyAuthenticationFailed() return request - - def handle_client_request( - self, request: HttpParser) -> Optional[HttpParser]: - return request - - def handle_upstream_chunk(self, chunk: memoryview) -> memoryview: - return chunk - - def on_upstream_connection_close(self) -> None: - pass diff --git a/proxy/http/proxy/plugin.py b/proxy/http/proxy/plugin.py index 7e5c1b0ba0..81392d1d63 100644 --- a/proxy/http/proxy/plugin.py +++ b/proxy/http/proxy/plugin.py @@ -7,14 +7,20 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http """ -from abc import ABC, abstractmethod import argparse -from typing import Optional + +from abc import ABC from uuid import UUID -from ..parser import HttpParser +from typing import Any, Dict, List, Optional, Tuple +from ..parser import HttpParser +from ...common.types import Readables, Writables from ...core.event import EventQueue from ...core.connection import TcpClientConnection @@ -29,7 +35,8 @@ def __init__( uid: UUID, flags: argparse.Namespace, client: TcpClientConnection, - event_queue: EventQueue) -> None: + event_queue: EventQueue, + ) -> None: self.uid = uid # pragma: no cover self.flags = flags # pragma: no cover self.client = client # pragma: no cover @@ -42,9 +49,58 @@ def name(self) -> str: access a specific plugin by its name.""" return self.__class__.__name__ # pragma: no cover - @abstractmethod + # TODO(abhinavsingh): get_descriptors, write_to_descriptors, read_from_descriptors + # can be placed into their own abstract class which can then be shared by + # HttpProxyBasePlugin, HttpWebServerBasePlugin and HttpProtocolHandlerPlugin class. + # + # Currently code has been shamelessly copied. Also these methods are not + # marked as abstract to avoid breaking custom plugins written by users for + # previous versions of proxy.py + # + # Since 3.4.0 + # + # @abstractmethod + def get_descriptors(self) -> Tuple[List[int], List[int]]: + return [], [] # pragma: no cover + + # @abstractmethod + def write_to_descriptors(self, w: Writables) -> bool: + """Implementations must now write/flush data over the socket. + + Note that buffer management is in-build into the connection classes. + Hence implementations MUST call + :meth:`~proxy.core.connection.connection.TcpConnection.flush` + here, to send any buffered data over the socket. + """ + return False # pragma: no cover + + # @abstractmethod + def read_from_descriptors(self, r: Readables) -> bool: + """Implementations must now read data over the socket.""" + return False # pragma: no cover + + def resolve_dns(self, host: str, port: int) -> Tuple[Optional[str], Optional[Tuple[str, int]]]: + """Resolve upstream server host to an IP address. + + Optionally also override the source address to use for + connection with upstream server. + + For upstream IP: + Return None to use default resolver available to the system. + Return IP address as string to use your custom resolver. + + For source address: + Return None to use default source address + Return 2-tuple representing (host, port) to use as source address + """ + return None, None + + # No longer abstract since 2.4.0 + # + # @abstractmethod def before_upstream_connection( - self, request: HttpParser) -> Optional[HttpParser]: + self, request: HttpParser, + ) -> Optional[HttpParser]: """Handler called just before Proxy upstream connection is established. Return optionally modified request object. @@ -53,9 +109,29 @@ def before_upstream_connection( Raise HttpRequestRejected or HttpProtocolException directly to drop the connection.""" return request # pragma: no cover - @abstractmethod + # Since 3.4.0 + # + # @abstractmethod + def handle_client_data( + self, raw: memoryview, + ) -> Optional[memoryview]: + """Handler called in special scenarios when an upstream server connection + is never established. + + Essentially, if you return None from within before_upstream_connection, + be prepared to handle_client_data and not handle_client_request. + + Raise HttpRequestRejected to tear down the connection + Return None to drop the connection + """ + return raw # pragma: no cover + + # No longer abstract since 2.4.0 + # + # @abstractmethod def handle_client_request( - self, request: HttpParser) -> Optional[HttpParser]: + self, request: HttpParser, + ) -> Optional[HttpParser]: """Handler called before dispatching client request to upstream. Note: For pipelined (keep-alive) connections, this handler can be @@ -68,11 +144,14 @@ def handle_client_request( Return optionally modified request object to dispatch to upstream. Return None to drop the request data, e.g. in case a response has already been queued. Raise HttpRequestRejected or HttpProtocolException directly to - teardown the connection with client. + tear down the connection with client. + """ return request # pragma: no cover - @abstractmethod + # No longer abstract since 2.4.0 + # + # @abstractmethod def handle_upstream_chunk(self, chunk: memoryview) -> memoryview: """Handler called right after receiving raw response from upstream server. @@ -80,7 +159,23 @@ def handle_upstream_chunk(self, chunk: memoryview) -> memoryview: TLS interception is also enabled.""" return chunk # pragma: no cover - @abstractmethod + # No longer abstract since 2.4.0 + # + # @abstractmethod def on_upstream_connection_close(self) -> None: """Handler called right after upstream connection has been closed.""" pass # pragma: no cover + + def on_access_log(self, context: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """Use this method to override default access log format (see + DEFAULT_HTTP_ACCESS_LOG_FORMAT and DEFAULT_HTTPS_ACCESS_LOG_FORMAT) and to + add/update/modify/delete context for next plugin.on_access_log invocation. + + This is specially useful if a plugins want to provide extra context + in the access log which may not available within other plugins' context or even + in proxy.py core. + + Returns Log context or None. If plugin chooses to access log, they ideally + must return None to prevent other plugin.on_access_log invocation. + """ + return context diff --git a/proxy/http/proxy/server.py b/proxy/http/proxy/server.py index 01c5e06442..c9f01d1639 100644 --- a/proxy/http/proxy/server.py +++ b/proxy/http/proxy/server.py @@ -7,114 +7,155 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + reusability """ -import logging -import threading -import subprocess import os import ssl -import socket import time import errno +import socket +import logging +import threading +import subprocess + from typing import Optional, List, Union, Dict, cast, Any, Tuple from .plugin import HttpProxyBasePlugin + +from ..methods import httpMethods +from ..codes import httpStatusCodes from ..plugin import HttpProtocolHandlerPlugin from ..exception import HttpProtocolException, ProxyConnectionFailed -from ..codes import httpStatusCodes from ..parser import HttpParser, httpParserStates, httpParserTypes -from ..methods import httpMethods from ...common.types import Readables, Writables from ...common.constants import DEFAULT_CA_CERT_DIR, DEFAULT_CA_CERT_FILE, DEFAULT_CA_FILE from ...common.constants import DEFAULT_CA_KEY_FILE, DEFAULT_CA_SIGNING_KEY_FILE from ...common.constants import COMMA, DEFAULT_SERVER_RECVBUF_SIZE, DEFAULT_CERT_FILE from ...common.constants import PROXY_AGENT_HEADER_VALUE, DEFAULT_DISABLE_HEADERS +from ...common.constants import DEFAULT_HTTP_ACCESS_LOG_FORMAT, DEFAULT_HTTPS_ACCESS_LOG_FORMAT +from ...common.constants import DEFAULT_DISABLE_HTTP_PROXY, PLUGIN_PROXY_AUTH from ...common.utils import build_http_response, text_ from ...common.pki import gen_public_key, gen_csr, sign_csr from ...core.event import eventNames -from ...core.connection import TcpServerConnection, TcpConnectionUninitializedException +from ...core.connection import TcpServerConnection, ConnectionPool +from ...core.connection import TcpConnectionUninitializedException from ...common.flag import flags logger = logging.getLogger(__name__) +flags.add_argument( + '--server-recvbuf-size', + type=int, + default=DEFAULT_SERVER_RECVBUF_SIZE, + help='Default: 1 MB. Maximum amount of data received from the ' + 'server in a single recv() operation. Bump this ' + 'value for faster downloads at the expense of ' + 'increased RAM.', +) + +flags.add_argument( + '--disable-http-proxy', + action='store_true', + default=DEFAULT_DISABLE_HTTP_PROXY, + help='Default: False. Whether to disable proxy.HttpProxyPlugin.', +) + +flags.add_argument( + '--disable-headers', + type=str, + default=COMMA.join(DEFAULT_DISABLE_HEADERS), + help='Default: None. Comma separated list of headers to remove before ' + 'dispatching client request to upstream server.', +) + flags.add_argument( '--ca-key-file', type=str, default=DEFAULT_CA_KEY_FILE, help='Default: None. CA key to use for signing dynamically generated ' - 'HTTPS certificates. If used, must also pass --ca-cert-file and --ca-signing-key-file' + 'HTTPS certificates. If used, must also pass --ca-cert-file and --ca-signing-key-file', ) + flags.add_argument( '--ca-cert-dir', type=str, default=DEFAULT_CA_CERT_DIR, help='Default: ~/.proxy.py. Directory to store dynamically generated certificates. ' - 'Also see --ca-key-file, --ca-cert-file and --ca-signing-key-file' + 'Also see --ca-key-file, --ca-cert-file and --ca-signing-key-file', ) + flags.add_argument( '--ca-cert-file', type=str, default=DEFAULT_CA_CERT_FILE, help='Default: None. Signing certificate to use for signing dynamically generated ' - 'HTTPS certificates. If used, must also pass --ca-key-file and --ca-signing-key-file' + 'HTTPS certificates. If used, must also pass --ca-key-file and --ca-signing-key-file', ) + flags.add_argument( '--ca-file', type=str, - default=DEFAULT_CA_FILE, - help='Default: None. Provide path to custom CA file for peer certificate validation. ' - 'Specially useful on MacOS.' + default=str(DEFAULT_CA_FILE), + help='Default: ' + str(DEFAULT_CA_FILE) + + '. Provide path to custom CA bundle for peer certificate verification', ) + flags.add_argument( '--ca-signing-key-file', type=str, default=DEFAULT_CA_SIGNING_KEY_FILE, help='Default: None. CA signing key to use for dynamic generation of ' - 'HTTPS certificates. If used, must also pass --ca-key-file and --ca-cert-file' + 'HTTPS certificates. If used, must also pass --ca-key-file and --ca-cert-file', ) + flags.add_argument( '--cert-file', type=str, default=DEFAULT_CERT_FILE, help='Default: None. Server certificate to enable end-to-end TLS encryption with clients. ' - 'If used, must also pass --key-file.' + 'If used, must also pass --key-file.', ) + flags.add_argument( - '--disable-headers', + '--auth-plugin', type=str, - default=COMMA.join(DEFAULT_DISABLE_HEADERS), - help='Default: None. Comma separated list of headers to remove before ' - 'dispatching client request to upstream server.') -flags.add_argument( - '--server-recvbuf-size', - type=int, - default=DEFAULT_SERVER_RECVBUF_SIZE, - help='Default: 1 MB. Maximum amount of data received from the ' - 'server in a single recv() operation. Bump this ' - 'value for faster downloads at the expense of ' - 'increased RAM.') + default=PLUGIN_PROXY_AUTH, + help='Default: ' + PLUGIN_PROXY_AUTH + '. ' + + 'Auth plugin to use instead of default basic auth plugin.', +) class HttpProxyPlugin(HttpProtocolHandlerPlugin): """HttpProtocolHandler plugin which implements HttpProxy specifications.""" - PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT = memoryview(build_http_response( - httpStatusCodes.OK, - reason=b'Connection established' - )) + PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT = memoryview( + build_http_response( + httpStatusCodes.OK, + reason=b'Connection established', + ), + ) - # Used to synchronization during certificate generation. + # Used to synchronization during certificate generation and + # connection pool operations. lock = threading.Lock() + # Shared connection pool + pool = ConnectionPool() + def __init__( self, - *args: Any, **kwargs: Any) -> None: + *args: Any, **kwargs: Any, + ) -> None: super().__init__(*args, **kwargs) self.start_time: float = time.time() - self.server: Optional[TcpServerConnection] = None + self.upstream: Optional[TcpServerConnection] = None self.response: HttpParser = HttpParser(httpParserTypes.RESPONSE_PARSER) self.pipeline_request: Optional[HttpParser] = None self.pipeline_response: Optional[HttpParser] = None @@ -122,11 +163,12 @@ def __init__( self.plugins: Dict[str, HttpProxyBasePlugin] = {} if b'HttpProxyBasePlugin' in self.flags.plugins: for klass in self.flags.plugins[b'HttpProxyBasePlugin']: - instance = klass( + instance: HttpProxyBasePlugin = klass( self.uid, self.flags, self.client, - self.event_queue) + self.event_queue, + ) self.plugins[instance.name()] = instance def tls_interception_enabled(self) -> bool: @@ -135,58 +177,104 @@ def tls_interception_enabled(self) -> bool: self.flags.ca_signing_key_file is not None and \ self.flags.ca_cert_file is not None - def get_descriptors( - self) -> Tuple[List[socket.socket], List[socket.socket]]: - if not self.request.has_upstream_server(): + def get_descriptors(self) -> Tuple[List[int], List[int]]: + if not self.request.has_host(): return [], [] - - r: List[socket.socket] = [] - w: List[socket.socket] = [] - if self.server and not self.server.closed and self.server.connection: - r.append(self.server.connection) - if self.server and not self.server.closed and \ - self.server.has_buffer() and self.server.connection: - w.append(self.server.connection) + r: List[int] = [] + w: List[int] = [] + if ( + self.upstream and + not self.upstream.closed and + self.upstream.connection + ): + r.append(self.upstream.connection.fileno()) + if ( + self.upstream and + not self.upstream.closed and + self.upstream.has_buffer() and + self.upstream.connection + ): + w.append(self.upstream.connection.fileno()) + # TODO(abhinavsingh): We need to keep a mapping of plugin and + # descriptors registered by them, so that within write/read blocks + # we can invoke the right plugin callbacks. + for plugin in self.plugins.values(): + plugin_read_desc, plugin_write_desc = plugin.get_descriptors() + r.extend(plugin_read_desc) + w.extend(plugin_write_desc) return r, w - def write_to_descriptors(self, w: Writables) -> bool: - if self.request.has_upstream_server() and \ - self.server and not self.server.closed and \ - self.server.has_buffer() and \ - self.server.connection in w: + def _close_and_release(self) -> bool: + if self.flags.enable_conn_pool: + assert self.upstream and not self.upstream.closed + self.upstream.closed = True + with self.lock: + self.pool.release(self.upstream) + self.upstream = None + return True + + async def write_to_descriptors(self, w: Writables) -> bool: + if (self.upstream and self.upstream.connection.fileno() not in w) or not self.upstream: + # Currently, we just call write/read block of each plugins. It is + # plugins responsibility to ignore this callback, if passed descriptors + # doesn't contain the descriptor they registered. + for plugin in self.plugins.values(): + teardown = plugin.write_to_descriptors(w) + if teardown: + return True + elif self.request.has_host() and \ + self.upstream and not self.upstream.closed and \ + self.upstream.has_buffer() and \ + self.upstream.connection.fileno() in w: logger.debug('Server is write ready, flushing buffer') try: - self.server.flush() + self.upstream.flush() except ssl.SSLWantWriteError: logger.warning( - 'SSLWantWriteError while trying to flush to server, will retry') + 'SSLWantWriteError while trying to flush to server, will retry', + ) return False except BrokenPipeError: logger.error( - 'BrokenPipeError when flushing buffer for server') - return True + 'BrokenPipeError when flushing buffer for server', + ) + return self._close_and_release() except OSError as e: logger.exception( - 'OSError when flushing buffer to server', exc_info=e) - return True + 'OSError when flushing buffer to server', exc_info=e, + ) + return self._close_and_release() return False - def read_from_descriptors(self, r: Readables) -> bool: - if self.request.has_upstream_server() \ - and self.server \ - and not self.server.closed \ - and self.server.connection in r: + async def read_from_descriptors(self, r: Readables) -> bool: + if ( + self.upstream and not + self.upstream.closed and + self.upstream.connection.fileno() not in r + ) or not self.upstream: + # Currently, we just call write/read block of each plugins. It is + # plugins responsibility to ignore this callback, if passed descriptors + # doesn't contain the descriptor they registered for. + for plugin in self.plugins.values(): + teardown = plugin.read_from_descriptors(r) + if teardown: + return True + elif self.request.has_host() \ + and self.upstream \ + and not self.upstream.closed \ + and self.upstream.connection.fileno() in r: logger.debug('Server is ready for reads, reading...') try: - raw = self.server.recv(self.flags.server_recvbuf_size) + raw = self.upstream.recv(self.flags.server_recvbuf_size) except TimeoutError as e: + self._close_and_release() if e.errno == errno.ETIMEDOUT: logger.warning( '%s:%d timed out on recv' % - self.server.addr) + self.upstream.addr, + ) return True - else: - raise e + raise e except ssl.SSLWantReadError: # Try again later # logger.warning('SSLWantReadError encountered while reading from server, will retry ...') return False @@ -194,19 +282,24 @@ def read_from_descriptors(self, r: Readables) -> bool: if e.errno == errno.EHOSTUNREACH: logger.warning( '%s:%d unreachable on recv' % - self.server.addr) - return True - elif e.errno == errno.ECONNRESET: - logger.warning('Connection reset by upstream: %r' % e) + self.upstream.addr, + ) + if e.errno == errno.ECONNRESET: + logger.warning( + 'Connection reset by upstream: {0}:{1}'.format( + *self.upstream.addr, + ), + ) else: - logger.exception( - 'Exception while receiving from %s connection %r with reason %r' % - (self.server.tag, self.server.connection, e)) - return True + logger.warning( + 'Exception while receiving from %s connection#%d with reason %r' % + (self.upstream.tag, self.upstream.connection.fileno(), e), + ) + return self._close_and_release() if raw is None: logger.debug('Server closed connection, tearing down...') - return True + return self._close_and_release() for plugin in self.plugins.values(): raw = plugin.handle_upstream_chunk(raw) @@ -214,7 +307,7 @@ def read_from_descriptors(self, r: Readables) -> bool: # parse incoming response packet # only for non-https requests and when # tls interception is enabled - if self.request.method != httpMethods.CONNECT: + if not self.request.is_https_tunnel(): # See https://github.com/abhinavsingh/proxy.py/issues/127 for why # currently response parsing is disabled when TLS interception is enabled. # @@ -224,8 +317,9 @@ def read_from_descriptors(self, r: Readables) -> bool: else: # TODO(abhinavsingh): Remove .tobytes after parser is # memoryview compliant - self.response.parse(raw.tobytes()) - self.emit_response_events() + chunk = raw.tobytes() + self.response.parse(chunk) + self.emit_response_events(len(chunk)) else: self.response.total_size += len(raw) # queue raw data for client @@ -233,35 +327,107 @@ def read_from_descriptors(self, r: Readables) -> bool: return False def on_client_connection_close(self) -> None: - if not self.request.has_upstream_server(): + if not self.request.has_host(): return - self.access_log() - - # If server was never initialized, return - if self.server is None: - return + context = { + 'client_ip': None if not self.client.addr else self.client.addr[0], + 'client_port': None if not self.client.addr else self.client.addr[1], + 'server_host': text_(self.upstream.addr[0] if self.upstream else None), + 'server_port': text_(self.upstream.addr[1] if self.upstream else None), + 'connection_time_ms': '%.2f' % ((time.time() - self.start_time) * 1000), + # Request + 'request_method': text_(self.request.method), + 'request_path': text_(self.request.path), + 'request_bytes': self.request.total_size, + 'request_code': self.request.code, + 'request_ua': self.request.header(b'user-agent') + if self.request.has_header(b'user-agent') + else None, + 'request_reason': self.request.reason, + 'request_version': self.request.version, + # Response + 'response_bytes': self.response.total_size, + 'response_code': text_(self.response.code), + 'response_reason': text_(self.response.reason), + } + if self.flags.enable_proxy_protocol: + assert self.request.protocol and self.request.protocol.family + context.update({ + 'protocol': { + 'family': text_(self.request.protocol.family), + }, + }) + if self.request.protocol.source: + context.update({ + 'protocol': { + 'source_ip': text_(self.request.protocol.source[0]), + 'source_port': self.request.protocol.source[1], + }, + }) + if self.request.protocol.destination: + context.update({ + 'protocol': { + 'destination_ip': text_(self.request.protocol.destination[0]), + 'destination_port': self.request.protocol.destination[1], + }, + }) + + log_handled = False + for plugin in self.plugins.values(): + ctx = plugin.on_access_log(context) + if ctx is None: + log_handled = True + break + context = ctx + if not log_handled: + self.access_log(context) # Note that, server instance was initialized # but not necessarily the connection object exists. + # + # Unfortunately this is still being called when an upstream + # server connection was never established. This is done currently + # to assist proxy pool plugin to close its upstream proxy connections. + # + # In short, treat on_upstream_connection_close as on_client_connection_close + # equivalent within proxy plugins. + # # Invoke plugin.on_upstream_connection_close for plugin in self.plugins.values(): plugin.on_upstream_connection_close() + # If server was never initialized or was _close_and_release + if self.upstream is None: + return + + if self.flags.enable_conn_pool: + # Release the connection for reusability + with self.lock: + self.pool.release(self.upstream) + return + try: try: - self.server.connection.shutdown(socket.SHUT_WR) + self.upstream.connection.shutdown(socket.SHUT_WR) except OSError: pass finally: # TODO: Unwrap if wrapped before close? - self.server.connection.close() + self.upstream.connection.close() except TcpConnectionUninitializedException: pass finally: logger.debug( 'Closed server connection, has buffer %s' % - self.server.has_buffer()) + self.upstream.has_buffer(), + ) + + def access_log(self, log_attrs: Dict[str, Any]) -> None: + access_log_format = DEFAULT_HTTPS_ACCESS_LOG_FORMAT + if not self.request.is_https_tunnel(): + access_log_format = DEFAULT_HTTP_ACCESS_LOG_FORMAT + logger.info(access_log_format.format_map(log_attrs)) def on_response_chunk(self, chunk: List[memoryview]) -> List[memoryview]: # TODO: Allow to output multiple access_log lines @@ -269,30 +435,55 @@ def on_response_chunk(self, chunk: List[memoryview]) -> List[memoryview]: # However, this must also be accompanied by resetting both request # and response objects. # - # if not self.request.method == httpMethods.CONNECT and \ + # if not self.request.is_https_tunnel() and \ # self.response.state == httpParserStates.COMPLETE: # self.access_log() return chunk + # Can return None to tear down connection def on_client_data(self, raw: memoryview) -> Optional[memoryview]: - if not self.request.has_upstream_server(): + if not self.request.has_host(): return raw - if self.server and not self.server.closed: + # For scenarios when an upstream connection was never established, + # let plugin do whatever they wish to. These are special scenarios + # where plugins are trying to do something magical. Within the core + # we don't know the context. In fact, we are not even sure if data + # exchanged is http spec compliant. + # + # Hence, here we pass raw data to HTTP proxy plugins as is. + # + # We only call handle_client_data once original request has been + # completely received + if not self.upstream: + for plugin in self.plugins.values(): + o = plugin.handle_client_data(raw) + if o is None: + return None + raw = o + elif self.upstream and not self.upstream.closed: + # For http proxy requests, handle pipeline case. + # We also handle pipeline scenario for https proxy + # requests is TLS interception is enabled. if self.request.state == httpParserStates.COMPLETE and ( - self.request.method != httpMethods.CONNECT or - self.tls_interception_enabled()): + not self.request.is_https_tunnel() or + self.tls_interception_enabled() + ): if self.pipeline_request is not None and \ self.pipeline_request.is_connection_upgrade(): # Previous pipelined request was a WebSocket # upgrade request. Incoming client data now # must be treated as WebSocket protocol packets. - self.server.queue(raw) + self.upstream.queue(raw) return None if self.pipeline_request is None: + # For pipeline requests, we never + # want to use --enable-proxy-protocol flag + # as proxy protocol header will not be present self.pipeline_request = HttpParser( - httpParserTypes.REQUEST_PARSER) + httpParserTypes.REQUEST_PARSER, + ) # TODO(abhinavsingh): Remove .tobytes after parser is # memoryview compliant @@ -307,25 +498,33 @@ def on_client_data(self, raw: memoryview) -> Optional[memoryview]: assert self.pipeline_request is not None # TODO(abhinavsingh): Remove memoryview wrapping here after # parser is fully memoryview compliant - self.server.queue( + self.upstream.queue( memoryview( - self.pipeline_request.build())) + self.pipeline_request.build(), + ), + ) if not self.pipeline_request.is_connection_upgrade(): self.pipeline_request = None + # For scenarios where we cannot peek into the data, + # simply queue for upstream server. else: - self.server.queue(raw) + self.upstream.queue(raw) return None - else: - return raw + return raw def on_request_complete(self) -> Union[socket.socket, bool]: - if not self.request.has_upstream_server(): + if not self.request.has_host(): return False self.emit_request_complete() - # Note: can raise HttpRequestRejected exception # Invoke plugin.before_upstream_connection + # + # before_upstream_connection can: + # 1) Raise HttpRequestRejected exception to reject the connection + # 2) return None to continue without establishing an upstream server connection + # e.g. for scenarios when plugins want to return response from cache, or, + # via out-of-band over the network request. do_connect = True for plugin in self.plugins.values(): r = plugin.before_upstream_connection(self.request) @@ -334,9 +533,11 @@ def on_request_complete(self) -> Union[socket.socket, bool]: break self.request = r + # Connect to upstream if do_connect: self.connect_upstream() + # Invoke plugin.handle_client_request for plugin in self.plugins.values(): assert self.request is not None r = plugin.handle_client_request(self.request) @@ -345,84 +546,117 @@ def on_request_complete(self) -> Union[socket.socket, bool]: else: return False - if self.request.method == httpMethods.CONNECT: - self.client.queue( - HttpProxyPlugin.PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT) - if self.tls_interception_enabled(): - return self.intercept() - elif self.server: - # - proxy-connection header is a mistake, it doesn't seem to be - # officially documented in any specification, drop it. - # - proxy-authorization is of no use for upstream, remove it. - self.request.del_headers( - [b'proxy-authorization', b'proxy-connection']) - # - For HTTP/1.0, connection header defaults to close - # - For HTTP/1.1, connection header defaults to keep-alive - # Respect headers sent by client instead of manipulating - # Connection or Keep-Alive header. However, note that per - # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Connection - # connection headers are meant for communication between client and - # first intercepting proxy. - self.request.add_headers( - [(b'Via', b'1.1 %s' % PROXY_AGENT_HEADER_VALUE)]) - # Disable args.disable_headers before dispatching to upstream - self.server.queue( - memoryview(self.request.build( - disable_headers=self.flags.disable_headers))) + # For https requests, respond back with tunnel established response. + # Optionally, setup interceptor if TLS interception is enabled. + if self.upstream: + if self.request.is_https_tunnel(): + self.client.queue( + HttpProxyPlugin.PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT, + ) + if self.tls_interception_enabled(): + return self.intercept() + # If an upstream server connection was established for http request, + # queue the request for upstream server. + else: + # - proxy-connection header is a mistake, it doesn't seem to be + # officially documented in any specification, drop it. + # - proxy-authorization is of no use for upstream, remove it. + self.request.del_headers( + [b'proxy-authorization', b'proxy-connection'], + ) + # - For HTTP/1.0, connection header defaults to close + # - For HTTP/1.1, connection header defaults to keep-alive + # Respect headers sent by client instead of manipulating + # Connection or Keep-Alive header. However, note that per + # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Connection + # connection headers are meant for communication between client and + # first intercepting proxy. + self.request.add_headers( + [(b'Via', b'1.1 %s' % PROXY_AGENT_HEADER_VALUE)], + ) + # Disable args.disable_headers before dispatching to upstream + self.upstream.queue( + memoryview( + self.request.build( + disable_headers=self.flags.disable_headers, + ), + ), + ) return False def handle_pipeline_response(self, raw: memoryview) -> None: if self.pipeline_response is None: self.pipeline_response = HttpParser( - httpParserTypes.RESPONSE_PARSER) + httpParserTypes.RESPONSE_PARSER, + ) # TODO(abhinavsingh): Remove .tobytes after parser is memoryview # compliant self.pipeline_response.parse(raw.tobytes()) if self.pipeline_response.state == httpParserStates.COMPLETE: self.pipeline_response = None - def access_log(self) -> None: - server_host, server_port = self.server.addr if self.server else ( - None, None) - connection_time_ms = (time.time() - self.start_time) * 1000 - if self.request.method == httpMethods.CONNECT: - logger.info( - '%s:%s - %s %s:%s - %s bytes - %.2f ms' % - (self.client.addr[0], - self.client.addr[1], - text_(self.request.method), - text_(server_host), - text_(server_port), - self.response.total_size, - connection_time_ms)) - elif self.request.method: - logger.info( - '%s:%s - %s %s:%s%s - %s %s - %s bytes - %.2f ms' % - (self.client.addr[0], self.client.addr[1], - text_(self.request.method), - text_(server_host), server_port, - text_(self.request.path), - text_(self.response.code), - text_(self.response.reason), - self.response.total_size, - connection_time_ms)) - def connect_upstream(self) -> None: host, port = self.request.host, self.request.port if host and port: - self.server = TcpServerConnection(text_(host), port) + if self.flags.enable_conn_pool: + with self.lock: + created, self.upstream = self.pool.acquire( + text_(host), port, + ) + else: + created, self.upstream = True, TcpServerConnection( + text_(host), port, + ) + if not created: + # NOTE: Acquired connection might be in an unusable state. + # + # This can only be confirmed by reading from connection. + # For stale connections, we will receive None, indicating + # to drop the connection. + # + # If that happen, we must acquire a fresh connection. + logger.info( + 'Reusing connection to upstream %s:%d' % + (text_(host), port), + ) + return try: logger.debug( - 'Connecting to upstream %s:%s' % - (text_(host), port)) - self.server.connect() - self.server.connection.setblocking(False) + 'Connecting to upstream %s:%d' % + (text_(host), port), + ) + # Invoke plugin.resolve_dns + upstream_ip, source_addr = None, None + for plugin in self.plugins.values(): + upstream_ip, source_addr = plugin.resolve_dns( + text_(host), port, + ) + if upstream_ip or source_addr: + break + # Connect with overridden upstream IP and source address + # if any of the plugin returned a non-null value. + self.upstream.connect( + addr=None if not upstream_ip else ( + upstream_ip, port, + ), source_address=source_addr, + ) + self.upstream.connection.setblocking(False) logger.debug( 'Connected to upstream %s:%s' % - (text_(host), port)) + (text_(host), port), + ) except Exception as e: # TimeoutError, socket.gaierror - self.server.closed = True - raise ProxyConnectionFailed(text_(host), port, repr(e)) from e + logger.warning( + 'Unable to connect with upstream %s:%d due to %s' % ( + text_(host), port, str(e), + ), + ) + if self.flags.enable_conn_pool: + with self.lock: + self.pool.release(self.upstream) + raise ProxyConnectionFailed( + text_(host), port, repr(e), + ) from e else: logger.exception('Both host and port must exist') raise HttpProtocolException() @@ -432,7 +666,8 @@ def connect_upstream(self) -> None: # def gen_ca_signed_certificate( - self, cert_file_path: str, certificate: Dict[str, Any]) -> None: + self, cert_file_path: str, certificate: Dict[str, Any], + ) -> None: '''CA signing key (default) is used for generating a public key for common_name, if one already doesn't exist. Using generated public key a CSR request is generated, which is then signed by @@ -440,12 +675,16 @@ def gen_ca_signed_certificate( certificate doesn't already exist. returns signed certificate path.''' - assert(self.request.host and self.flags.ca_cert_dir and self.flags.ca_signing_key_file and - self.flags.ca_key_file and self.flags.ca_cert_file) + assert( + self.request.host and self.flags.ca_cert_dir and self.flags.ca_signing_key_file and + self.flags.ca_key_file and self.flags.ca_cert_file + ) upstream_subject = {s[0][0]: s[0][1] for s in certificate['subject']} - public_key_path = os.path.join(self.flags.ca_cert_dir, - '{0}.{1}'.format(text_(self.request.host), 'pub')) + public_key_path = os.path.join( + self.flags.ca_cert_dir, + '{0}.{1}'.format(text_(self.request.host), 'pub'), + ) private_key_path = self.flags.ca_signing_key_file private_key_password = '' @@ -461,28 +700,36 @@ def gen_ca_signed_certificate( subject = '' for key in keys: if upstream_subject.get(keys[key], None): - subject += '/{0}={1}'.format(key, - upstream_subject.get(keys[key])) - alt_subj_names = [text_(self.request.host), ] + subject += '/{0}={1}'.format( + key, + upstream_subject.get(keys[key]), + ) + alt_subj_names = [text_(self.request.host)] validity_in_days = 365 * 2 timeout = 10 # Generate a public key for the common name if not os.path.isfile(public_key_path): logger.debug('Generating public key %s', public_key_path) - resp = gen_public_key(public_key_path=public_key_path, private_key_path=private_key_path, - private_key_password=private_key_password, subject=subject, alt_subj_names=alt_subj_names, - validity_in_days=validity_in_days, timeout=timeout) + resp = gen_public_key( + public_key_path=public_key_path, private_key_path=private_key_path, + private_key_password=private_key_password, subject=subject, alt_subj_names=alt_subj_names, + validity_in_days=validity_in_days, timeout=timeout, + ) assert(resp is True) - csr_path = os.path.join(self.flags.ca_cert_dir, - '{0}.{1}'.format(text_(self.request.host), 'csr')) + csr_path = os.path.join( + self.flags.ca_cert_dir, + '{0}.{1}'.format(text_(self.request.host), 'csr'), + ) # Generate a CSR request for this common name if not os.path.isfile(csr_path): logger.debug('Generating CSR %s', csr_path) - resp = gen_csr(csr_path=csr_path, key_path=private_key_path, password=private_key_password, - crt_path=public_key_path, timeout=timeout) + resp = gen_csr( + csr_path=csr_path, key_path=private_key_path, password=private_key_password, + crt_path=public_key_path, timeout=timeout, + ) assert(resp is True) ca_key_path = self.flags.ca_key_file @@ -493,10 +740,12 @@ def gen_ca_signed_certificate( # Sign generated CSR if not os.path.isfile(cert_file_path): logger.debug('Signing CSR %s', cert_file_path) - resp = sign_csr(csr_path=csr_path, crt_path=cert_file_path, ca_key_path=ca_key_path, - ca_key_password=ca_key_password, ca_crt_path=ca_crt_path, - serial=str(serial), alt_subj_names=alt_subj_names, - validity_in_days=validity_in_days, timeout=timeout) + resp = sign_csr( + csr_path=csr_path, crt_path=cert_file_path, ca_key_path=ca_key_path, + ca_key_password=ca_key_password, ca_crt_path=ca_crt_path, + serial=str(serial), alt_subj_names=alt_subj_names, + validity_in_days=validity_in_days, timeout=timeout, + ) assert(resp is True) @staticmethod @@ -504,16 +753,21 @@ def generated_cert_file_path(ca_cert_dir: str, host: str) -> str: return os.path.join(ca_cert_dir, '%s.pem' % host) def generate_upstream_certificate( - self, certificate: Dict[str, Any]) -> str: - if not (self.flags.ca_cert_dir and self.flags.ca_signing_key_file and - self.flags.ca_cert_file and self.flags.ca_key_file): + self, certificate: Dict[str, Any], + ) -> str: + if not ( + self.flags.ca_cert_dir and self.flags.ca_signing_key_file and + self.flags.ca_cert_file and self.flags.ca_key_file + ): raise HttpProtocolException( f'For certificate generation all the following flags are mandatory: ' f'--ca-cert-file:{ self.flags.ca_cert_file }, ' f'--ca-key-file:{ self.flags.ca_key_file }, ' - f'--ca-signing-key-file:{ self.flags.ca_signing_key_file }') + f'--ca-signing-key-file:{ self.flags.ca_signing_key_file }', + ) cert_file_path = HttpProxyPlugin.generated_cert_file_path( - self.flags.ca_cert_dir, text_(self.request.host)) + self.flags.ca_cert_dir, text_(self.request.host), + ) with self.lock: if not os.path.isfile(cert_file_path): self.gen_ca_signed_certificate(cert_file_path, certificate) @@ -523,42 +777,124 @@ def intercept(self) -> Union[socket.socket, bool]: # Perform SSL/TLS handshake with upstream self.wrap_server() # Generate certificate and perform handshake with client + # wrap_client also flushes client data before wrapping + # sending to client can raise, handle expected exceptions + self.wrap_client() + # Update all plugin connection reference + # TODO(abhinavsingh): Is this required? + for plugin in self.plugins.values(): + plugin.client._conn = self.client.connection + return self.client.connection + + def wrap_server(self) -> bool: + assert self.upstream is not None + assert isinstance(self.upstream.connection, socket.socket) + do_close = False try: - # wrap_client also flushes client data before wrapping - # sending to client can raise, handle expected exceptions - self.wrap_client() + self.upstream.wrap(text_(self.request.host), self.flags.ca_file) + except ssl.SSLCertVerificationError: # Server raised certificate verification error + # When --disable-interception-on-ssl-cert-verification-error flag is on, + # we will cache such upstream hosts and avoid intercepting them for future + # requests. + logger.warning( + 'ssl.SSLCertVerificationError: ' + + 'Server raised cert verification error for upstream: {0}'.format( + self.upstream.addr[0], + ), + ) + do_close = True + except ssl.SSLError as e: + if e.reason == 'SSLV3_ALERT_HANDSHAKE_FAILURE': + logger.warning( + '{0}: '.format(e.reason) + + 'Server raised handshake alert failure for upstream: {0}'.format( + self.upstream.addr[0], + ), + ) + else: + logger.exception( + 'SSLError when wrapping client for upstream: {0}'.format( + self.upstream.addr[0], + ), exc_info=e, + ) + do_close = True + finally: + if do_close: + raise HttpProtocolException( + 'Exception when wrapping server for interception', + ) + assert isinstance(self.upstream.connection, ssl.SSLSocket) + return False + + def wrap_client(self) -> bool: + assert self.upstream is not None and self.flags.ca_signing_key_file is not None + assert isinstance(self.upstream.connection, ssl.SSLSocket) + do_close = False + try: + # TODO: Perform async certificate generation + generated_cert = self.generate_upstream_certificate( + cast(Dict[str, Any], self.upstream.connection.getpeercert()), + ) + self.client.wrap(self.flags.ca_signing_key_file, generated_cert) except subprocess.TimeoutExpired as e: # Popen communicate timeout logger.exception( - 'TimeoutExpired during certificate generation', exc_info=e) - return True + 'TimeoutExpired during certificate generation', exc_info=e, + ) + do_close = True + except ssl.SSLCertVerificationError: # Client raised certificate verification error + # When --disable-interception-on-ssl-cert-verification-error flag is on, + # we will cache such upstream hosts and avoid intercepting them for future + # requests. + logger.warning( + 'ssl.SSLCertVerificationError: ' + + 'Client raised cert verification error for upstream: {0}'.format( + self.upstream.addr[0], + ), + ) + do_close = True + except ssl.SSLEOFError as e: + logger.warning( + 'ssl.SSLEOFError {0} when wrapping client for upstream: {1}'.format( + str(e), self.upstream.addr[0], + ), + ) + do_close = True + except ssl.SSLError as e: + if e.reason in ('TLSV1_ALERT_UNKNOWN_CA', 'UNSUPPORTED_PROTOCOL'): + logger.warning( + '{0}: '.format(e.reason) + + 'Client raised cert verification error for upstream: {0}'.format( + self.upstream.addr[0], + ), + ) + else: + logger.exception( + 'OSError when wrapping client for upstream: {0}'.format( + self.upstream.addr[0], + ), exc_info=e, + ) + do_close = True except BrokenPipeError: logger.error( - 'BrokenPipeError when wrapping client') - return True + 'BrokenPipeError when wrapping client for upstream: {0}'.format( + self.upstream.addr[0], + ), + ) + do_close = True except OSError as e: logger.exception( - 'OSError when wrapping client', exc_info=e) - return True - # Update all plugin connection reference - # TODO(abhinavsingh): Is this required? - for plugin in self.plugins.values(): - plugin.client._conn = self.client.connection - return self.client.connection - - def wrap_server(self) -> None: - assert self.server is not None - assert isinstance(self.server.connection, socket.socket) - self.server.wrap(text_(self.request.host), self.flags.ca_file) - assert isinstance(self.server.connection, ssl.SSLSocket) - - def wrap_client(self) -> None: - assert self.server is not None and self.flags.ca_signing_key_file is not None - assert isinstance(self.server.connection, ssl.SSLSocket) - generated_cert = self.generate_upstream_certificate( - cast(Dict[str, Any], self.server.connection.getpeercert())) - self.client.wrap(self.flags.ca_signing_key_file, generated_cert) - logger.debug( - 'TLS interception using %s', generated_cert) + 'OSError when wrapping client for upstream: {0}'.format( + self.upstream.addr[0], + ), exc_info=e, + ) + do_close = True + finally: + if do_close: + raise HttpProtocolException( + 'Exception when wrapping client for interception', + ) + logger.debug('TLS intercepting using %s', generated_cert) + return False # # Event emitter callbacks @@ -568,32 +904,31 @@ def emit_request_complete(self) -> None: if not self.flags.enable_events: return - assert self.request.path assert self.request.port self.event_queue.publish( request_id=self.uid.hex, event_name=eventNames.REQUEST_COMPLETE, event_payload={ 'url': text_(self.request.path) - if self.request.method == httpMethods.CONNECT + if self.request.is_https_tunnel() else 'http://%s:%d%s' % (text_(self.request.host), self.request.port, text_(self.request.path)), 'method': text_(self.request.method), 'headers': {text_(k): text_(v[1]) for k, v in self.request.headers.items()}, 'body': text_(self.request.body) if self.request.method == httpMethods.POST - else None + else None, }, - publisher_id=self.__class__.__name__ + publisher_id=self.__class__.__name__, ) - def emit_response_events(self) -> None: + def emit_response_events(self, chunk_size: int) -> None: if not self.flags.enable_events: return if self.response.state == httpParserStates.COMPLETE: self.emit_response_complete() elif self.response.state == httpParserStates.RCVING_BODY: - self.emit_response_chunk_received() + self.emit_response_chunk_received(chunk_size) elif self.response.state == httpParserStates.HEADERS_COMPLETE: self.emit_response_headers_complete() @@ -601,10 +936,38 @@ def emit_response_headers_complete(self) -> None: if not self.flags.enable_events: return - def emit_response_chunk_received(self) -> None: + self.event_queue.publish( + request_id=self.uid.hex, + event_name=eventNames.RESPONSE_HEADERS_COMPLETE, + event_payload={ + 'headers': {text_(k): text_(v[1]) for k, v in self.response.headers.items()}, + }, + publisher_id=self.__class__.__name__, + ) + + def emit_response_chunk_received(self, chunk_size: int) -> None: if not self.flags.enable_events: return + self.event_queue.publish( + request_id=self.uid.hex, + event_name=eventNames.RESPONSE_CHUNK_RECEIVED, + event_payload={ + 'chunk_size': chunk_size, + 'encoded_chunk_size': chunk_size, + }, + publisher_id=self.__class__.__name__, + ) + def emit_response_complete(self) -> None: if not self.flags.enable_events: return + + self.event_queue.publish( + request_id=self.uid.hex, + event_name=eventNames.RESPONSE_COMPLETE, + event_payload={ + 'encoded_response_size': self.response.total_size, + }, + publisher_id=self.__class__.__name__, + ) diff --git a/proxy/http/server/__init__.py b/proxy/http/server/__init__.py index 059c2cc128..f100ddaf31 100644 --- a/proxy/http/server/__init__.py +++ b/proxy/http/server/__init__.py @@ -7,6 +7,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + Submodules """ from .web import HttpWebServerPlugin from .pac_plugin import HttpWebServerPacFilePlugin diff --git a/proxy/http/server/pac_plugin.py b/proxy/http/server/pac_plugin.py index 20f131ddb9..68aad02cc8 100644 --- a/proxy/http/server/pac_plugin.py +++ b/proxy/http/server/pac_plugin.py @@ -7,14 +7,21 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + pac """ import gzip + from typing import List, Tuple, Optional, Any from .plugin import HttpWebServerBasePlugin from .protocols import httpProtocolTypes -from ..websocket import WebsocketFrame + from ..parser import HttpParser + from ...common.utils import bytes_, text_, build_http_response from ...common.flag import flags from ...common.constants import DEFAULT_PAC_FILE, DEFAULT_PAC_FILE_URL_PATH @@ -26,13 +33,15 @@ default=DEFAULT_PAC_FILE, help='A file (Proxy Auto Configuration) or string to serve when ' 'the server receives a direct file request. ' - 'Using this option enables proxy.HttpWebServerPlugin.') + 'Using this option enables proxy.HttpWebServerPlugin.', +) flags.add_argument( '--pac-file-url-path', type=str, default=text_(DEFAULT_PAC_FILE_URL_PATH), help='Default: %s. Web server path to serve the PAC file.' % - text_(DEFAULT_PAC_FILE_URL_PATH)) + text_(DEFAULT_PAC_FILE_URL_PATH), +) class HttpWebServerPacFilePlugin(HttpWebServerBasePlugin): @@ -54,15 +63,6 @@ def handle_request(self, request: HttpParser) -> None: if self.flags.pac_file and self.pac_file_response: self.client.queue(self.pac_file_response) - def on_websocket_open(self) -> None: - pass # pragma: no cover - - def on_websocket_message(self, frame: WebsocketFrame) -> None: - pass # pragma: no cover - - def on_websocket_close(self) -> None: - pass # pragma: no cover - def cache_pac_file_response(self) -> None: if self.flags.pac_file: try: @@ -70,9 +70,11 @@ def cache_pac_file_response(self) -> None: content = f.read() except IOError: content = bytes_(self.flags.pac_file) - self.pac_file_response = memoryview(build_http_response( - 200, reason=b'OK', headers={ - b'Content-Type': b'application/x-ns-proxy-autoconfig', - b'Content-Encoding': b'gzip', - }, body=gzip.compress(content) - )) + self.pac_file_response = memoryview( + build_http_response( + 200, reason=b'OK', headers={ + b'Content-Type': b'application/x-ns-proxy-autoconfig', + b'Content-Encoding': b'gzip', + }, body=gzip.compress(content), + ), + ) diff --git a/proxy/http/server/plugin.py b/proxy/http/server/plugin.py index 3cb737ea75..c9a8fe2da0 100644 --- a/proxy/http/server/plugin.py +++ b/proxy/http/server/plugin.py @@ -7,14 +7,21 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http """ -from abc import ABC, abstractmethod import argparse -from typing import List, Tuple + from uuid import UUID +from abc import ABC, abstractmethod +from typing import Any, Dict, List, Optional, Tuple + from ..websocket import WebsocketFrame from ..parser import HttpParser +from ...common.types import Readables, Writables from ...core.connection import TcpClientConnection from ...core.event import EventQueue @@ -27,12 +34,50 @@ def __init__( uid: UUID, flags: argparse.Namespace, client: TcpClientConnection, - event_queue: EventQueue): + event_queue: EventQueue, + ): self.uid = uid self.flags = flags self.client = client self.event_queue = event_queue + def name(self) -> str: + """A unique name for your plugin. + + Defaults to name of the class. This helps plugin developers to directly + access a specific plugin by its name.""" + return self.__class__.__name__ # pragma: no cover + + # TODO(abhinavsingh): get_descriptors, write_to_descriptors, read_from_descriptors + # can be placed into their own abstract class which can then be shared by + # HttpProxyBasePlugin, HttpWebServerBasePlugin and HttpProtocolHandlerPlugin class. + # + # Currently code has been shamelessly copied. Also these methods are not + # marked as abstract to avoid breaking custom plugins written by users for + # previous versions of proxy.py + # + # Since 3.4.0 + # + # @abstractmethod + def get_descriptors(self) -> Tuple[List[int], List[int]]: + return [], [] # pragma: no cover + + # @abstractmethod + def write_to_descriptors(self, w: Writables) -> bool: + """Implementations must now write/flush data over the socket. + + Note that buffer management is in-build into the connection classes. + Hence implementations MUST call + :meth:`~proxy.core.connection.connection.TcpConnection.flush` + here, to send any buffered data over the socket. + """ + return False # pragma: no cover + + # @abstractmethod + def read_from_descriptors(self, r: Readables) -> bool: + """Implementations must now read data over the socket.""" + return False # pragma: no cover + @abstractmethod def routes(self) -> List[Tuple[int, str]]: """Return List(protocol, path) that this plugin handles.""" @@ -43,17 +88,42 @@ def handle_request(self, request: HttpParser) -> None: """Handle the request and serve response.""" raise NotImplementedError() # pragma: no cover - @abstractmethod + def on_client_connection_close(self) -> None: + """Client has closed the connection, do any clean up task now.""" + pass + + # No longer abstract since v2.4.0 + # + # @abstractmethod def on_websocket_open(self) -> None: """Called when websocket handshake has finished.""" - raise NotImplementedError() # pragma: no cover + pass # pragma: no cover - @abstractmethod + # No longer abstract since v2.4.0 + # + # @abstractmethod def on_websocket_message(self, frame: WebsocketFrame) -> None: """Handle websocket frame.""" - raise NotImplementedError() # pragma: no cover + return None # pragma: no cover - @abstractmethod - def on_websocket_close(self) -> None: - """Called when websocket connection has been closed.""" - raise NotImplementedError() # pragma: no cover + # Deprecated since v2.4.0 + # + # Instead use on_client_connection_close. + # + # This callback is no longer invoked. Kindly + # update your plugin before upgrading to v2.4.0. + # + # @abstractmethod + # def on_websocket_close(self) -> None: + # """Called when websocket connection has been closed.""" + # raise NotImplementedError() # pragma: no cover + + def on_access_log(self, context: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """Use this method to override default access log format (see + DEFAULT_WEB_ACCESS_LOG_FORMAT) or to add/update/modify passed context + for usage by default access logger. + + Return updated log context to use for default logging format, OR + Return None if plugin has logged the request. + """ + return context diff --git a/proxy/http/server/protocols.py b/proxy/http/server/protocols.py index e2a99ae9e2..a561cb936f 100644 --- a/proxy/http/server/protocols.py +++ b/proxy/http/server/protocols.py @@ -7,12 +7,20 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + iterable """ from typing import NamedTuple -HttpProtocolTypes = NamedTuple('HttpProtocolTypes', [ - ('HTTP', int), - ('HTTPS', int), - ('WEBSOCKET', int), -]) +HttpProtocolTypes = NamedTuple( + 'HttpProtocolTypes', [ + ('HTTP', int), + ('HTTPS', int), + ('WEBSOCKET', int), + ], +) + httpProtocolTypes = HttpProtocolTypes(1, 2, 3) diff --git a/proxy/http/server/web.py b/proxy/http/server/web.py index d46e011021..129c088ffb 100644 --- a/proxy/http/server/web.py +++ b/proxy/http/server/web.py @@ -7,117 +7,167 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http """ -import gzip import re +import gzip import time +import socket import logging -import os import mimetypes -import socket + from typing import List, Tuple, Optional, Dict, Union, Any, Pattern -from .plugin import HttpWebServerBasePlugin -from .protocols import httpProtocolTypes +from ...common.constants import DEFAULT_STATIC_SERVER_DIR, PROXY_AGENT_HEADER_VALUE +from ...common.constants import DEFAULT_ENABLE_STATIC_SERVER, DEFAULT_ENABLE_WEB_SERVER +from ...common.constants import DEFAULT_MIN_COMPRESSION_LIMIT, DEFAULT_WEB_ACCESS_LOG_FORMAT +from ...common.utils import bytes_, text_, build_http_response, build_websocket_handshake_response +from ...common.backports import cached_property +from ...common.types import Readables, Writables +from ...common.flag import flags + +from ..codes import httpStatusCodes from ..exception import HttpProtocolException +from ..plugin import HttpProtocolHandlerPlugin from ..websocket import WebsocketFrame, websocketOpcodes -from ..codes import httpStatusCodes from ..parser import HttpParser, httpParserStates, httpParserTypes -from ..plugin import HttpProtocolHandlerPlugin -from ...common.utils import bytes_, text_, build_http_response, build_websocket_handshake_response -from ...common.constants import DEFAULT_STATIC_SERVER_DIR, PROXY_AGENT_HEADER_VALUE -from ...common.types import Readables, Writables -from ...common.flag import flags +from .plugin import HttpWebServerBasePlugin +from .protocols import httpProtocolTypes logger = logging.getLogger(__name__) +flags.add_argument( + '--enable-web-server', + action='store_true', + default=DEFAULT_ENABLE_WEB_SERVER, + help='Default: False. Whether to enable proxy.HttpWebServerPlugin.', +) + +flags.add_argument( + '--enable-static-server', + action='store_true', + default=DEFAULT_ENABLE_STATIC_SERVER, + help='Default: False. Enable inbuilt static file server. ' + 'Optionally, also use --static-server-dir to serve static content ' + 'from custom directory. By default, static file server serves ' + 'out of installed proxy.py python module folder.', +) + flags.add_argument( '--static-server-dir', type=str, default=DEFAULT_STATIC_SERVER_DIR, help='Default: "public" folder in directory where proxy.py is placed. ' 'This option is only applicable when static server is also enabled. ' - 'See --enable-static-server.' + 'See --enable-static-server.', +) + +flags.add_argument( + '--min-compression-length', + type=int, + default=DEFAULT_MIN_COMPRESSION_LIMIT, + help='Default: ' + str(DEFAULT_MIN_COMPRESSION_LIMIT) + ' bytes. ' + + 'Sets the minimum length of a response that will be compressed (gzipped).', ) class HttpWebServerPlugin(HttpProtocolHandlerPlugin): """HttpProtocolHandler plugin which handles incoming requests to local web server.""" - DEFAULT_404_RESPONSE = memoryview(build_http_response( - httpStatusCodes.NOT_FOUND, - reason=b'NOT FOUND', - headers={b'Server': PROXY_AGENT_HEADER_VALUE, - b'Connection': b'close'} - )) + DEFAULT_404_RESPONSE = memoryview( + build_http_response( + httpStatusCodes.NOT_FOUND, + reason=b'NOT FOUND', + headers={ + b'Server': PROXY_AGENT_HEADER_VALUE, + b'Content-Length': b'0', + b'Connection': b'close', + }, + ), + ) - DEFAULT_501_RESPONSE = memoryview(build_http_response( - httpStatusCodes.NOT_IMPLEMENTED, - reason=b'NOT IMPLEMENTED', - headers={b'Server': PROXY_AGENT_HEADER_VALUE, - b'Connection': b'close'} - )) + DEFAULT_501_RESPONSE = memoryview( + build_http_response( + httpStatusCodes.NOT_IMPLEMENTED, + reason=b'NOT IMPLEMENTED', + headers={ + b'Server': PROXY_AGENT_HEADER_VALUE, + b'Content-Length': b'0', + b'Connection': b'close', + }, + ), + ) def __init__( self, - *args: Any, **kwargs: Any) -> None: + *args: Any, **kwargs: Any, + ) -> None: super().__init__(*args, **kwargs) self.start_time: float = time.time() self.pipeline_request: Optional[HttpParser] = None self.switched_protocol: Optional[int] = None - self.routes: Dict[int, Dict[Pattern[str], HttpWebServerBasePlugin]] = { - httpProtocolTypes.HTTP: {}, - httpProtocolTypes.HTTPS: {}, - httpProtocolTypes.WEBSOCKET: {}, - } self.route: Optional[HttpWebServerBasePlugin] = None + self.plugins: Dict[str, HttpWebServerBasePlugin] = {} if b'HttpWebServerBasePlugin' in self.flags.plugins: for klass in self.flags.plugins[b'HttpWebServerBasePlugin']: - instance = klass( + instance: HttpWebServerBasePlugin = klass( self.uid, self.flags, self.client, - self.event_queue) - for (protocol, route) in instance.routes(): - self.routes[protocol][re.compile(route)] = instance + self.event_queue, + ) + self.plugins[instance.name()] = instance + + @cached_property(ttl=0) + def routes(self) -> Dict[int, Dict[Pattern[str], HttpWebServerBasePlugin]]: + r: Dict[int, Dict[Pattern[str], HttpWebServerBasePlugin]] = { + httpProtocolTypes.HTTP: {}, + httpProtocolTypes.HTTPS: {}, + httpProtocolTypes.WEBSOCKET: {}, + } + for name in self.plugins: + for (protocol, route) in self.plugins[name].routes(): + r[protocol][re.compile(route)] = self.plugins[name] + return r def encryption_enabled(self) -> bool: return self.flags.keyfile is not None and \ self.flags.certfile is not None @staticmethod - def read_and_build_static_file_response(path: str) -> memoryview: - with open(path, 'rb') as f: - content = f.read() - content_type = mimetypes.guess_type(path)[0] - if content_type is None: - content_type = 'text/plain' - return memoryview(build_http_response( - httpStatusCodes.OK, - reason=b'OK', - headers={ + def read_and_build_static_file_response(path: str, min_compression_limit: int) -> memoryview: + try: + with open(path, 'rb') as f: + content = f.read() + content_type = mimetypes.guess_type(path)[0] + if content_type is None: + content_type = 'text/plain' + headers = { b'Content-Type': bytes_(content_type), b'Cache-Control': b'max-age=86400', - b'Content-Encoding': b'gzip', b'Connection': b'close', - }, - body=gzip.compress(content))) - - def serve_file_or_404(self, path: str) -> bool: - """Read and serves a file from disk. - - Queues 404 Not Found for IOError. - Shouldn't this be server error? - """ - try: - self.client.queue( - self.read_and_build_static_file_response(path)) - except IOError: - self.client.queue(self.DEFAULT_404_RESPONSE) - return True + } + do_compress = len(content) > min_compression_limit + if do_compress: + headers.update({ + b'Content-Encoding': b'gzip', + }) + return memoryview( + build_http_response( + httpStatusCodes.OK, + reason=b'OK', + headers=headers, + body=gzip.compress(content) if do_compress else content, + ), + ) + except FileNotFoundError: + return HttpWebServerPlugin.DEFAULT_404_RESPONSE def try_upgrade(self) -> bool: if self.request.has_header(b'connection') and \ @@ -125,9 +175,14 @@ def try_upgrade(self) -> bool: if self.request.has_header(b'upgrade') and \ self.request.header(b'upgrade').lower() == b'websocket': self.client.queue( - memoryview(build_websocket_handshake_response( - WebsocketFrame.key_to_accept( - self.request.header(b'Sec-WebSocket-Key'))))) + memoryview( + build_websocket_handshake_response( + WebsocketFrame.key_to_accept( + self.request.header(b'Sec-WebSocket-Key'), + ), + ), + ), + ) self.switched_protocol = httpProtocolTypes.WEBSOCKET else: self.client.queue(self.DEFAULT_501_RESPONSE) @@ -135,57 +190,72 @@ def try_upgrade(self) -> bool: return False def on_request_complete(self) -> Union[socket.socket, bool]: - if self.request.has_upstream_server(): + if self.request.has_host(): return False - - assert self.request.path - + path = self.request.path or b'/' + # Routing for Http(s) requests + protocol = httpProtocolTypes.HTTPS \ + if self.encryption_enabled() else \ + httpProtocolTypes.HTTP + for route in self.routes[protocol]: + if route.match(text_(path)): + self.route = self.routes[protocol][route] + assert self.route + self.route.handle_request(self.request) + if self.request.has_header(b'connection') and \ + self.request.header(b'connection').lower() == b'close': + return True + return False # If a websocket route exists for the path, try upgrade for route in self.routes[httpProtocolTypes.WEBSOCKET]: - match = route.match(text_(self.request.path)) - if match: + if route.match(text_(path)): self.route = self.routes[httpProtocolTypes.WEBSOCKET][route] - # Connection upgrade teardown = self.try_upgrade() if teardown: return True - # For upgraded connections, nothing more to do if self.switched_protocol: # Invoke plugin.on_websocket_open + assert self.route self.route.on_websocket_open() return False - break - - # Routing for Http(s) requests - protocol = httpProtocolTypes.HTTPS \ - if self.encryption_enabled() else \ - httpProtocolTypes.HTTP - for route in self.routes[protocol]: - match = route.match(text_(self.request.path)) - if match: - self.route = self.routes[protocol][route] - self.route.handle_request(self.request) - return False - # No-route found, try static serving if enabled if self.flags.enable_static_server: - path = text_(self.request.path).split('?')[0] - if os.path.isfile(self.flags.static_server_dir + path): - return self.serve_file_or_404( - self.flags.static_server_dir + path) - + path = text_(path).split('?')[0] + self.client.queue( + self.read_and_build_static_file_response( + self.flags.static_server_dir + path, + self.flags.min_compression_limit, + ), + ) + return True # Catch all unhandled web server requests, return 404 self.client.queue(self.DEFAULT_404_RESPONSE) return True - def write_to_descriptors(self, w: Writables) -> bool: - pass + def get_descriptors(self) -> Tuple[List[int], List[int]]: + r, w = [], [] + for plugin in self.plugins.values(): + r1, w1 = plugin.get_descriptors() + r.extend(r1) + w.extend(w1) + return r, w + + async def write_to_descriptors(self, w: Writables) -> bool: + for plugin in self.plugins.values(): + teardown = plugin.write_to_descriptors(w) + if teardown: + return True + return False - def read_from_descriptors(self, r: Readables) -> bool: - pass + async def read_from_descriptors(self, r: Readables) -> bool: + for plugin in self.plugins.values(): + teardown = plugin.read_from_descriptors(r) + if teardown: + return True + return False def on_client_data(self, raw: memoryview) -> Optional[memoryview]: if self.switched_protocol == httpProtocolTypes.WEBSOCKET: @@ -194,11 +264,12 @@ def on_client_data(self, raw: memoryview) -> Optional[memoryview]: remaining = raw.tobytes() frame = WebsocketFrame() while remaining != b'': - # TODO: Teardown if invalid protocol exception + # TODO: Tear down if invalid protocol exception remaining = frame.parse(remaining) if frame.opcode == websocketOpcodes.CONNECTION_CLOSE: logger.warning( - 'Client sent connection close packet') + 'Client sent connection close packet', + ) raise HttpProtocolException() else: assert self.route @@ -207,12 +278,13 @@ def on_client_data(self, raw: memoryview) -> Optional[memoryview]: return None # If 1st valid request was completed and it's a HTTP/1.1 keep-alive # And only if we have a route, parse pipeline requests - elif self.request.state == httpParserStates.COMPLETE and \ + if self.request.state == httpParserStates.COMPLETE and \ self.request.is_http_1_1_keep_alive() and \ self.route is not None: if self.pipeline_request is None: self.pipeline_request = HttpParser( - httpParserTypes.REQUEST_PARSER) + httpParserTypes.REQUEST_PARSER, + ) # TODO(abhinavsingh): Remove .tobytes after parser is memoryview # compliant self.pipeline_request.parse(raw.tobytes()) @@ -220,7 +292,8 @@ def on_client_data(self, raw: memoryview) -> Optional[memoryview]: self.route.handle_request(self.pipeline_request) if not self.pipeline_request.is_http_1_1_keep_alive(): logger.error( - 'Pipelined request is not keep-alive, will teardown request...') + 'Pipelined request is not keep-alive, will tear down request...', + ) raise HttpProtocolException() self.pipeline_request = None return raw @@ -229,23 +302,26 @@ def on_response_chunk(self, chunk: List[memoryview]) -> List[memoryview]: return chunk def on_client_connection_close(self) -> None: - if self.request.has_upstream_server(): + if self.request.has_host(): return - if self.switched_protocol: - # Invoke plugin.on_websocket_close - assert self.route - self.route.on_websocket_close() - self.access_log() - - def access_log(self) -> None: - logger.info( - '%s:%s - %s %s - %.2f ms' % - (self.client.addr[0], - self.client.addr[1], - text_(self.request.method), - text_(self.request.path), - (time.time() - self.start_time) * 1000)) - - def get_descriptors( - self) -> Tuple[List[socket.socket], List[socket.socket]]: - return [], [] + context = { + 'client_addr': self.client.address, + 'request_method': text_(self.request.method), + 'request_path': text_(self.request.path), + 'connection_time_ms': '%.2f' % ((time.time() - self.start_time) * 1000), + } + log_handled = False + if self.route: + # May be merge on_client_connection_close and on_access_log??? + # probably by simply deprecating on_client_connection_close in future. + self.route.on_client_connection_close() + ctx = self.route.on_access_log(context) + if ctx is None: + log_handled = True + else: + context = ctx + if not log_handled: + self.access_log(context) + + def access_log(self, context: Dict[str, Any]) -> None: + logger.info(DEFAULT_WEB_ACCESS_LOG_FORMAT.format_map(context)) diff --git a/proxy/http/url.py b/proxy/http/url.py new file mode 100644 index 0000000000..2d50743a71 --- /dev/null +++ b/proxy/http/url.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + url +""" +from typing import Optional, Tuple + +from ..common.constants import COLON, SLASH +from ..common.utils import text_ + + +class Url: + """``urllib.urlparse`` doesn't work for proxy.py, so we wrote a simple URL. + + Currently, URL only implements what is necessary for HttpParser to work. + """ + + def __init__( + self, + scheme: Optional[bytes] = None, + hostname: Optional[bytes] = None, + port: Optional[int] = None, + remainder: Optional[bytes] = None, + ) -> None: + self.scheme: Optional[bytes] = scheme + self.hostname: Optional[bytes] = hostname + self.port: Optional[int] = port + self.remainder: Optional[bytes] = remainder + + def __str__(self) -> str: + url = '' + if self.scheme: + url += '{0}://'.format(text_(self.scheme)) + if self.hostname: + url += text_(self.hostname) + if self.port: + url += ':{0}'.format(self.port) + if self.remainder: + url += text_(self.remainder) + return url + + @classmethod + def from_bytes(cls, raw: bytes) -> 'Url': + """A URL within proxy.py core can have several styles, + because proxy.py supports both proxy and web server use cases. + + Example: + For a Web server, url is like ``/`` or ``/get`` or ``/get?key=value`` + For a HTTPS connect tunnel, url is like ``httpbin.org:443`` + For a HTTP proxy request, url is like ``http://httpbin.org/get`` + + Further: + 1) URL may contain unicode characters + 2) URL may contain IPv4 and IPv6 format addresses instead of domain names + + We use heuristics based approach for our URL parser. + """ + sraw = raw.decode('utf-8') + if sraw[0] == SLASH.decode('utf-8'): + return cls(remainder=raw) + if sraw.startswith('https://') or sraw.startswith('http://'): + is_https = sraw.startswith('https://') + rest = raw[len(b'https://'):] \ + if is_https \ + else raw[len(b'http://'):] + parts = rest.split(SLASH) + host, port = Url.parse_host_and_port(parts[0]) + return cls( + scheme=b'https' if is_https else b'http', + hostname=host, + port=port, + remainder=None if len(parts) == 1 else ( + SLASH + SLASH.join(parts[1:]) + ), + ) + host, port = Url.parse_host_and_port(raw) + return cls(hostname=host, port=port) + + @staticmethod + def parse_host_and_port(raw: bytes) -> Tuple[bytes, Optional[int]]: + parts = raw.split(COLON) + port: Optional[int] = None + if len(parts) == 1: + return parts[0], None + if len(parts) == 2: + host, port = COLON.join(parts[:-1]), int(parts[-1]) + if len(parts) > 2: + try: + port = int(parts[-1]) + host = COLON.join(parts[:-1]) + except ValueError: + # If unable to convert last part into port, + # this is the IPv6 scenario. Treat entire + # data as host + host, port = raw, None + # patch up invalid ipv6 scenario + rhost = host.decode('utf-8') + if COLON.decode('utf-8') in rhost and rhost[0] != '[' and rhost[-1] != ']': + host = b'[' + host + b']' + return host, port diff --git a/proxy/http/websocket/__init__.py b/proxy/http/websocket/__init__.py index 2870e3b26f..350ab7b8ac 100644 --- a/proxy/http/websocket/__init__.py +++ b/proxy/http/websocket/__init__.py @@ -7,6 +7,13 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + Submodules + websocket + Websocket """ from .frame import WebsocketFrame, websocketOpcodes from .client import WebsocketClient diff --git a/proxy/http/websocket/client.py b/proxy/http/websocket/client.py index 716d0faea3..f4e25573bb 100644 --- a/proxy/http/websocket/client.py +++ b/proxy/http/websocket/client.py @@ -7,12 +7,17 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + websocket """ +import ssl import base64 -import selectors import socket import secrets -import ssl +import selectors from typing import Optional, Union, Callable @@ -20,26 +25,35 @@ from ..parser import httpParserTypes, HttpParser -from ...common.constants import DEFAULT_BUFFER_SIZE +from ...common.constants import DEFAULT_BUFFER_SIZE, DEFAULT_SELECTOR_SELECT_TIMEOUT from ...common.utils import new_socket_connection, build_websocket_handshake_request, text_ from ...core.connection import tcpConnectionTypes, TcpConnection class WebsocketClient(TcpConnection): - def __init__(self, - hostname: bytes, - port: int, - path: bytes = b'/', - on_message: Optional[Callable[[WebsocketFrame], None]] = None) -> None: + def __init__( + self, + hostname: bytes, + port: int, + path: bytes = b'/', + on_message: Optional[Callable[[WebsocketFrame], None]] = None, + ) -> None: super().__init__(tcpConnectionTypes.CLIENT) self.hostname: bytes = hostname self.port: int = port self.path: bytes = path self.sock: socket.socket = new_socket_connection( - (socket.gethostbyname(text_(self.hostname)), self.port)) - self.on_message: Optional[Callable[[ - WebsocketFrame], None]] = on_message + (socket.gethostbyname(text_(self.hostname)), self.port), + ) + self.on_message: Optional[ + Callable[ + [ + WebsocketFrame, + ], + None, + ] + ] = on_message self.selector: selectors.DefaultSelector = selectors.DefaultSelector() @property @@ -56,17 +70,19 @@ def upgrade(self) -> None: build_websocket_handshake_request( key, url=self.path, - host=self.hostname)) + host=self.hostname, + ), + ) response = HttpParser(httpParserTypes.RESPONSE_PARSER) response.parse(self.sock.recv(DEFAULT_BUFFER_SIZE)) accept = response.header(b'Sec-Websocket-Accept') assert WebsocketFrame.key_to_accept(key) == accept def ping(self, data: Optional[bytes] = None) -> None: - pass + pass # pragma: no cover def pong(self, data: Optional[bytes] = None) -> None: - pass + pass # pragma: no cover def shutdown(self, _data: Optional[bytes] = None) -> None: """Closes connection with the server.""" @@ -77,7 +93,7 @@ def run_once(self) -> bool: if self.has_buffer(): ev |= selectors.EVENT_WRITE self.selector.register(self.sock.fileno(), ev) - events = self.selector.select(timeout=1) + events = self.selector.select(timeout=DEFAULT_SELECTOR_SELECT_TIMEOUT) self.selector.unregister(self.sock) for _, mask in events: if mask & selectors.EVENT_READ and self.on_message: @@ -97,8 +113,7 @@ def run_once(self) -> bool: def run(self) -> None: try: while not self.closed: - teardown = self.run_once() - if teardown: + if self.run_once(): break except KeyboardInterrupt: pass diff --git a/proxy/http/websocket/frame.py b/proxy/http/websocket/frame.py index 55f9d91b16..97387e59ef 100644 --- a/proxy/http/websocket/frame.py +++ b/proxy/http/websocket/frame.py @@ -7,25 +7,34 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + iterable + websocket + Websocket """ +import io import hashlib import base64 import struct import secrets import logging -import io from typing import TypeVar, Type, Optional, NamedTuple -WebsocketOpcodes = NamedTuple('WebsocketOpcodes', [ - ('CONTINUATION_FRAME', int), - ('TEXT_FRAME', int), - ('BINARY_FRAME', int), - ('CONNECTION_CLOSE', int), - ('PING', int), - ('PONG', int), -]) +WebsocketOpcodes = NamedTuple( + 'WebsocketOpcodes', [ + ('CONTINUATION_FRAME', int), + ('TEXT_FRAME', int), + ('BINARY_FRAME', int), + ('CONNECTION_CLOSE', int), + ('PING', int), + ('PONG', int), + ], +) websocketOpcodes = WebsocketOpcodes(0x0, 0x1, 0x2, 0x8, 0x9, 0xA) @@ -91,35 +100,38 @@ def build(self) -> bytes: (1 << 6 if self.rsv1 else 0) | (1 << 5 if self.rsv2 else 0) | (1 << 4 if self.rsv3 else 0) | - self.opcode - )) + self.opcode, + ), + ) assert self.payload_length is not None if self.payload_length < 126: raw.write( struct.pack( '!B', - (1 << 7 if self.masked else 0) | self.payload_length - ) + (1 << 7 if self.masked else 0) | self.payload_length, + ), ) elif self.payload_length < 1 << 16: raw.write( struct.pack( '!BH', (1 << 7 if self.masked else 0) | 126, - self.payload_length - ) + self.payload_length, + ), ) elif self.payload_length < 1 << 64: raw.write( struct.pack( '!BHQ', (1 << 7 if self.masked else 0) | 127, - self.payload_length - ) + self.payload_length, + ), ) else: - raise ValueError(f'Invalid payload_length { self.payload_length },' - f'maximum allowed { 1 << 64 }') + raise ValueError( + f'Invalid payload_length { self.payload_length },' + f'maximum allowed { 1 << 64 }', + ) if self.masked and self.data: mask = secrets.token_bytes(4) if self.mask is None else self.mask raw.write(mask) diff --git a/proxy/plugin/__init__.py b/proxy/plugin/__init__.py index 8450e8f66f..1581c64933 100644 --- a/proxy/plugin/__init__.py +++ b/proxy/plugin/__init__.py @@ -7,6 +7,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + Cloudflare + Submodules """ from .cache import CacheResponsesPlugin, BaseCacheResponsesPlugin from .filter_by_upstream import FilterByUpstreamHostPlugin @@ -21,6 +26,8 @@ from .filter_by_client_ip import FilterByClientIpPlugin from .filter_by_url_regex import FilterByURLRegexPlugin from .modify_chunk_response import ModifyChunkResponsePlugin +from .custom_dns_resolver import CustomDnsResolverPlugin +from .cloudflare_dns import CloudflareDnsResolverPlugin __all__ = [ 'CacheResponsesPlugin', @@ -37,4 +44,6 @@ 'FilterByClientIpPlugin', 'ModifyChunkResponsePlugin', 'FilterByURLRegexPlugin', + 'CustomDnsResolverPlugin', + 'CloudflareDnsResolverPlugin', ] diff --git a/proxy/plugin/adblock.json b/proxy/plugin/adblock.json new file mode 100644 index 0000000000..3484197cda --- /dev/null +++ b/proxy/plugin/adblock.json @@ -0,0 +1,32 @@ +[{ + "regex": "tpc.googlesyndication.com/simgad/.*", + "notes": "Google image ads" +}, +{ + "regex": "tpc.googlesyndication.com/sadbundle/.*", + "notes": "Google animated ad bundles" +}, +{ + "regex": "pagead\\d+.googlesyndication.com/.*", + "notes": "Google tracking" +}, +{ + "regex": "(www){0,1}.google-analytics.com/r/collect\\?.*", + "notes": "Google tracking" +}, +{ + "regex": "(www){0,1}.facebook.com/tr/.*", + "notes": "Facebook tracking" +}, +{ + "regex": "tpc.googlesyndication.com/daca_images/simgad/.*", + "notes": "Google image ads" +}, +{ + "regex": ".*.2mdn.net/videoplayback/.*", + "notes": "Twitch.tv video ads" +}, +{ + "regex": "(www.){0,1}google.com(.*)/pagead/.*", + "notes": "Google ads" +}] diff --git a/proxy/plugin/cache/base.py b/proxy/plugin/cache/base.py index 81a2ef65f9..e74d7dd960 100644 --- a/proxy/plugin/cache/base.py +++ b/proxy/plugin/cache/base.py @@ -30,7 +30,8 @@ class BaseCacheResponsesPlugin(HttpProxyBasePlugin): def __init__( self, *args: Any, - **kwargs: Any) -> None: + **kwargs: Any, + ) -> None: super().__init__(*args, **kwargs) self.store: Optional[CacheStore] = None @@ -38,7 +39,8 @@ def set_store(self, store: CacheStore) -> None: self.store = store def before_upstream_connection( - self, request: HttpParser) -> Optional[HttpParser]: + self, request: HttpParser, + ) -> Optional[HttpParser]: assert self.store try: self.store.open(request) @@ -47,7 +49,8 @@ def before_upstream_connection( return request def handle_client_request( - self, request: HttpParser) -> Optional[HttpParser]: + self, request: HttpParser, + ) -> Optional[HttpParser]: assert self.store return self.store.cache_request(request) diff --git a/proxy/plugin/cache/cache_responses.py b/proxy/plugin/cache/cache_responses.py index f6da087e5d..200a60cb39 100644 --- a/proxy/plugin/cache/cache_responses.py +++ b/proxy/plugin/cache/cache_responses.py @@ -24,5 +24,6 @@ class CacheResponsesPlugin(BaseCacheResponsesPlugin): def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) self.disk_store = OnDiskCacheStore( - uid=self.uid, cache_dir=self.flags.cache_dir) + uid=self.uid, cache_dir=self.flags.cache_dir, + ) self.set_store(self.disk_store) diff --git a/proxy/plugin/cache/store/disk.py b/proxy/plugin/cache/store/disk.py index 1f472a12c5..41429809b4 100644 --- a/proxy/plugin/cache/store/disk.py +++ b/proxy/plugin/cache/store/disk.py @@ -27,7 +27,8 @@ '--cache-dir', type=str, default=tempfile.gettempdir(), - help='Default: A temporary directory. Flag only applicable when cache plugin is used with on-disk storage.' + help='Default: A temporary directory. ' + + 'Flag only applicable when cache plugin is used with on-disk storage.', ) @@ -42,7 +43,8 @@ def __init__(self, uid: UUID, cache_dir: str) -> None: def open(self, request: HttpParser) -> None: self.cache_file_path = os.path.join( self.cache_dir, - '%s-%s.txt' % (text_(request.host), self.uid.hex)) + '%s-%s.txt' % (text_(request.host), self.uid.hex), + ) self.cache_file = open(self.cache_file_path, "wb") def cache_request(self, request: HttpParser) -> Optional[HttpParser]: diff --git a/proxy/plugin/cloudflare_dns.py b/proxy/plugin/cloudflare_dns.py new file mode 100644 index 0000000000..781cb8d489 --- /dev/null +++ b/proxy/plugin/cloudflare_dns.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. + + .. spelling:: + + Cloudflare + cloudflare + dns +""" +import logging + +try: + import httpx +except ImportError: # pragma: no cover + pass + +from typing import Optional, Tuple + +from ..common.flag import flags +from ..http.proxy import HttpProxyBasePlugin + +logger = logging.getLogger(__name__) + + +flags.add_argument( + '--cloudflare-dns-mode', + type=str, + default='security', + help='Default: security. Either "security" (for malware protection) ' + + 'or "family" (for malware and adult content protection)', +) + + +class CloudflareDnsResolverPlugin(HttpProxyBasePlugin): + """This plugin uses Cloudflare DNS resolver to provide protection + against malware and adult content. Implementation uses :term:`DoH` + specification. + + See https://developers.cloudflare.com/1.1.1.1/1.1.1.1-for-families + See https://developers.cloudflare.com/1.1.1.1/encrypted-dns/dns-over-https/make-api-requests/dns-json + + .. note:: + + For this plugin to work, make sure to bypass proxy for 1.1.1.1 + + .. note:: + + This plugin requires additional dependency because :term:`DoH` + mandates a HTTP2 complaint client. Install `httpx` dependency + as:: + + pip install "httpx[http2]" + """ + + def resolve_dns(self, host: str, port: int) -> Tuple[Optional[str], Optional[Tuple[str, int]]]: + try: + context = httpx.create_ssl_context(http2=True) + # TODO: Support resolution via Authority (SOA) to add support for + # AAAA (IPv6) query + r = httpx.get( + 'https://{0}.cloudflare-dns.com/dns-query?name={1}&type=A'.format( + self.flags.cloudflare_dns_mode, host, + ), + headers={'accept': 'application/dns-json'}, + verify=context, + timeout=httpx.Timeout(timeout=5.0), + proxies={ + 'all://': None, + }, + ) + if r.status_code != 200: + return None, None + response = r.json() + answers = response.get('Answer', []) + if len(answers) == 0: + return None, None + # TODO: Utilize TTL to cache response locally + # instead of making a DNS query repeatedly for the same host. + return answers[0]['data'], None + except Exception as e: + logger.info( + 'Unable to resolve DNS-over-HTTPS for host {0} : {1}'.format( + host, str(e), + ), + ) + return None, None diff --git a/proxy/plugin/custom_dns_resolver.py b/proxy/plugin/custom_dns_resolver.py new file mode 100644 index 0000000000..f40a93e8ed --- /dev/null +++ b/proxy/plugin/custom_dns_resolver.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. + + .. spelling:: + + dns +""" +import socket + +from typing import Optional, Tuple + +from ..http.proxy import HttpProxyBasePlugin + + +class CustomDnsResolverPlugin(HttpProxyBasePlugin): + """This plugin demonstrate how to use your own custom DNS resolver.""" + + def resolve_dns(self, host: str, port: int) -> Tuple[Optional[str], Optional[Tuple[str, int]]]: + """Here we are using in-built python resolver for demonstration. + + Ideally you would like to query your custom DNS server or even + use :term:`DoH` to make real sense out of this plugin. + + The second parameter returned is None. Return a 2-tuple to + configure underlying interface to use for connection to the + upstream server. + """ + try: + return socket.getaddrinfo(host, port, proto=socket.IPPROTO_TCP)[0][4][0], None + except socket.gaierror: + # Ideally we can also thrown HttpRequestRejected or HttpProtocolException here + # Returning None simply fallback to core generated exceptions. + return None, None diff --git a/proxy/plugin/filter_by_client_ip.py b/proxy/plugin/filter_by_client_ip.py index 95169b4884..4b3724e96d 100644 --- a/proxy/plugin/filter_by_client_ip.py +++ b/proxy/plugin/filter_by_client_ip.py @@ -7,21 +7,26 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + ip """ from typing import Optional from ..common.flag import flags -from ..http.exception import HttpRequestRejected + +from ..http import httpStatusCodes from ..http.parser import HttpParser -from ..http.codes import httpStatusCodes from ..http.proxy import HttpProxyBasePlugin +from ..http.exception import HttpRequestRejected flags.add_argument( '--filtered-client-ips', type=str, default='127.0.0.1,::1', - help='Default: 127.0.0.1,::1. Comma separated list of IPv4 and IPv6 addresses.' + help='Default: 127.0.0.1,::1. Comma separated list of IPv4 and IPv6 addresses.', ) @@ -29,22 +34,14 @@ class FilterByClientIpPlugin(HttpProxyBasePlugin): """Drop traffic by inspecting incoming client IP address.""" def before_upstream_connection( - self, request: HttpParser) -> Optional[HttpParser]: + self, request: HttpParser, + ) -> Optional[HttpParser]: + assert not self.flags.unix_socket_path and self.client.addr if self.client.addr[0] in self.flags.filtered_client_ips.split(','): raise HttpRequestRejected( status_code=httpStatusCodes.I_AM_A_TEAPOT, reason=b'I\'m a tea pot', headers={ b'Connection': b'close', - } + }, ) return request - - def handle_client_request( - self, request: HttpParser) -> Optional[HttpParser]: - return request - - def handle_upstream_chunk(self, chunk: memoryview) -> memoryview: - return chunk - - def on_upstream_connection_close(self) -> None: - pass diff --git a/proxy/plugin/filter_by_upstream.py b/proxy/plugin/filter_by_upstream.py index a919bd15be..0970fa6eac 100644 --- a/proxy/plugin/filter_by_upstream.py +++ b/proxy/plugin/filter_by_upstream.py @@ -10,34 +10,34 @@ """ from typing import Optional -from ..http.exception import HttpRequestRejected +from ..common.utils import text_ +from ..common.flag import flags + +from ..http import httpStatusCodes from ..http.parser import HttpParser -from ..http.codes import httpStatusCodes from ..http.proxy import HttpProxyBasePlugin +from ..http.exception import HttpRequestRejected + + +flags.add_argument( + '--filtered-upstream-hosts', + type=str, + default='facebook.com,www.facebook.com', + help='Default: Blocks Facebook. Comma separated list of IPv4 and IPv6 addresses.', +) class FilterByUpstreamHostPlugin(HttpProxyBasePlugin): """Drop traffic by inspecting upstream host.""" - FILTERED_DOMAINS = [b'google.com', b'www.google.com'] - def before_upstream_connection( - self, request: HttpParser) -> Optional[HttpParser]: - if request.host in self.FILTERED_DOMAINS: + self, request: HttpParser, + ) -> Optional[HttpParser]: + if text_(request.host) in self.flags.filtered_upstream_hosts.split(','): raise HttpRequestRejected( status_code=httpStatusCodes.I_AM_A_TEAPOT, reason=b'I\'m a tea pot', headers={ b'Connection': b'close', - } + }, ) return request - - def handle_client_request( - self, request: HttpParser) -> Optional[HttpParser]: - return request - - def handle_upstream_chunk(self, chunk: memoryview) -> memoryview: - return chunk - - def on_upstream_connection_close(self) -> None: - pass diff --git a/proxy/plugin/filter_by_url_regex.py b/proxy/plugin/filter_by_url_regex.py index 1e9d5efbc0..ef5dd80530 100644 --- a/proxy/plugin/filter_by_url_regex.py +++ b/proxy/plugin/filter_by_url_regex.py @@ -7,22 +7,36 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. -""" + .. spelling:: + + url +""" +import json import logging from typing import Optional, List, Dict, Any -from ..http.exception import HttpRequestRejected +from ..common.flag import flags +from ..common.utils import text_ + +from ..http import httpStatusCodes from ..http.parser import HttpParser -from ..http.codes import httpStatusCodes from ..http.proxy import HttpProxyBasePlugin -from ..common.utils import text_ +from ..http.exception import HttpRequestRejected import re logger = logging.getLogger(__name__) +# See adblock.json file in repository for sample example config +flags.add_argument( + '--filtered-url-regex-config', + type=str, + default='', + help='Default: No config. Comma separated list of IPv4 and IPv6 addresses.', +) + class FilterByURLRegexPlugin(HttpProxyBasePlugin): """Drops traffic by inspecting request URL and checking @@ -31,56 +45,16 @@ class FilterByURLRegexPlugin(HttpProxyBasePlugin): filtering ads. """ - FILTER_LIST: List[Dict[str, Any]] = [ - { - 'regex': b'tpc.googlesyndication.com/simgad/.*', - 'status_code': httpStatusCodes.NOT_FOUND, - 'notes': 'Google image ads', - }, - { - 'regex': b'tpc.googlesyndication.com/sadbundle/.*', - 'status_code': httpStatusCodes.NOT_FOUND, - 'notes': 'Google animated ad bundles', - }, - { - 'regex': b'pagead\\d+.googlesyndication.com/.*', - 'status_code': httpStatusCodes.NOT_FOUND, - 'notes': 'Google tracking', - }, - { - 'regex': b'(www){0,1}.google-analytics.com/r/collect\\?.*', - 'status_code': httpStatusCodes.NOT_FOUND, - 'notes': 'Google tracking', - }, - { - 'regex': b'(www){0,1}.facebook.com/tr/.*', - 'status_code': httpStatusCodes.NOT_FOUND, - 'notes': 'Facebook tracking', - }, - { - 'regex': b'tpc.googlesyndication.com/daca_images/simgad/.*', - 'status_code': httpStatusCodes.NOT_FOUND, - 'notes': 'Google image ads', - }, - { - 'regex': b'.*.2mdn.net/videoplayback/.*', - 'status_code': httpStatusCodes.NOT_FOUND, - 'notes': 'Twitch.tv video ads', - }, - { - 'regex': b'(www.){0,1}google.com(.*)/pagead/.*', - 'status_code': httpStatusCodes.NOT_FOUND, - 'notes': 'Google ads', - }, - ] - - def before_upstream_connection( - self, request: HttpParser) -> Optional[HttpParser]: - return request + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.filters: List[Dict[str, Any]] = [] + if self.flags.filtered_url_regex_config != '': + with open(self.flags.filtered_url_regex_config, 'rb') as f: + self.filters = json.load(f) def handle_client_request( - self, request: HttpParser) -> Optional[HttpParser]: - + self, request: HttpParser, + ) -> Optional[HttpParser]: # determine host request_host = None if request.host: @@ -98,39 +72,28 @@ def handle_client_request( request_host, request.path, ) - # check URL against list rule_number = 1 - for blocked_entry in self.FILTER_LIST: - + for blocked_entry in self.filters: # if regex matches on URL if re.search(text_(blocked_entry['regex']), text_(url)): - # log that the request has been filtered - logger.info("Blocked: %r with status_code '%r' by rule number '%r'" % ( - text_(url), - blocked_entry['status_code'], - rule_number, - )) - + logger.info( + "Blocked: %r with status_code '%r' by rule number '%r'" % ( + text_(url), + httpStatusCodes.NOT_FOUND, + rule_number, + ), + ) # close the connection with the status code from the filter # list raise HttpRequestRejected( - status_code=blocked_entry['status_code'], + status_code=httpStatusCodes.NOT_FOUND, headers={b'Connection': b'close'}, reason=b'Blocked', ) - # stop looping through filter list break - # increment rule number rule_number += 1 - return request - - def handle_upstream_chunk(self, chunk: memoryview) -> memoryview: - return chunk - - def on_upstream_connection_close(self) -> None: - pass diff --git a/proxy/plugin/man_in_the_middle.py b/proxy/plugin/man_in_the_middle.py index cc3ab63e7e..970547e6f2 100644 --- a/proxy/plugin/man_in_the_middle.py +++ b/proxy/plugin/man_in_the_middle.py @@ -8,29 +8,19 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from typing import Optional - from ..common.utils import build_http_response -from ..http.parser import HttpParser -from ..http.codes import httpStatusCodes +from ..http import httpStatusCodes from ..http.proxy import HttpProxyBasePlugin class ManInTheMiddlePlugin(HttpProxyBasePlugin): """Modifies upstream server responses.""" - def before_upstream_connection( - self, request: HttpParser) -> Optional[HttpParser]: - return request - - def handle_client_request( - self, request: HttpParser) -> Optional[HttpParser]: - return request - def handle_upstream_chunk(self, chunk: memoryview) -> memoryview: - return memoryview(build_http_response( - httpStatusCodes.OK, - reason=b'OK', body=b'Hello from man in the middle')) - - def on_upstream_connection_close(self) -> None: - pass + return memoryview( + build_http_response( + httpStatusCodes.OK, + reason=b'OK', + body=b'Hello from man in the middle', + ), + ) diff --git a/proxy/plugin/mock_rest_api.py b/proxy/plugin/mock_rest_api.py index 270c864408..12e3543deb 100644 --- a/proxy/plugin/mock_rest_api.py +++ b/proxy/plugin/mock_rest_api.py @@ -7,14 +7,19 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + api """ import json from typing import Optional from ..common.utils import bytes_, build_http_response, text_ + +from ..http import httpStatusCodes from ..http.parser import HttpParser from ..http.proxy import HttpProxyBasePlugin -from ..http.codes import httpStatusCodes class ProposedRestApiPlugin(HttpProxyBasePlugin): @@ -27,7 +32,7 @@ class ProposedRestApiPlugin(HttpProxyBasePlugin): without establishing upstream connection. Note: This plugin won't work if your client is making - HTTPS connection to api.example.com. + HTTPS connection to ``api.example.com``. """ API_SERVER = b'api.example.com' @@ -50,39 +55,46 @@ class ProposedRestApiPlugin(HttpProxyBasePlugin): 'url': text_(API_SERVER) + '/v1/users/2/', 'username': 'someone', }, - ] + ], }, } def before_upstream_connection( - self, request: HttpParser) -> Optional[HttpParser]: + self, request: HttpParser, + ) -> Optional[HttpParser]: # Return None to disable establishing connection to upstream # Most likely our api.example.com won't even exist under development # scenario return None def handle_client_request( - self, request: HttpParser) -> Optional[HttpParser]: + self, request: HttpParser, + ) -> Optional[HttpParser]: if request.host != self.API_SERVER: return request assert request.path if request.path in self.REST_API_SPEC: - self.client.queue(memoryview(build_http_response( - httpStatusCodes.OK, - reason=b'OK', - headers={b'Content-Type': b'application/json'}, - body=bytes_(json.dumps( - self.REST_API_SPEC[request.path])) - ))) + self.client.queue( + memoryview( + build_http_response( + httpStatusCodes.OK, + reason=b'OK', + headers={b'Content-Type': b'application/json'}, + body=bytes_( + json.dumps( + self.REST_API_SPEC[request.path], + ), + ), + ), + ), + ) else: - self.client.queue(memoryview(build_http_response( - httpStatusCodes.NOT_FOUND, - reason=b'NOT FOUND', body=b'Not Found' - ))) + self.client.queue( + memoryview( + build_http_response( + httpStatusCodes.NOT_FOUND, + reason=b'NOT FOUND', body=b'Not Found', + ), + ), + ) return None - - def handle_upstream_chunk(self, chunk: memoryview) -> memoryview: - return chunk - - def on_upstream_connection_close(self) -> None: - pass diff --git a/proxy/plugin/modify_chunk_response.py b/proxy/plugin/modify_chunk_response.py index 707da5de5f..ccfba8cb5c 100644 --- a/proxy/plugin/modify_chunk_response.py +++ b/proxy/plugin/modify_chunk_response.py @@ -8,7 +8,7 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from typing import Optional, Any +from typing import Any from ..http.parser import HttpParser, httpParserTypes, httpParserStates from ..http.proxy import HttpProxyBasePlugin @@ -29,23 +29,15 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: # Create a new http protocol parser for response payloads self.response = HttpParser(httpParserTypes.RESPONSE_PARSER) - def before_upstream_connection( - self, request: HttpParser) -> Optional[HttpParser]: - return request - - def handle_client_request( - self, request: HttpParser) -> Optional[HttpParser]: - return request - def handle_upstream_chunk(self, chunk: memoryview) -> memoryview: # Parse the response. # Note that these chunks also include headers self.response.parse(chunk.tobytes()) # If response is complete, modify and dispatch to client if self.response.state == httpParserStates.COMPLETE: - self.response.body = b'\n'.join(self.DEFAULT_CHUNKS) + b'\n' + # Avoid setting a body for responses where a body is not expected. + # Otherwise, example curl will report warnings. + if self.response.body_expected(): + self.response.body = b'\n'.join(self.DEFAULT_CHUNKS) + b'\n' self.client.queue(memoryview(self.response.build_response())) return memoryview(b'') - - def on_upstream_connection_close(self) -> None: - pass diff --git a/proxy/plugin/modify_post_data.py b/proxy/plugin/modify_post_data.py index 98b89daf5c..97d696261a 100644 --- a/proxy/plugin/modify_post_data.py +++ b/proxy/plugin/modify_post_data.py @@ -11,9 +11,10 @@ from typing import Optional from ..common.utils import bytes_ + +from ..http import httpMethods from ..http.parser import HttpParser from ..http.proxy import HttpProxyBasePlugin -from ..http.methods import httpMethods class ModifyPostDataPlugin(HttpProxyBasePlugin): @@ -22,26 +23,24 @@ class ModifyPostDataPlugin(HttpProxyBasePlugin): MODIFIED_BODY = b'{"key": "modified"}' def before_upstream_connection( - self, request: HttpParser) -> Optional[HttpParser]: + self, request: HttpParser, + ) -> Optional[HttpParser]: return request def handle_client_request( - self, request: HttpParser) -> Optional[HttpParser]: + self, request: HttpParser, + ) -> Optional[HttpParser]: if request.method == httpMethods.POST: request.body = ModifyPostDataPlugin.MODIFIED_BODY # Update Content-Length header only when request is NOT chunked # encoded if not request.is_chunked_encoded(): - request.add_header(b'Content-Length', - bytes_(len(request.body))) + request.add_header( + b'Content-Length', + bytes_(len(request.body)), + ) # Enforce content-type json if request.has_header(b'Content-Type'): request.del_header(b'Content-Type') request.add_header(b'Content-Type', b'application/json') return request - - def handle_upstream_chunk(self, chunk: memoryview) -> memoryview: - return chunk - - def on_upstream_connection_close(self) -> None: - pass diff --git a/proxy/plugin/proxy_pool.py b/proxy/plugin/proxy_pool.py index 3cb664c700..7ae7372c5f 100644 --- a/proxy/plugin/proxy_pool.py +++ b/proxy/plugin/proxy_pool.py @@ -9,76 +9,188 @@ :license: BSD, see LICENSE for more details. """ import random -import socket -from typing import Optional, Any +import logging -from ..common.constants import DEFAULT_BUFFER_SIZE, SLASH, COLON -from ..common.utils import new_socket_connection -from ..http.proxy import HttpProxyBasePlugin +from typing import Dict, List, Optional, Any + +from ..common.flag import flags + +from ..http import Url, httpMethods from ..http.parser import HttpParser +from ..http.exception import HttpProtocolException +from ..http.proxy import HttpProxyBasePlugin + +from ..core.base import TcpUpstreamConnectionHandler + +logger = logging.getLogger(__name__) + +DEFAULT_HTTP_ACCESS_LOG_FORMAT = '{client_ip}:{client_port} - ' + \ + '{request_method} {server_host}:{server_port}{request_path} -> ' + \ + '{upstream_proxy_host}:{upstream_proxy_port} - ' + \ + '{response_code} {response_reason} - {response_bytes} bytes - ' + \ + '{connection_time_ms} ms' + +DEFAULT_HTTPS_ACCESS_LOG_FORMAT = '{client_ip}:{client_port} - ' + \ + '{request_method} {server_host}:{server_port} -> ' + \ + '{upstream_proxy_host}:{upstream_proxy_port} - ' + \ + '{response_bytes} bytes - {connection_time_ms} ms' + +# Run two separate instances of proxy.py +# on port 9000 and 9001 BUT WITHOUT ProxyPool plugin +# to avoid infinite loops. +DEFAULT_PROXY_POOL: List[str] = [ + # Yes you may use the instance running with ProxyPoolPlugin itself. + # ProxyPool plugin will act as a no-op. + # 'localhost:8899', + # + # Remote proxies + # 'localhost:9000', + # 'localhost:9001', +] + +flags.add_argument( + '--proxy-pool', + action='append', + nargs=1, + default=DEFAULT_PROXY_POOL, + help='List of upstream proxies to use in the pool', +) -class ProxyPoolPlugin(HttpProxyBasePlugin): - """Proxy incoming client proxy requests through a set of upstream proxies.""" +class ProxyPoolPlugin(TcpUpstreamConnectionHandler, HttpProxyBasePlugin): + """Proxy pool plugin simply acts as a proxy adapter for proxy.py itself. - # Run two separate instances of proxy.py - # on port 9000 and 9001 BUT WITHOUT ProxyPool plugin - # to avoid infinite loops. - UPSTREAM_PROXY_POOL = [ - ('localhost', 9000), - ('localhost', 9001), - ] + Imagine this plugin as setting up proxy settings for proxy.py instance itself. + All incoming client requests are proxied to configured upstream proxies.""" def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) - self.conn: Optional[socket.socket] = None + # Cached attributes to be used during access log override + self.request_host_port_path_method: List[Any] = [ + None, None, None, None, + ] + + def handle_upstream_data(self, raw: memoryview) -> None: + self.client.queue(raw) def before_upstream_connection( - self, request: HttpParser) -> Optional[HttpParser]: - """Avoid upstream connection of server in the request. - Initialize, connection to upstream proxy. + self, request: HttpParser, + ) -> Optional[HttpParser]: + """Avoids establishing the default connection to upstream server + by returning None. """ + # TODO(abhinavsingh): Ideally connection to upstream proxy endpoints + # must be bootstrapped within it's own re-usable and gc'd pool, to avoid establishing + # a fresh upstream proxy connection for each client request. + # + # See :class:`~proxy.core.connection.pool.ConnectionPool` which is a work + # in progress for SSL cache handling. + # # Implement your own logic here e.g. round-robin, least connection etc. - self.conn = new_socket_connection( - random.choice(self.UPSTREAM_PROXY_POOL)) + endpoint = random.choice(self.flags.proxy_pool)[0].split(':') + if endpoint[0] == 'localhost' and endpoint[1] == '8899': + return request + logger.debug('Using endpoint: {0}:{1}'.format(*endpoint)) + self.initialize_upstream(endpoint[0], int(endpoint[1])) + assert self.upstream + try: + self.upstream.connect() + except TimeoutError: + logger.info( + 'Timed out connecting to upstream proxy {0}:{1}'.format( + *endpoint, + ), + ) + raise HttpProtocolException() + except ConnectionRefusedError: + # TODO(abhinavsingh): Try another choice, when all (or max configured) choices have + # exhausted, retry for configured number of times before giving up. + # + # Failing upstream proxies, must be removed from the pool temporarily. + # A periodic health check must put them back in the pool. This can be achieved + # using a datastructure without having to spawn separate thread/process for health + # check. + logger.info( + 'Connection refused by upstream proxy {0}:{1}'.format( + *endpoint, + ), + ) + raise HttpProtocolException() + logger.debug( + 'Established connection to upstream proxy {0}:{1}'.format( + *endpoint, + ), + ) return None def handle_client_request( - self, request: HttpParser) -> Optional[HttpParser]: - request.path = self.rebuild_original_path(request) - self.tunnel(request) - # Returning None indicates core to gracefully - # flush client buffer and teardown the connection - return None + self, request: HttpParser, + ) -> Optional[HttpParser]: + """Only invoked once after client original proxy request has been received completely.""" + if not self.upstream: + return request + assert self.upstream + # For log sanity (i.e. to avoid None:None), expose upstream host:port from headers + host, port = None, None + # Browser or applications may sometime send + # + # "CONNECT / HTTP/1.0\r\n\r\n" + # + # for proxy keep alive checks. + if request.has_header(b'host'): + url = Url.from_bytes(request.header(b'host')) + assert url.hostname + host, port = url.hostname.decode('utf-8'), url.port + port = port if port else ( + 443 if request.is_https_tunnel() else 80 + ) + path = None if not request.path else request.path.decode() + self.request_host_port_path_method = [ + host, port, path, request.method, + ] + # Queue original request to upstream proxy + self.upstream.queue(memoryview(request.build(for_proxy=True))) + return request + + def handle_client_data(self, raw: memoryview) -> Optional[memoryview]: + """Only invoked when before_upstream_connection returns None""" + # Queue data to the proxy endpoint + assert self.upstream + self.upstream.queue(raw) + return raw def handle_upstream_chunk(self, chunk: memoryview) -> memoryview: """Will never be called since we didn't establish an upstream connection.""" - return chunk + if not self.upstream: + return chunk + raise Exception("This should have never been called") def on_upstream_connection_close(self) -> None: - """Will never be called since we didn't establish an upstream connection.""" - pass + """Called when client connection has been closed.""" + if self.upstream and not self.upstream.closed: + logger.debug('Closing upstream proxy connection') + self.upstream.close() + self.upstream = None - def tunnel(self, request: HttpParser) -> None: - """Send to upstream proxy, receive from upstream proxy, queue back to client.""" - assert self.conn - self.conn.send(request.build()) - response = self.conn.recv(DEFAULT_BUFFER_SIZE) - self.client.queue(memoryview(response)) - - @staticmethod - def rebuild_original_path(request: HttpParser) -> bytes: - """Re-builds original upstream server URL. + def on_access_log(self, context: Dict[str, Any]) -> Optional[Dict[str, Any]]: + if not self.upstream: + return context + addr, port = (self.upstream.addr[0], self.upstream.addr[1]) \ + if self.upstream else (None, None) + context.update({ + 'upstream_proxy_host': addr, + 'upstream_proxy_port': port, + 'server_host': self.request_host_port_path_method[0], + 'server_port': self.request_host_port_path_method[1], + 'request_path': self.request_host_port_path_method[2], + 'response_bytes': self.total_size, + }) + self.access_log(context) + return None - proxy server core by default strips upstream host:port - from incoming client proxy request. - """ - assert request.url and request.host and request.port and request.path - return ( - request.url.scheme + - COLON + SLASH + SLASH + - request.host + - COLON + - str(request.port).encode() + - request.path - ) + def access_log(self, log_attrs: Dict[str, Any]) -> None: + access_log_format = DEFAULT_HTTPS_ACCESS_LOG_FORMAT + request_method = self.request_host_port_path_method[3] + if request_method and request_method != httpMethods.CONNECT: + access_log_format = DEFAULT_HTTP_ACCESS_LOG_FORMAT + logger.info(access_log_format.format_map(log_attrs)) diff --git a/proxy/plugin/redirect_to_custom_server.py b/proxy/plugin/redirect_to_custom_server.py index d2118e5fea..d04d7ca089 100644 --- a/proxy/plugin/redirect_to_custom_server.py +++ b/proxy/plugin/redirect_to_custom_server.py @@ -13,7 +13,6 @@ from ..http.proxy import HttpProxyBasePlugin from ..http.parser import HttpParser -from ..http.methods import httpMethods class RedirectToCustomServerPlugin(HttpProxyBasePlugin): @@ -22,24 +21,17 @@ class RedirectToCustomServerPlugin(HttpProxyBasePlugin): UPSTREAM_SERVER = b'http://localhost:8899/' def before_upstream_connection( - self, request: HttpParser) -> Optional[HttpParser]: + self, request: HttpParser, + ) -> Optional[HttpParser]: # Redirect all non-https requests to inbuilt WebServer. - if request.method != httpMethods.CONNECT: + if not request.is_https_tunnel(): request.set_url(self.UPSTREAM_SERVER) # Update Host header too, otherwise upstream can reject our request if request.has_header(b'Host'): request.del_header(b'Host') request.add_header( b'Host', urlparse.urlsplit( - self.UPSTREAM_SERVER).netloc) + self.UPSTREAM_SERVER, + ).netloc, + ) return request - - def handle_client_request( - self, request: HttpParser) -> Optional[HttpParser]: - return request - - def handle_upstream_chunk(self, chunk: memoryview) -> memoryview: - return chunk - - def on_upstream_connection_close(self) -> None: - pass diff --git a/proxy/plugin/reverse_proxy.py b/proxy/plugin/reverse_proxy.py index 3675b9eacd..0715d184f8 100644 --- a/proxy/plugin/reverse_proxy.py +++ b/proxy/plugin/reverse_proxy.py @@ -9,27 +9,37 @@ :license: BSD, see LICENSE for more details. """ import random -from typing import List, Tuple -from urllib import parse as urlparse +import logging -from ..common.constants import DEFAULT_BUFFER_SIZE, DEFAULT_HTTP_PORT -from ..common.utils import socket_connection, text_ +from typing import List, Tuple, Any, Dict, Optional + +from ..common.utils import text_ +from ..common.constants import DEFAULT_HTTPS_PORT, DEFAULT_HTTP_PORT + +from ..http import Url +from ..http.exception import HttpProtocolException from ..http.parser import HttpParser -from ..http.websocket import WebsocketFrame from ..http.server import HttpWebServerBasePlugin, httpProtocolTypes +from ..core.base import TcpUpstreamConnectionHandler + +logger = logging.getLogger(__name__) -class ReverseProxyPlugin(HttpWebServerBasePlugin): + +class ReverseProxyPlugin(TcpUpstreamConnectionHandler, HttpWebServerBasePlugin): """Extend in-built Web Server to add Reverse Proxy capabilities. - This example plugin is equivalent to following Nginx configuration: + This example plugin is equivalent to following Nginx configuration:: + ```text location /get { proxy_pass http://httpbin.org/get } + ``` - Example: + Example:: + ```console $ curl http://localhost:9000/get { "args": {}, @@ -39,34 +49,75 @@ class ReverseProxyPlugin(HttpWebServerBasePlugin): "User-Agent": "curl/7.64.1" }, "origin": "1.2.3.4, 5.6.7.8", - "url": "https://localhost/get" + "url": "http://localhost/get" } + ``` """ + # TODO: We must use nginx python parser and + # make this plugin nginx.conf complaint. REVERSE_PROXY_LOCATION: str = r'/get$' + # Randomly choose either http or https upstream endpoint. + # + # This is just to demonstrate that both http and https upstream + # reverse proxy works. REVERSE_PROXY_PASS = [ - b'http://httpbin.org/get' + b'http://httpbin.org/get', + b'https://httpbin.org/get', ] + def __init__(self, *args: Any, **kwargs: Any): + super().__init__(*args, **kwargs) + self.choice: Optional[Url] = None + + def handle_upstream_data(self, raw: memoryview) -> None: + self.client.queue(raw) + def routes(self) -> List[Tuple[int, str]]: return [ (httpProtocolTypes.HTTP, ReverseProxyPlugin.REVERSE_PROXY_LOCATION), - (httpProtocolTypes.HTTPS, ReverseProxyPlugin.REVERSE_PROXY_LOCATION) + (httpProtocolTypes.HTTPS, ReverseProxyPlugin.REVERSE_PROXY_LOCATION), ] def handle_request(self, request: HttpParser) -> None: - upstream = random.choice(ReverseProxyPlugin.REVERSE_PROXY_PASS) - url = urlparse.urlsplit(upstream) - assert url.hostname - with socket_connection((text_(url.hostname), url.port if url.port else DEFAULT_HTTP_PORT)) as conn: - conn.send(request.build()) - self.client.queue(memoryview(conn.recv(DEFAULT_BUFFER_SIZE))) + self.choice = Url.from_bytes( + random.choice(ReverseProxyPlugin.REVERSE_PROXY_PASS), + ) + assert self.choice.hostname + port = self.choice.port or \ + DEFAULT_HTTP_PORT \ + if self.choice.scheme == b'http' \ + else DEFAULT_HTTPS_PORT - def on_websocket_open(self) -> None: - pass + self.initialize_upstream(text_(self.choice.hostname), port) + assert self.upstream + try: + self.upstream.connect() + if self.choice.scheme == b'https': + self.upstream.wrap( + text_( + self.choice.hostname, + ), ca_file=str(self.flags.ca_file), + ) + self.upstream.queue(memoryview(request.build())) + except ConnectionRefusedError: + logger.info( + 'Connection refused by upstream server {0}:{1}'.format( + text_(self.choice.hostname), port, + ), + ) + raise HttpProtocolException() - def on_websocket_message(self, frame: WebsocketFrame) -> None: - pass + def on_client_connection_close(self) -> None: + if self.upstream and not self.upstream.closed: + logger.debug('Closing upstream server connection') + self.upstream.close() + self.upstream = None - def on_websocket_close(self) -> None: - pass + def on_access_log(self, context: Dict[str, Any]) -> Optional[Dict[str, Any]]: + log_format = '{client_addr} - {request_method} {request_path} -> {upstream_proxy_pass} - {connection_time_ms}ms' + context.update({ + 'upstream_proxy_pass': str(self.choice) if self.choice else None, + }) + logger.info(log_format.format_map(context)) + return None diff --git a/proxy/plugin/shortlink.py b/proxy/plugin/shortlink.py index 309fc1fbc2..5b1ff48ffa 100644 --- a/proxy/plugin/shortlink.py +++ b/proxy/plugin/shortlink.py @@ -7,13 +7,18 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + shortlink """ from typing import Optional from ..common.constants import DOT, SLASH from ..common.utils import build_http_response + +from ..http import httpStatusCodes from ..http.parser import HttpParser -from ..http.codes import httpStatusCodes from ..http.proxy import HttpProxyBasePlugin @@ -22,15 +27,15 @@ class ShortLinkPlugin(HttpProxyBasePlugin): Enable ShortLinkPlugin and speed up your daily browsing experience. - Example: - * f/ for facebook.com - * g/ for google.com - * t/ for twitter.com - * y/ for youtube.com - * proxy/ for py internal web servers. + Example:: + * ``f/`` for ``facebook.com`` + * ``g/`` for ``google.com` + * ``t/`` for ``twitter.com`` + * ``y/`` for ``youtube.com`` + * ``proxy/`` for ``py`` internal web servers. Customize map below for your taste and need. - Paths are also preserved. E.g. t/imoracle will + Paths are also preserved. E.g. ``t/imoracle`` will resolve to http://twitter.com/imoracle. """ @@ -47,38 +52,42 @@ class ShortLinkPlugin(HttpProxyBasePlugin): } def before_upstream_connection( - self, request: HttpParser) -> Optional[HttpParser]: + self, request: HttpParser, + ) -> Optional[HttpParser]: if request.host and request.host != b'localhost' and DOT not in request.host: # Avoid connecting to upstream return None return request def handle_client_request( - self, request: HttpParser) -> Optional[HttpParser]: + self, request: HttpParser, + ) -> Optional[HttpParser]: if request.host and request.host != b'localhost' and DOT not in request.host: if request.host in self.SHORT_LINKS: path = SLASH if not request.path else request.path - self.client.queue(memoryview(build_http_response( - httpStatusCodes.SEE_OTHER, reason=b'See Other', - headers={ - b'Location': b'http://' + self.SHORT_LINKS[request.host] + path, - b'Content-Length': b'0', - b'Connection': b'close', - } - ))) + self.client.queue( + memoryview( + build_http_response( + httpStatusCodes.SEE_OTHER, reason=b'See Other', + headers={ + b'Location': b'http://' + self.SHORT_LINKS[request.host] + path, + b'Content-Length': b'0', + b'Connection': b'close', + }, + ), + ), + ) else: - self.client.queue(memoryview(build_http_response( - httpStatusCodes.NOT_FOUND, reason=b'NOT FOUND', - headers={ - b'Content-Length': b'0', - b'Connection': b'close', - } - ))) + self.client.queue( + memoryview( + build_http_response( + httpStatusCodes.NOT_FOUND, reason=b'NOT FOUND', + headers={ + b'Content-Length': b'0', + b'Connection': b'close', + }, + ), + ), + ) return None return request - - def handle_upstream_chunk(self, chunk: memoryview) -> memoryview: - return chunk - - def on_upstream_connection_close(self) -> None: - pass diff --git a/proxy/plugin/web_server_route.py b/proxy/plugin/web_server_route.py index c8b4731a44..52f85deb48 100644 --- a/proxy/plugin/web_server_route.py +++ b/proxy/plugin/web_server_route.py @@ -12,8 +12,9 @@ from typing import List, Tuple from ..common.utils import build_http_response + +from ..http import httpStatusCodes from ..http.parser import HttpParser -from ..http.codes import httpStatusCodes from ..http.websocket import WebsocketFrame from ..http.server import HttpWebServerBasePlugin, httpProtocolTypes @@ -32,11 +33,21 @@ def routes(self) -> List[Tuple[int, str]]: def handle_request(self, request: HttpParser) -> None: if request.path == b'/http-route-example': - self.client.queue(memoryview(build_http_response( - httpStatusCodes.OK, body=b'HTTP route response'))) + self.client.queue( + memoryview( + build_http_response( + httpStatusCodes.OK, body=b'HTTP route response', + ), + ), + ) elif request.path == b'/https-route-example': - self.client.queue(memoryview(build_http_response( - httpStatusCodes.OK, body=b'HTTPS route response'))) + self.client.queue( + memoryview( + build_http_response( + httpStatusCodes.OK, body=b'HTTPS route response', + ), + ), + ) def on_websocket_open(self) -> None: logger.info('Websocket open') @@ -44,5 +55,5 @@ def on_websocket_open(self) -> None: def on_websocket_message(self, frame: WebsocketFrame) -> None: logger.info(frame.data) - def on_websocket_close(self) -> None: - logger.info('Websocket close') + def on_client_connection_close(self) -> None: + logger.debug('Client connection close') diff --git a/proxy/proxy.py b/proxy/proxy.py index 023c43cb4d..0f2346f935 100644 --- a/proxy/proxy.py +++ b/proxy/proxy.py @@ -7,438 +7,221 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + eventing """ -import abc -import argparse -import base64 -import collections -import contextlib -import ipaddress -import multiprocessing import os -import socket import sys import time import logging -import importlib -import inspect - -from types import TracebackType -from typing import Dict, List, Optional, Generator, Any, Tuple, Type, Union, cast - -from .common.utils import bytes_, text_ -from .common.types import IpAddress -from .common.version import __version__ -from .core.acceptor import AcceptorPool -from .http.handler import HttpProtocolHandler -from .common.flag import flags -from .common.constants import COMMA, DEFAULT_DATA_DIRECTORY_PATH, PLUGIN_PROXY_AUTH -from .common.constants import DEFAULT_DEVTOOLS_WS_PATH, DEFAULT_DISABLE_HEADERS -from .common.constants import DEFAULT_DISABLE_HTTP_PROXY, DEFAULT_NUM_WORKERS -from .common.constants import DEFAULT_ENABLE_DASHBOARD, DEFAULT_ENABLE_DEVTOOLS -from .common.constants import DEFAULT_ENABLE_STATIC_SERVER, DEFAULT_ENABLE_WEB_SERVER + +from typing import List, Optional, Any + +from .core.acceptor import AcceptorPool, ThreadlessPool, Listener +from .core.event import EventManager +from .common.utils import bytes_ +from .common.flag import FlagParser, flags from .common.constants import DEFAULT_LOG_FILE, DEFAULT_LOG_FORMAT, DEFAULT_LOG_LEVEL -from .common.constants import DEFAULT_OPEN_FILE_LIMIT, DEFAULT_PID_FILE, DEFAULT_PLUGINS -from .common.constants import DEFAULT_VERSION, DOT, PLUGIN_DASHBOARD, PLUGIN_DEVTOOLS_PROTOCOL -from .common.constants import PLUGIN_HTTP_PROXY, PLUGIN_INSPECT_TRAFFIC, PLUGIN_PAC_FILE -from .common.constants import PLUGIN_WEB_SERVER, PY2_DEPRECATION_MESSAGE +from .common.constants import DEFAULT_OPEN_FILE_LIMIT, DEFAULT_PLUGINS, DEFAULT_VERSION +from .common.constants import DEFAULT_ENABLE_DASHBOARD, DEFAULT_WORK_KLASS, DEFAULT_PID_FILE -if os.name != 'nt': - import resource logger = logging.getLogger(__name__) -flags.add_argument( - '--pid-file', - type=str, - default=DEFAULT_PID_FILE, - help='Default: None. Save parent process ID to a file.') flags.add_argument( '--version', '-v', action='store_true', default=DEFAULT_VERSION, - help='Prints proxy.py version.') -flags.add_argument( - '--disable-http-proxy', - action='store_true', - default=DEFAULT_DISABLE_HTTP_PROXY, - help='Default: False. Whether to disable proxy.HttpProxyPlugin.') -flags.add_argument( - '--enable-dashboard', - action='store_true', - default=DEFAULT_ENABLE_DASHBOARD, - help='Default: False. Enables proxy.py dashboard.' + help='Prints proxy.py version.', ) -flags.add_argument( - '--enable-devtools', - action='store_true', - default=DEFAULT_ENABLE_DEVTOOLS, - help='Default: False. Enables integration with Chrome Devtool Frontend. Also see --devtools-ws-path.' -) -flags.add_argument( - '--enable-static-server', - action='store_true', - default=DEFAULT_ENABLE_STATIC_SERVER, - help='Default: False. Enable inbuilt static file server. ' - 'Optionally, also use --static-server-dir to serve static content ' - 'from custom directory. By default, static file server serves ' - 'out of installed proxy.py python module folder.' -) -flags.add_argument( - '--enable-web-server', - action='store_true', - default=DEFAULT_ENABLE_WEB_SERVER, - help='Default: False. Whether to enable proxy.HttpWebServerPlugin.') + +# TODO: Convert me into 1-letter choices +# TODO: Add --verbose option which also +# starts to log traffic flowing between +# clients and upstream servers. flags.add_argument( '--log-level', type=str, default=DEFAULT_LOG_LEVEL, help='Valid options: DEBUG, INFO (default), WARNING, ERROR, CRITICAL. ' 'Both upper and lowercase values are allowed. ' - 'You may also simply use the leading character e.g. --log-level d') + 'You may also simply use the leading character e.g. --log-level d', +) + flags.add_argument( '--log-file', type=str, default=DEFAULT_LOG_FILE, - help='Default: sys.stdout. Log file destination.') + help='Default: sys.stdout. Log file destination.', +) + flags.add_argument( '--log-format', type=str, default=DEFAULT_LOG_FORMAT, - help='Log format for Python logger.') + help='Log format for Python logger.', +) + flags.add_argument( '--open-file-limit', type=int, default=DEFAULT_OPEN_FILE_LIMIT, help='Default: 1024. Maximum number of files (TCP connections) ' - 'that proxy.py can open concurrently.') + 'that proxy.py can open concurrently.', +) + flags.add_argument( '--plugins', - type=str, + action='append', + nargs='+', default=DEFAULT_PLUGINS, - help='Comma separated plugins') + help='Comma separated plugins. ' + + 'You may use --plugins flag multiple times.', +) + +# TODO: Ideally all `--enable-*` flags must be at the top-level. +# --enable-dashboard is specially needed here because +# ProxyDashboard class is not imported anywhere. +# +# Due to which, if we move this flag definition within dashboard +# plugin, users will have to explicitly enable dashboard plugin +# to also use flags provided by it. +flags.add_argument( + '--enable-dashboard', + action='store_true', + default=DEFAULT_ENABLE_DASHBOARD, + help='Default: False. Enables proxy.py dashboard.', +) + +flags.add_argument( + '--work-klass', + type=str, + default=DEFAULT_WORK_KLASS, + help='Default: ' + DEFAULT_WORK_KLASS + + '. Work klass to use for work execution.', +) + +flags.add_argument( + '--pid-file', + type=str, + default=DEFAULT_PID_FILE, + help='Default: None. Save "parent" process ID to a file.', +) class Proxy: - """Context manager for controlling core AcceptorPool server lifecycle. + """Proxy is a context manager to control proxy.py library core. - By default this context manager starts AcceptorPool with HttpProtocolHandler - worker class. - """ + By default, :class:`~proxy.core.pool.AcceptorPool` is started with + :class:`~proxy.http.handler.HttpProtocolHandler` work class. + By definition, it expects HTTP traffic to flow between clients and server. - def __init__(self, input_args: Optional[List[str]], **opts: Any) -> None: - self.flags = Proxy.initialize(input_args, **opts) - self.acceptors: Optional[AcceptorPool] = None + In ``--threadless`` mode and without ``--local-executor``, + a :class:`~proxy.core.executors.ThreadlessPool` is also started. + Executor pool receives newly accepted work by :class:`~proxy.core.acceptor.Acceptor` + and creates an instance of work class for processing the received work. - def write_pid_file(self) -> None: - if self.flags.pid_file is not None: - with open(self.flags.pid_file, 'wb') as pid_file: - pid_file.write(bytes_(os.getpid())) + Optionally, Proxy class also initializes the EventManager. + A multi-process safe pubsub system which can be used to build various + patterns for message sharing and/or signaling. + """ - def delete_pid_file(self) -> None: - if self.flags.pid_file and os.path.exists(self.flags.pid_file): - os.remove(self.flags.pid_file) + def __init__(self, input_args: Optional[List[str]] = None, **opts: Any) -> None: + self.flags = FlagParser.initialize(input_args, **opts) + self.listener: Optional[Listener] = None + self.executors: Optional[ThreadlessPool] = None + self.acceptors: Optional[AcceptorPool] = None + self.event_manager: Optional[EventManager] = None def __enter__(self) -> 'Proxy': + self.setup() + return self + + def __exit__(self, *args: Any) -> None: + self.shutdown() + + def setup(self) -> None: + # TODO: Introduce cron feature + # https://github.com/abhinavsingh/proxy.py/issues/392 + # + # TODO: Introduce ability to publish + # adhoc events which can modify behaviour of server + # at runtime. Example, updating flags, plugin + # configuration etc. + # + # TODO: Python shell within running proxy.py environment? + # + # TODO: Pid watcher which watches for processes started + # by proxy.py core. May be alert or restart those processes + # on failure. + self._write_pid_file() + # We setup listeners first because of flags.port override + # in case of ephemeral port being used + self.listener = Listener(flags=self.flags) + self.listener.setup() + # Override flags.port to match the actual port + # we are listening upon. This is necessary to preserve + # the server port when `--port=0` is used. + self.flags.port = self.listener._port + # Setup EventManager + if self.flags.enable_events: + logger.info('Core Event enabled') + self.event_manager = EventManager() + self.event_manager.setup() + event_queue = self.event_manager.queue \ + if self.event_manager is not None \ + else None + # Setup remote executors + if not self.flags.local_executor: + self.executors = ThreadlessPool( + flags=self.flags, + event_queue=event_queue, + ) + self.executors.setup() + # Setup acceptors self.acceptors = AcceptorPool( flags=self.flags, - work_klass=HttpProtocolHandler + listener=self.listener, + executor_queues=self.executors.work_queues if self.executors else [], + executor_pids=self.executors.work_pids if self.executors else [], + executor_locks=self.executors.work_locks if self.executors else [], + event_queue=event_queue, ) self.acceptors.setup() - self.write_pid_file() - return self + # TODO: May be close listener fd as we don't need it now - def __exit__( - self, - exc_type: Optional[Type[BaseException]], - exc_val: Optional[BaseException], - exc_tb: Optional[TracebackType]) -> None: + def shutdown(self) -> None: assert self.acceptors self.acceptors.shutdown() - self.delete_pid_file() - - @staticmethod - def initialize(input_args: Optional[List[str]] - = None, **opts: Any) -> argparse.Namespace: - if input_args is None: - input_args = [] - - if not Proxy.is_py3(): - print(PY2_DEPRECATION_MESSAGE) - sys.exit(1) - - # Discover flags from requested plugin. - # This also surface external plugin flags under --help - for i, f in enumerate(input_args): - if f == '--plugin': - Proxy.import_plugin(bytes_(input_args[i + 1])) - - # Parse flags - args = flags.parse_args(input_args) - - # Print version and exit - if args.version: - print(__version__) - sys.exit(0) - - # Setup logging module - Proxy.setup_logger(args.log_file, args.log_level, args.log_format) - - # Setup limits - Proxy.set_open_file_limit(args.open_file_limit) - - # Load plugins - default_plugins = Proxy.get_default_plugins(args) - - # Load default plugins along with user provided --plugins - plugins = Proxy.load_plugins( - [bytes_(p) for p in collections.OrderedDict(default_plugins).keys()] + - [p if isinstance(p, type) else bytes_(p) for p in opts.get( - 'plugins', args.plugins.split(text_(COMMA)))] - ) + if not self.flags.local_executor: + assert self.executors + self.executors.shutdown() + if self.flags.enable_events: + assert self.event_manager is not None + self.event_manager.shutdown() + assert self.listener + self.listener.shutdown() + self._delete_pid_file() + + def _write_pid_file(self) -> None: + if self.flags.pid_file is not None: + # NOTE: Multiple instances of proxy.py running on + # same host machine will currently result in overwriting the PID file + with open(self.flags.pid_file, 'wb') as pid_file: + pid_file.write(bytes_(os.getpid())) - # proxy.py currently cannot serve over HTTPS and perform TLS interception - # at the same time. Check if user is trying to enable both feature - # at the same time. - if (args.cert_file and args.key_file) and \ - (args.ca_key_file and args.ca_cert_file and args.ca_signing_key_file): - print('You can either enable end-to-end encryption OR TLS interception,' - 'not both together.') - sys.exit(1) - - # Generate auth_code required for basic authentication if enabled - auth_code = None - if args.basic_auth: - auth_code = base64.b64encode(bytes_(args.basic_auth)) - - args.plugins = plugins - args.auth_code = cast( - Optional[bytes], - opts.get( - 'auth_code', - auth_code)) - args.server_recvbuf_size = cast( - int, - opts.get( - 'server_recvbuf_size', - args.server_recvbuf_size)) - args.client_recvbuf_size = cast( - int, - opts.get( - 'client_recvbuf_size', - args.client_recvbuf_size)) - args.pac_file = cast( - Optional[str], opts.get( - 'pac_file', bytes_( - args.pac_file))) - args.pac_file_url_path = cast( - Optional[bytes], opts.get( - 'pac_file_url_path', bytes_( - args.pac_file_url_path))) - disabled_headers = cast(Optional[List[bytes]], opts.get('disable_headers', [ - header.lower() for header in bytes_( - args.disable_headers).split(COMMA) if header.strip() != b''])) - args.disable_headers = disabled_headers if disabled_headers is not None else DEFAULT_DISABLE_HEADERS - args.certfile = cast( - Optional[str], opts.get( - 'cert_file', args.cert_file)) - args.keyfile = cast(Optional[str], opts.get('key_file', args.key_file)) - args.ca_key_file = cast( - Optional[str], opts.get( - 'ca_key_file', args.ca_key_file)) - args.ca_cert_file = cast( - Optional[str], opts.get( - 'ca_cert_file', args.ca_cert_file)) - args.ca_signing_key_file = cast( - Optional[str], - opts.get( - 'ca_signing_key_file', - args.ca_signing_key_file)) - args.ca_file = cast( - Optional[str], - opts.get( - 'ca_file', - args.ca_file)) - args.hostname = cast(IpAddress, - opts.get('hostname', ipaddress.ip_address(args.hostname))) - args.family = socket.AF_INET6 if args.hostname.version == 6 else socket.AF_INET - args.port = cast(int, opts.get('port', args.port)) - args.backlog = cast(int, opts.get('backlog', args.backlog)) - num_workers = opts.get('num_workers', args.num_workers) - num_workers = num_workers if num_workers is not None else DEFAULT_NUM_WORKERS - args.num_workers = cast( - int, num_workers if num_workers > 0 else multiprocessing.cpu_count()) - args.static_server_dir = cast( - str, - opts.get( - 'static_server_dir', - args.static_server_dir)) - args.enable_static_server = cast( - bool, - opts.get( - 'enable_static_server', - args.enable_static_server)) - args.devtools_ws_path = cast( - bytes, - opts.get( - 'devtools_ws_path', - getattr(args, 'devtools_ws_path', DEFAULT_DEVTOOLS_WS_PATH))) - args.timeout = cast(int, opts.get('timeout', args.timeout)) - args.threadless = cast(bool, opts.get('threadless', args.threadless)) - args.enable_events = cast( - bool, - opts.get( - 'enable_events', - args.enable_events)) - args.pid_file = cast( - Optional[str], opts.get( - 'pid_file', args.pid_file)) - - args.proxy_py_data_dir = DEFAULT_DATA_DIRECTORY_PATH - os.makedirs(args.proxy_py_data_dir, exist_ok=True) - - ca_cert_dir = opts.get('ca_cert_dir', args.ca_cert_dir) - args.ca_cert_dir = cast(Optional[str], ca_cert_dir) - if args.ca_cert_dir is None: - args.ca_cert_dir = os.path.join( - args.proxy_py_data_dir, 'certificates') - os.makedirs(args.ca_cert_dir, exist_ok=True) - - return args - - @staticmethod - def load_plugins(plugins: List[Union[bytes, type]] - ) -> Dict[bytes, List[type]]: - """Accepts a comma separated list of Python modules and returns - a list of respective Python classes.""" - p: Dict[bytes, List[type]] = { - b'HttpProtocolHandlerPlugin': [], - b'HttpProxyBasePlugin': [], - b'HttpWebServerBasePlugin': [], - b'ProxyDashboardWebsocketPlugin': [] - } - for plugin_ in plugins: - klass, module_name = Proxy.import_plugin(plugin_) - if klass is None and module_name is None: - continue - mro = list(inspect.getmro(klass)) - mro.reverse() - iterator = iter(mro) - while next(iterator) is not abc.ABC: - pass - base_klass = next(iterator) - if klass not in p[bytes_(base_klass.__name__)]: - p[bytes_(base_klass.__name__)].append(klass) - logger.info('Loaded plugin %s.%s', module_name, klass.__name__) - return p - - @staticmethod - def import_plugin(plugin: Union[bytes, type]) -> Any: - if isinstance(plugin, type): - module_name = '__main__' - klass = plugin - else: - plugin_ = text_(plugin.strip()) - if plugin_ == '': - return (None, None) - module_name, klass_name = plugin_.rsplit(text_(DOT), 1) - klass = getattr( - importlib.import_module( - module_name.replace( - os.path.sep, text_(DOT))), - klass_name) - return (klass, module_name) - - @staticmethod - def get_default_plugins( - args: argparse.Namespace) -> List[Tuple[str, bool]]: - # Prepare list of plugins to load based upon - # --enable-*, --disable-* and --basic-auth flags. - default_plugins: List[Tuple[str, bool]] = [] - if args.basic_auth is not None: - default_plugins.append((PLUGIN_PROXY_AUTH, True)) - if args.enable_dashboard: - default_plugins.append((PLUGIN_WEB_SERVER, True)) - args.enable_static_server = True - default_plugins.append((PLUGIN_DASHBOARD, True)) - default_plugins.append((PLUGIN_INSPECT_TRAFFIC, True)) - args.enable_events = True - args.enable_devtools = True - if args.enable_devtools: - default_plugins.append((PLUGIN_DEVTOOLS_PROTOCOL, True)) - default_plugins.append((PLUGIN_WEB_SERVER, True)) - if not args.disable_http_proxy: - default_plugins.append((PLUGIN_HTTP_PROXY, True)) - if args.enable_web_server or \ - args.pac_file is not None or \ - args.enable_static_server: - default_plugins.append((PLUGIN_WEB_SERVER, True)) - if args.pac_file is not None: - default_plugins.append((PLUGIN_PAC_FILE, True)) - return default_plugins - - @staticmethod - def is_py3() -> bool: - """Exists only to avoid mocking sys.version_info in tests.""" - return sys.version_info[0] == 3 - - @staticmethod - def setup_logger( - log_file: Optional[str] = DEFAULT_LOG_FILE, - log_level: str = DEFAULT_LOG_LEVEL, - log_format: str = DEFAULT_LOG_FORMAT) -> None: - ll = getattr( - logging, - {'D': 'DEBUG', - 'I': 'INFO', - 'W': 'WARNING', - 'E': 'ERROR', - 'C': 'CRITICAL'}[log_level.upper()[0]]) - if log_file: - logging.basicConfig( - filename=log_file, - filemode='a', - level=ll, - format=log_format) - else: - logging.basicConfig(level=ll, format=log_format) - - @staticmethod - def set_open_file_limit(soft_limit: int) -> None: - """Configure open file description soft limit on supported OS.""" - if os.name != 'nt': # resource module not available on Windows OS - curr_soft_limit, curr_hard_limit = resource.getrlimit( - resource.RLIMIT_NOFILE) - if curr_soft_limit < soft_limit < curr_hard_limit: - resource.setrlimit( - resource.RLIMIT_NOFILE, (soft_limit, curr_hard_limit)) - logger.debug( - 'Open file soft limit set to %d', soft_limit) - - -@contextlib.contextmanager -def start( - input_args: Optional[List[str]] = None, - **opts: Any) -> Generator[Proxy, None, None]: - """Deprecated. Kept for backward compatibility. - - New users must directly use proxy.Proxy context manager class.""" - try: - with Proxy(input_args, **opts) as p: - yield p - except KeyboardInterrupt: - pass + def _delete_pid_file(self) -> None: + if self.flags.pid_file and os.path.exists(self.flags.pid_file): + os.remove(self.flags.pid_file) -def main( - input_args: Optional[List[str]] = None, - **opts: Any) -> None: +def main(**opts: Any) -> None: try: - with Proxy(input_args=input_args, **opts): - # TODO: Introduce cron feature - # https://github.com/abhinavsingh/proxy.py/issues/392 + with Proxy(sys.argv[1:], **opts): while True: time.sleep(1) except KeyboardInterrupt: @@ -446,4 +229,4 @@ def main( def entry_point() -> None: - main(input_args=sys.argv[1:]) + main() diff --git a/proxy/testing/__init__.py b/proxy/testing/__init__.py index 232621f0b5..e841545b30 100644 --- a/proxy/testing/__init__.py +++ b/proxy/testing/__init__.py @@ -8,3 +8,8 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ +from .test_case import TestCase + +__all__ = [ + 'TestCase', +] diff --git a/proxy/testing/test_case.py b/proxy/testing/test_case.py index 55f4b952fd..d9f47c143f 100644 --- a/proxy/testing/test_case.py +++ b/proxy/testing/test_case.py @@ -15,64 +15,66 @@ from ..proxy import Proxy from ..common.constants import DEFAULT_TIMEOUT -from ..common.utils import get_available_port, new_socket_connection +from ..common.utils import new_socket_connection from ..plugin import CacheResponsesPlugin class TestCase(unittest.TestCase): - """Base TestCase class that automatically setup and teardown proxy.py.""" + """Base TestCase class that automatically setup and tear down proxy.py.""" DEFAULT_PROXY_PY_STARTUP_FLAGS = [ '--num-workers', '1', + '--num-acceptors', '1', '--threadless', ] - PROXY_PORT: int = 8899 PROXY: Optional[Proxy] = None INPUT_ARGS: Optional[List[str]] = None @classmethod def setUpClass(cls) -> None: - cls.PROXY_PORT = get_available_port() - cls.INPUT_ARGS = getattr(cls, 'PROXY_PY_STARTUP_FLAGS') \ if hasattr(cls, 'PROXY_PY_STARTUP_FLAGS') \ else cls.DEFAULT_PROXY_PY_STARTUP_FLAGS cls.INPUT_ARGS.append('--hostname') cls.INPUT_ARGS.append('0.0.0.0') cls.INPUT_ARGS.append('--port') - cls.INPUT_ARGS.append(str(cls.PROXY_PORT)) + cls.INPUT_ARGS.append('0') - cls.PROXY = Proxy(input_args=cls.INPUT_ARGS) + cls.PROXY = Proxy(cls.INPUT_ARGS) cls.PROXY.flags.plugins[b'HttpProxyBasePlugin'].append( - CacheResponsesPlugin) + CacheResponsesPlugin, + ) cls.PROXY.__enter__() - cls.wait_for_server(cls.PROXY_PORT) + assert cls.PROXY.acceptors + cls.wait_for_server(cls.PROXY.acceptors.flags.port) @staticmethod - def wait_for_server(proxy_port: int, - wait_for_seconds: int = DEFAULT_TIMEOUT) -> None: + def wait_for_server( + proxy_port: int, + wait_for_seconds: float = DEFAULT_TIMEOUT, + ) -> None: """Wait for proxy.py server to come up.""" start_time = time.time() while True: try: - conn = new_socket_connection( - ('localhost', proxy_port)) - conn.close() + new_socket_connection( + ('localhost', proxy_port), + ).close() break except ConnectionRefusedError: time.sleep(0.1) if time.time() - start_time > wait_for_seconds: raise TimeoutError( - 'Timed out while waiting for proxy.py to start...') + 'Timed out while waiting for proxy.py to start...', + ) @classmethod def tearDownClass(cls) -> None: assert cls.PROXY cls.PROXY.__exit__(None, None, None) - cls.PROXY_PORT = 8899 cls.PROXY = None cls.INPUT_ARGS = None diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..6ac5005a09 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,13 @@ +[build-system] +requires = [ + # Essentials + "setuptools", + + # Plugins + "setuptools-scm[toml] >= 6", + "setuptools-scm-git-archive >= 1.1", +] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] +write_to = "proxy/common/_scm_version.py" diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000000..bc8ff5be0f --- /dev/null +++ b/pytest.ini @@ -0,0 +1,88 @@ +[pytest] +addopts = + # FIXME: Enable this once the test suite has no race conditions + # `pytest-xdist`: + # --numprocesses=auto + + # Show 10 slowest invocations: + --durations=10 + + # A bit of verbosity doesn't hurt: + -v + + # Report all the things == -rxXs: + -ra + + # Show values of the local vars in errors/tracebacks: + --showlocals + + # Autocollect and invoke the doctests from all modules: + # https://docs.pytest.org/en/stable/doctest.html + --doctest-modules + + # Dump the test results in junit format: + --junitxml=.tox/tmp/test-results/pytest/results.xml + + # Fail on config parsing warnings: + --strict-config + + # Fail on non-existing markers: + # * Deprecated since v6.2.0 but may be reintroduced later covering a + # broader scope: + # --strict + # * Exists since v4.5.0 (advised to be used instead of `--strict`): + --strict-markers + + # `pytest-cov`: + # `pytest-cov`, "-p" preloads the module early: + -p pytest_cov + --no-cov-on-fail + --cov=proxy + --cov=tests/ + --cov-branch + --cov-report=term-missing:skip-covered + --cov-report=html:.tox/tmp/test-results/pytest/cov/ + --cov-report=xml + --cov-context=test + --cov-config=.coveragerc + +doctest_optionflags = ALLOW_UNICODE ELLIPSIS + +# Marks tests with an empty parameterset as xfail(run=False) +empty_parameter_set_mark = xfail + +faulthandler_timeout = 30 + +filterwarnings = + error + +junit_duration_report = call +# xunit1 contains more metadata than xunit2 so it's better for CI UIs: +junit_family = xunit1 +junit_logging = all +junit_log_passing_tests = true +junit_suite_name = proxy_py_test_suite + +# A mapping of markers to their descriptions allowed in strict mode: +markers = + smoke: Quick self-check smoke tests + +minversion = 6.2.0 + +# Optimize pytest's lookup by restricting potentially deep dir tree scan: +norecursedirs = + build + dist + docs + examples + proxy.egg-info + .cache + .eggs + .git + .github + .tox + *.egg + +testpaths = tests/ + +xfail_strict = true diff --git a/requirements-release.txt b/requirements-release.txt index 43e86b68f2..ae95ad7991 100644 --- a/requirements-release.txt +++ b/requirements-release.txt @@ -1,2 +1,2 @@ -twine==3.3.0 -wheel==0.36.2 +setuptools-scm == 6.3.2 +twine==3.5.0 diff --git a/requirements-testing.txt b/requirements-testing.txt index af0e4b9dcf..5a8b8a0331 100644 --- a/requirements-testing.txt +++ b/requirements-testing.txt @@ -1,13 +1,17 @@ +wheel==0.37.0 python-coveralls==2.9.3 -coverage==5.4 -flake8==3.8.4 -pytest==6.2.2 -pytest-cov==2.11.1 -autopep8==1.5.4 -mypy==0.790 -py-spy==0.3.3 -codecov==2.1.11 -tox==3.21.3 +coverage==6.1.2 +flake8==4.0.1 +pytest==6.2.5 +pytest-cov==3.0.0 +pytest-xdist == 2.4.0 +pytest-mock==3.6.1 +pytest-asyncio==0.16.0 +autopep8==1.6.0 +mypy==0.910 +py-spy==0.3.10 +codecov==2.1.12 +tox==3.24.4 mccabe==0.6.1 -pylint==2.6.0 -rope==0.18.0 +pylint==2.11.1 +rope==0.22.0 diff --git a/requirements-tunnel.txt b/requirements-tunnel.txt index 58f39bb285..b247379dba 100644 --- a/requirements-tunnel.txt +++ b/requirements-tunnel.txt @@ -1 +1,2 @@ -paramiko==2.7.2 +paramiko==2.8.0 +types-paramiko==2.8.1 diff --git a/requirements.txt b/requirements.txt index 9ae66d0278..50d4894952 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -typing-extensions==3.7.4.3 +typing-extensions; python_version < "3.8" diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000000..fe1fcf3b08 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,122 @@ +[metadata] +name = proxy.py +version = attr: proxy.common.version.__version__ +url = https://github.com/abhinavsingh/proxy.py +project_urls = + Container Image: DockerHub = https://hub.docker.com/r/abhinavsingh/proxy.py + Chat: Gitter = https://gitter.im/proxy.py/community + Coverage: Codecov = https://codecov.io/github/abhinavsingh/proxy.py + CI: GitHub = https://github.com/abhinavsingh/proxy.py/actions + Docs: Changelog = https://proxypy.rtfd.io/en/latest/#changelog + Docs: RTD = https://proxypy.rtfd.io/en/latest/ + Docs: Intro = https://abhinavsingh.com/proxy-py-a-lightweight-single-file-http-proxy-server-in-python/ + GitHub: discussions = https://github.com/abhinavsingh/proxy.py/discussions + GitHub: issues = https://github.com/abhinavsingh/proxy.py/issues + GitHub: repo = https://github.com/abhinavsingh/proxy.py +download_url = https://github.com/abhinavsingh/proxy.py/archive/master.zip +description = โšก Fast โ€ข ๐Ÿชถ Lightweight โ€ข 0๏ธโƒฃ Dependency โ€ข ๐Ÿ”Œ Pluggable โ€ข ๐Ÿ˜ˆ TLS interception โ€ข ๐Ÿ”’ DNS-over-HTTPS โ€ข ๐Ÿ”ฅ Poor Mans VPN โ€ข โช Reverse & โฉ Forward โ€ข ๐Ÿ‘ฎ๐Ÿฟ Proxy Server framework โ€ข ๐ŸŒ Web Server framework โ€ข โžต โžถ โžท โž  PubSub framework โ€ข ๐Ÿ‘ท Work acceptor & executor framework. +long_description = file: README.md +long_description_content_type = text/markdown +author = Abhinav Singh +author_email = mailsforabhinav+proxy@gmail.com +license = 'BSD' +license_files = + LICENSE.md +classifiers = + Development Status :: 5 - Production/Stable + + Environment :: Console + Environment :: No Input/Output (Daemon) + Environment :: Web Environment + Environment :: MacOS X + Environment :: Plugins + Environment :: Win32 (MS Windows) + + Framework :: Robot Framework + Framework :: Robot Framework :: Library + + Intended Audience :: Developers + Intended Audience :: Education + Intended Audience :: End Users/Desktop + Intended Audience :: System Administrators + Intended Audience :: Science/Research + + License :: OSI Approved :: BSD License + + Natural Language :: English + + Operating System :: MacOS + Operating System :: MacOS :: MacOS 9 + Operating System :: MacOS :: MacOS X + Operating System :: POSIX + Operating System :: POSIX :: Linux + Operating System :: Unix + Operating System :: Microsoft + Operating System :: Microsoft :: Windows + Operating System :: Microsoft :: Windows :: Windows 10 + Operating System :: Android + Operating System :: OS Independent + + Programming Language :: Python :: Implementation + Programming Language :: Python :: 3 :: Only + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + + Topic :: Internet + Topic :: Internet :: Proxy Servers + Topic :: Internet :: WWW/HTTP + Topic :: Internet :: WWW/HTTP :: Browsers + Topic :: Internet :: WWW/HTTP :: Dynamic Content + Topic :: Internet :: WWW/HTTP :: Dynamic Content :: CGI Tools/Libraries + Topic :: Internet :: WWW/HTTP :: HTTP Servers + + Topic :: Scientific/Engineering :: Information Analysis + + Topic :: Software Development :: Debuggers + Topic :: Software Development :: Libraries :: Python Modules + + Topic :: System :: Monitoring + Topic :: System :: Networking + Topic :: System :: Networking :: Firewalls + Topic :: System :: Networking :: Monitoring + + Topic :: Utilities + + Typing :: Typed +keywords = + http + proxy + http proxy server + proxy server + http server + http web server + proxy framework + web framework + Python3 + +[options] +python_requires = >= 3.6 +packages = find: +include_package_data = True +zip_safe = False + +# These are required in actual runtime: +install_requires = + typing-extensions; python_version < "3.8" + +[options.entry_points] +console_scripts = + proxy = proxy:entry_point + +[options.package_data] +proxy = + py.typed + +[options.packages.find] +exclude = + tests + tests.* diff --git a/setup.py b/setup.py deleted file mode 100644 index 19f6f7b1d0..0000000000 --- a/setup.py +++ /dev/null @@ -1,102 +0,0 @@ -# -*- coding: utf-8 -*- -""" - proxy.py - ~~~~~~~~ - โšกโšกโšก Fast, Lightweight, Programmable, TLS interception capable - proxy server for Application debugging, testing and development. - - :copyright: (c) 2013-present by Abhinav Singh and contributors. - :license: BSD, see LICENSE for more details. -""" -from setuptools import setup, find_packages - -VERSION = (2, 3, 1) -__version__ = '.'.join(map(str, VERSION[0:3])) -__description__ = '''โšกโšกโšกFast, Lightweight, Pluggable, TLS interception capable proxy server - focused on Network monitoring, controls & Application development, testing, debugging.''' -__author__ = 'Abhinav Singh' -__author_email__ = 'mailsforabhinav@gmail.com' -__homepage__ = 'https://github.com/abhinavsingh/proxy.py' -__download_url__ = '%s/archive/master.zip' % __homepage__ -__license__ = 'BSD' - -if __name__ == '__main__': - setup( - name='proxy.py', - version=__version__, - author=__author__, - author_email=__author_email__, - url=__homepage__, - description=__description__, - long_description=open( - 'README.md', 'r', encoding='utf-8').read().strip(), - long_description_content_type='text/markdown', - download_url=__download_url__, - license=__license__, - python_requires='!=2.*, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*', - zip_safe=False, - packages=find_packages(exclude=['tests', 'tests.*']), - package_data={'proxy': ['py.typed']}, - install_requires=open('requirements.txt', 'r').read().strip().split(), - entry_points={ - 'console_scripts': [ - 'proxy = proxy:entry_point' - ] - }, - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Environment :: Console', - 'Environment :: No Input/Output (Daemon)', - 'Environment :: Web Environment', - 'Environment :: MacOS X', - 'Environment :: Plugins', - 'Environment :: Win32 (MS Windows)', - 'Framework :: Robot Framework', - 'Framework :: Robot Framework :: Library', - 'Intended Audience :: Developers', - 'Intended Audience :: Education', - 'Intended Audience :: End Users/Desktop', - 'Intended Audience :: System Administrators', - 'Intended Audience :: Science/Research', - 'License :: OSI Approved :: BSD License', - 'Natural Language :: English', - 'Operating System :: MacOS', - 'Operating System :: MacOS :: MacOS 9', - 'Operating System :: MacOS :: MacOS X', - 'Operating System :: POSIX', - 'Operating System :: POSIX :: Linux', - 'Operating System :: Unix', - 'Operating System :: Microsoft', - 'Operating System :: Microsoft :: Windows', - 'Operating System :: Microsoft :: Windows :: Windows 10', - 'Operating System :: Android', - 'Operating System :: OS Independent', - 'Programming Language :: Python :: Implementation', - 'Programming Language :: Python :: 3 :: Only', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Topic :: Internet', - 'Topic :: Internet :: Proxy Servers', - 'Topic :: Internet :: WWW/HTTP', - 'Topic :: Internet :: WWW/HTTP :: Browsers', - 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', - 'Topic :: Internet :: WWW/HTTP :: Dynamic Content :: CGI Tools/Libraries', - 'Topic :: Internet :: WWW/HTTP :: HTTP Servers', - 'Topic :: Scientific/Engineering :: Information Analysis', - 'Topic :: Software Development :: Debuggers', - 'Topic :: Software Development :: Libraries :: Python Modules', - 'Topic :: System :: Monitoring', - 'Topic :: System :: Networking', - 'Topic :: System :: Networking :: Firewalls', - 'Topic :: System :: Networking :: Monitoring', - 'Topic :: Utilities', - 'Typing :: Typed', - ], - keywords=( - 'http, proxy, http proxy server, proxy server, http server,' - 'http web server, proxy framework, web framework, Python3' - ) - ) diff --git a/shortlink.gif b/shortlink.gif index 433f2d8c29..8371412f1e 100644 Binary files a/shortlink.gif and b/shortlink.gif differ diff --git a/tests/common/test_flags.py b/tests/common/test_flags.py index c36f8e6355..a8791d5ce4 100644 --- a/tests/common/test_flags.py +++ b/tests/common/test_flags.py @@ -8,16 +8,18 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from proxy.common.utils import bytes_ -from proxy.common.constants import PLUGIN_HTTP_PROXY import unittest +from unittest import mock from typing import List, Dict -from proxy.proxy import Proxy from proxy.http.proxy import HttpProxyPlugin from proxy.plugin import CacheResponsesPlugin from proxy.plugin import FilterByUpstreamHostPlugin +from proxy.common.utils import bytes_ +from proxy.common.flag import FlagParser +from proxy.common.version import __version__ +from proxy.common.constants import PLUGIN_HTTP_PROXY, PY2_DEPRECATION_MESSAGE class TestFlags(unittest.TestCase): @@ -27,88 +29,154 @@ def assert_plugins(self, expected: Dict[str, List[type]]) -> None: for p in expected[k]: self.assertIn(p, self.flags.plugins[k.encode()]) self.assertEqual( - len([o for o in self.flags.plugins[k.encode()] if o == p]), 1) + len([o for o in self.flags.plugins[k.encode()] if o == p]), 1, + ) def test_load_plugin_from_bytes(self) -> None: - self.flags = Proxy.initialize([], plugins=[ - b'proxy.plugin.CacheResponsesPlugin', - ]) + self.flags = FlagParser.initialize( + [], plugins=[ + b'proxy.plugin.CacheResponsesPlugin', + ], + ) self.assert_plugins({'HttpProxyBasePlugin': [CacheResponsesPlugin]}) def test_load_plugins_from_bytes(self) -> None: - self.flags = Proxy.initialize([], plugins=[ - b'proxy.plugin.CacheResponsesPlugin', - b'proxy.plugin.FilterByUpstreamHostPlugin', - ]) - self.assert_plugins({'HttpProxyBasePlugin': [ - CacheResponsesPlugin, - FilterByUpstreamHostPlugin, - ]}) + self.flags = FlagParser.initialize( + [], plugins=[ + b'proxy.plugin.CacheResponsesPlugin', + b'proxy.plugin.FilterByUpstreamHostPlugin', + ], + ) + self.assert_plugins({ + 'HttpProxyBasePlugin': [ + CacheResponsesPlugin, + FilterByUpstreamHostPlugin, + ], + }) def test_load_plugin_from_args(self) -> None: - self.flags = Proxy.initialize([ + self.flags = FlagParser.initialize([ '--plugins', 'proxy.plugin.CacheResponsesPlugin', ]) self.assert_plugins({'HttpProxyBasePlugin': [CacheResponsesPlugin]}) def test_load_plugins_from_args(self) -> None: - self.flags = Proxy.initialize([ + self.flags = FlagParser.initialize([ '--plugins', 'proxy.plugin.CacheResponsesPlugin,proxy.plugin.FilterByUpstreamHostPlugin', ]) - self.assert_plugins({'HttpProxyBasePlugin': [ - CacheResponsesPlugin, - FilterByUpstreamHostPlugin, - ]}) + self.assert_plugins({ + 'HttpProxyBasePlugin': [ + CacheResponsesPlugin, + FilterByUpstreamHostPlugin, + ], + }) def test_load_plugin_from_class(self) -> None: - self.flags = Proxy.initialize([], plugins=[ - CacheResponsesPlugin, - ]) + self.flags = FlagParser.initialize( + [], plugins=[ + CacheResponsesPlugin, + ], + ) self.assert_plugins({'HttpProxyBasePlugin': [CacheResponsesPlugin]}) def test_load_plugins_from_class(self) -> None: - self.flags = Proxy.initialize([], plugins=[ - CacheResponsesPlugin, - FilterByUpstreamHostPlugin, - ]) - self.assert_plugins({'HttpProxyBasePlugin': [ - CacheResponsesPlugin, - FilterByUpstreamHostPlugin, - ]}) + self.flags = FlagParser.initialize( + [], plugins=[ + CacheResponsesPlugin, + FilterByUpstreamHostPlugin, + ], + ) + self.assert_plugins({ + 'HttpProxyBasePlugin': [ + CacheResponsesPlugin, + FilterByUpstreamHostPlugin, + ], + }) def test_load_plugins_from_bytes_and_class(self) -> None: - self.flags = Proxy.initialize([], plugins=[ - CacheResponsesPlugin, - b'proxy.plugin.FilterByUpstreamHostPlugin', - ]) - self.assert_plugins({'HttpProxyBasePlugin': [ - CacheResponsesPlugin, - FilterByUpstreamHostPlugin, - ]}) + self.flags = FlagParser.initialize( + [], plugins=[ + CacheResponsesPlugin, + b'proxy.plugin.FilterByUpstreamHostPlugin', + ], + ) + self.assert_plugins({ + 'HttpProxyBasePlugin': [ + CacheResponsesPlugin, + FilterByUpstreamHostPlugin, + ], + }) def test_unique_plugin_from_bytes(self) -> None: - self.flags = Proxy.initialize([], plugins=[ - bytes_(PLUGIN_HTTP_PROXY), - ]) - self.assert_plugins({'HttpProtocolHandlerPlugin': [ - HttpProxyPlugin, - ]}) + self.flags = FlagParser.initialize( + [], plugins=[ + bytes_(PLUGIN_HTTP_PROXY), + ], + ) + self.assert_plugins({ + 'HttpProtocolHandlerPlugin': [ + HttpProxyPlugin, + ], + }) def test_unique_plugin_from_args(self) -> None: - self.flags = Proxy.initialize([ + self.flags = FlagParser.initialize([ '--plugins', PLUGIN_HTTP_PROXY, ]) - self.assert_plugins({'HttpProtocolHandlerPlugin': [ - HttpProxyPlugin, - ]}) + self.assert_plugins({ + 'HttpProtocolHandlerPlugin': [ + HttpProxyPlugin, + ], + }) def test_unique_plugin_from_class(self) -> None: - self.flags = Proxy.initialize([], plugins=[ - HttpProxyPlugin, - ]) - self.assert_plugins({'HttpProtocolHandlerPlugin': [ - HttpProxyPlugin, - ]}) + self.flags = FlagParser.initialize( + [], plugins=[ + HttpProxyPlugin, + ], + ) + self.assert_plugins({ + 'HttpProtocolHandlerPlugin': [ + HttpProxyPlugin, + ], + }) + + def test_basic_auth_flag_is_base64_encoded(self) -> None: + flags = FlagParser.initialize(['--basic-auth', 'user:pass']) + self.assertEqual(flags.auth_code, b'dXNlcjpwYXNz') + + @mock.patch('builtins.print') + def test_main_version(self, mock_print: mock.Mock) -> None: + with self.assertRaises(SystemExit) as e: + FlagParser.initialize(['--version']) + mock_print.assert_called_with(__version__) + self.assertEqual(e.exception.code, 0) + + @mock.patch('builtins.print') + @mock.patch('proxy.common.flag.is_py2') + def test_main_py2_exit( + self, + mock_is_py2: mock.Mock, + mock_print: mock.Mock, + ) -> None: + mock_is_py2.return_value = True + with self.assertRaises(SystemExit) as e: + FlagParser.initialize() + mock_print.assert_called_with(PY2_DEPRECATION_MESSAGE) + self.assertEqual(e.exception.code, 1) + mock_is_py2.assert_called() + + @mock.patch('builtins.print') + @mock.patch('proxy.common.flag.is_py2') + def test_main_py3_runs( + self, + mock_is_py2: mock.Mock, + mock_print: mock.Mock, + ) -> None: + mock_is_py2.return_value = False + FlagParser.initialize() + mock_is_py2.assert_called() + mock_print.assert_not_called() if __name__ == '__main__': diff --git a/tests/common/test_pki.py b/tests/common/test_pki.py index e55c063795..1abcb1c625 100644 --- a/tests/common/test_pki.py +++ b/tests/common/test_pki.py @@ -20,6 +20,10 @@ class TestPki(unittest.TestCase): + def setUp(self) -> None: + self._tempdir = tempfile.gettempdir() + return super().setUp() + @mock.patch('subprocess.Popen') def test_run_openssl_command(self, mock_popen: mock.Mock) -> None: command = ['my', 'custom', 'command'] @@ -28,7 +32,8 @@ def test_run_openssl_command(self, mock_popen: mock.Mock) -> None: mock_popen.assert_called_with( command, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + stderr=subprocess.PIPE, + ) def test_get_ext_config(self) -> None: self.assertEqual(pki.get_ext_config(None, None), b'') @@ -36,17 +41,25 @@ def test_get_ext_config(self) -> None: self.assertEqual( pki.get_ext_config( ['proxy.py'], - None), - b'\nsubjectAltName=DNS:proxy.py') + None, + ), + b'\nsubjectAltName=DNS:proxy.py', + ) self.assertEqual( pki.get_ext_config( None, - 'serverAuth'), - b'\nextendedKeyUsage=serverAuth') - self.assertEqual(pki.get_ext_config(['proxy.py'], 'serverAuth'), - b'\nsubjectAltName=DNS:proxy.py\nextendedKeyUsage=serverAuth') - self.assertEqual(pki.get_ext_config(['proxy.py', 'www.proxy.py'], 'serverAuth'), - b'\nsubjectAltName=DNS:proxy.py,DNS:www.proxy.py\nextendedKeyUsage=serverAuth') + 'serverAuth', + ), + b'\nextendedKeyUsage=serverAuth', + ) + self.assertEqual( + pki.get_ext_config(['proxy.py'], 'serverAuth'), + b'\nsubjectAltName=DNS:proxy.py\nextendedKeyUsage=serverAuth', + ) + self.assertEqual( + pki.get_ext_config(['proxy.py', 'www.proxy.py'], 'serverAuth'), + b'\nsubjectAltName=DNS:proxy.py,DNS:www.proxy.py\nextendedKeyUsage=serverAuth', + ) def test_ssl_config_no_ext(self) -> None: with pki.ssl_config() as (config_path, has_extension): @@ -61,7 +74,8 @@ def test_ssl_config(self) -> None: self.assertEqual( config.read(), pki.DEFAULT_CONFIG + - b'\n[PROXY]\nsubjectAltName=DNS:proxy.py') + b'\n[PROXY]\nsubjectAltName=DNS:proxy.py', + ) def test_extfile_no_ext(self) -> None: with pki.ext_file() as config_path: @@ -73,7 +87,8 @@ def test_extfile(self) -> None: with open(config_path, 'rb') as config: self.assertEqual( config.read(), - b'\nsubjectAltName=DNS:proxy.py') + b'\nsubjectAltName=DNS:proxy.py', + ) def test_gen_private_key(self) -> None: key_path, nopass_key_path = self._gen_private_key() @@ -92,7 +107,7 @@ def test_gen_public_key(self) -> None: def test_gen_csr(self) -> None: key_path, nopass_key_path, crt_path = self._gen_public_private_key() - csr_path = os.path.join(tempfile.gettempdir(), 'test_gen_public.csr') + csr_path = os.path.join(self._tempdir, 'test_gen_public.csr') pki.gen_csr(csr_path, key_path, 'password', crt_path) self.assertTrue(os.path.exists(csr_path)) # TODO: Assert CSR is valid for provided crt and key @@ -106,15 +121,16 @@ def test_sign_csr(self) -> None: def _gen_public_private_key(self) -> Tuple[str, str, str]: key_path, nopass_key_path = self._gen_private_key() - crt_path = os.path.join(tempfile.gettempdir(), 'test_gen_public.crt') + crt_path = os.path.join(self._tempdir, 'test_gen_public.crt') pki.gen_public_key(crt_path, key_path, 'password', '/CN=example.com') return (key_path, nopass_key_path, crt_path) def _gen_private_key(self) -> Tuple[str, str]: - key_path = os.path.join(tempfile.gettempdir(), 'test_gen_private.key') + key_path = os.path.join(self._tempdir, 'test_gen_private.key') nopass_key_path = os.path.join( - tempfile.gettempdir(), - 'test_gen_private_nopass.key') + self._tempdir, + 'test_gen_private_nopass.key', + ) pki.gen_private_key(key_path, 'password') pki.remove_passphrase(key_path, 'password', nopass_key_path) return (key_path, nopass_key_path) diff --git a/tests/common/test_utils.py b/tests/common/test_utils.py index b75b2b569c..6d6217a641 100644 --- a/tests/common/test_utils.py +++ b/tests/common/test_utils.py @@ -37,12 +37,15 @@ def test_new_socket_connection_ipv6(self, mock_socket: mock.Mock) -> None: mock_socket.assert_called_with(socket.AF_INET6, socket.SOCK_STREAM, 0) self.assertEqual(conn, mock_socket.return_value) mock_socket.return_value.connect.assert_called_with( - (self.addr_ipv6[0], self.addr_ipv6[1], 0, 0)) + (self.addr_ipv6[0], self.addr_ipv6[1], 0, 0), + ) @mock.patch('socket.create_connection') def test_new_socket_connection_dual(self, mock_socket: mock.Mock) -> None: conn = new_socket_connection(self.addr_dual) - mock_socket.assert_called_with(self.addr_dual, timeout=DEFAULT_TIMEOUT) + mock_socket.assert_called_with( + self.addr_dual, timeout=DEFAULT_TIMEOUT, source_address=None, + ) self.assertEqual(conn, mock_socket.return_value) @mock.patch('proxy.common.utils.new_socket_connection') @@ -54,6 +57,7 @@ def dummy(conn: socket.socket) -> None: @mock.patch('proxy.common.utils.new_socket_connection') def test_context_manager( - self, mock_new_socket_connection: mock.Mock) -> None: + self, mock_new_socket_connection: mock.Mock, + ) -> None: with socket_connection(self.addr_ipv4) as conn: self.assertEqual(conn, mock_new_socket_connection.return_value) diff --git a/tests/core/test_acceptor.py b/tests/core/test_acceptor.py index aae3de5caf..a923a97357 100644 --- a/tests/core/test_acceptor.py +++ b/tests/core/test_acceptor.py @@ -15,22 +15,27 @@ from unittest import mock from proxy.core.acceptor import Acceptor -from proxy.proxy import Proxy +from proxy.common.flag import FlagParser class TestAcceptor(unittest.TestCase): def setUp(self) -> None: self.acceptor_id = 1 - self.mock_protocol_handler = mock.MagicMock() self.pipe = multiprocessing.Pipe() - self.flags = Proxy.initialize() + self.flags = FlagParser.initialize( + threaded=True, + work_klass=mock.MagicMock(), + ) self.acceptor = Acceptor( idd=self.acceptor_id, - work_queue=self.pipe[1], + fd_queue=self.pipe[1], flags=self.flags, lock=multiprocessing.Lock(), - work_klass=self.mock_protocol_handler) + executor_queues=[], + executor_pids=[], + executor_locks=[], + ) @mock.patch('selectors.DefaultSelector') @mock.patch('socket.fromfd') @@ -39,7 +44,8 @@ def test_continues_when_no_events( self, mock_recv_handle: mock.Mock, mock_fromfd: mock.Mock, - mock_selector: mock.Mock) -> None: + mock_selector: mock.Mock, + ) -> None: fileno = 10 conn = mock.MagicMock() addr = mock.MagicMock() @@ -53,9 +59,9 @@ def test_continues_when_no_events( self.acceptor.run() sock.accept.assert_not_called() - self.mock_protocol_handler.assert_not_called() + self.flags.work_klass.assert_not_called() - @mock.patch('proxy.core.acceptor.acceptor.TcpClientConnection') + @mock.patch('proxy.core.acceptor.executors.TcpClientConnection') @mock.patch('threading.Thread') @mock.patch('selectors.DefaultSelector') @mock.patch('socket.fromfd') @@ -66,7 +72,8 @@ def test_accepts_client_from_server_socket( mock_fromfd: mock.Mock, mock_selector: mock.Mock, mock_thread: mock.Mock, - mock_client: mock.Mock) -> None: + mock_client: mock.Mock, + ) -> None: fileno = 10 conn = mock.MagicMock() addr = mock.MagicMock() @@ -77,7 +84,7 @@ def test_accepts_client_from_server_socket( mock_thread.return_value.start.side_effect = KeyboardInterrupt() selector = mock_selector.return_value - selector.select.return_value = [(None, None)] + selector.select.return_value = [(None, selectors.EVENT_READ)] self.acceptor.run() @@ -87,14 +94,15 @@ def test_accepts_client_from_server_socket( mock_fromfd.assert_called_with( fileno, family=socket.AF_INET6, - type=socket.SOCK_STREAM + type=socket.SOCK_STREAM, ) - self.mock_protocol_handler.assert_called_with( + self.flags.work_klass.assert_called_with( mock_client.return_value, flags=self.flags, event_queue=None, ) mock_thread.assert_called_with( - target=self.mock_protocol_handler.return_value.run) + target=self.flags.work_klass.return_value.run, + ) mock_thread.return_value.start.assert_called() sock.close.assert_called() diff --git a/tests/core/test_acceptor_pool.py b/tests/core/test_acceptor_pool.py index 3142e39bf9..5241bd4341 100644 --- a/tests/core/test_acceptor_pool.py +++ b/tests/core/test_acceptor_pool.py @@ -9,10 +9,10 @@ :license: BSD, see LICENSE for more details. """ import unittest -import socket + from unittest import mock -from proxy.proxy import Proxy +from proxy.common.flag import FlagParser from proxy.core.acceptor import AcceptorPool @@ -20,48 +20,62 @@ class TestAcceptorPool(unittest.TestCase): @mock.patch('proxy.core.acceptor.pool.send_handle') @mock.patch('multiprocessing.Pipe') - @mock.patch('socket.socket') @mock.patch('proxy.core.acceptor.pool.Acceptor') + @mock.patch('proxy.core.acceptor.Listener') def test_setup_and_shutdown( self, + mock_listener: mock.Mock, mock_acceptor: mock.Mock, - mock_socket: mock.Mock, mock_pipe: mock.Mock, - mock_send_handle: mock.Mock) -> None: + mock_send_handle: mock.Mock, + ) -> None: acceptor1 = mock.MagicMock() acceptor2 = mock.MagicMock() mock_acceptor.side_effect = [acceptor1, acceptor2] - num_workers = 2 - sock = mock_socket.return_value - work_klass = mock.MagicMock() - flags = Proxy.initialize(num_workers=2) + num_acceptors = 2 + flags = FlagParser.initialize( + num_acceptors=num_acceptors, + threaded=True, + ) + self.assertEqual(flags.num_acceptors, num_acceptors) - pool = AcceptorPool(flags=flags, work_klass=work_klass) + pool = AcceptorPool( + flags=flags, listener=mock_listener.return_value, + executor_queues=[], executor_pids=[], executor_locks=[], + ) pool.setup() + + self.assertEqual(mock_pipe.call_count, num_acceptors) + self.assertEqual(mock_acceptor.call_count, num_acceptors) mock_send_handle.assert_called() + self.assertEqual(mock_send_handle.call_count, num_acceptors) - work_klass.assert_not_called() - mock_socket.assert_called_with( - socket.AF_INET6 if pool.flags.hostname.version == 6 else socket.AF_INET, - socket.SOCK_STREAM + self.assertEqual( + mock_acceptor.call_args_list[0][1]['idd'], 0, + ) + self.assertEqual( + mock_acceptor.call_args_list[0][1]['fd_queue'], mock_pipe.return_value[1], + ) + self.assertEqual( + mock_acceptor.call_args_list[0][1]['flags'], flags, + ) + self.assertEqual( + mock_acceptor.call_args_list[0][1]['event_queue'], None, + ) + # executor_queues=[], + # executor_pids=[] + self.assertEqual( + mock_acceptor.call_args_list[1][1]['idd'], 1, ) - sock.setsockopt.assert_called_with( - socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind.assert_called_with( - (str(pool.flags.hostname), pool.flags.port)) - sock.listen.assert_called_with(pool.flags.backlog) - sock.setblocking.assert_called_with(False) - self.assertTrue(mock_pipe.call_count, num_workers) - self.assertTrue(mock_acceptor.call_count, num_workers) - acceptor1.start.assert_called() - acceptor2.start.assert_called() + acceptor1.start.assert_called_once() + acceptor2.start.assert_called_once() + mock_listener.return_value.fileno.assert_called_once() + acceptor1.join.assert_not_called() acceptor2.join.assert_not_called() - sock.close.assert_called() - pool.shutdown() - acceptor1.join.assert_called() - acceptor2.join.assert_called() + acceptor1.join.assert_called_once() + acceptor2.join.assert_called_once() diff --git a/tests/core/test_conn_pool.py b/tests/core/test_conn_pool.py new file mode 100644 index 0000000000..3eaad052f3 --- /dev/null +++ b/tests/core/test_conn_pool.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import unittest + +from unittest import mock + +from proxy.core.connection import ConnectionPool + + +class TestConnectionPool(unittest.TestCase): + + @mock.patch('proxy.core.connection.pool.TcpServerConnection') + def test_acquire_and_release_and_reacquire(self, mock_tcp_server_connection: mock.Mock) -> None: + pool = ConnectionPool() + addr = ('localhost', 1234) + # Mock + mock_conn = mock_tcp_server_connection.return_value + mock_conn.is_reusable.side_effect = [ + False, True, True, + ] + mock_conn.closed = False + # Acquire + created, conn = pool.acquire(*addr) + self.assertTrue(created) + mock_tcp_server_connection.assert_called_once_with(*addr) + self.assertEqual(conn, mock_conn) + self.assertEqual(len(pool.pools[addr]), 1) + self.assertTrue(conn in pool.pools[addr]) + # Release (connection must be retained because not closed) + pool.release(conn) + self.assertEqual(len(pool.pools[addr]), 1) + self.assertTrue(conn in pool.pools[addr]) + # Reacquire + created, conn = pool.acquire(*addr) + self.assertFalse(created) + mock_conn.reset.assert_called_once() + self.assertEqual(conn, mock_conn) + self.assertEqual(len(pool.pools[addr]), 1) + self.assertTrue(conn in pool.pools[addr]) + + @mock.patch('proxy.core.connection.pool.TcpServerConnection') + def test_closed_connections_are_removed_on_release( + self, mock_tcp_server_connection: mock.Mock, + ) -> None: + pool = ConnectionPool() + addr = ('localhost', 1234) + # Mock + mock_conn = mock_tcp_server_connection.return_value + mock_conn.closed = True + mock_conn.addr = addr + # Acquire + created, conn = pool.acquire(*addr) + self.assertTrue(created) + mock_tcp_server_connection.assert_called_once_with(*addr) + self.assertEqual(conn, mock_conn) + self.assertEqual(len(pool.pools[addr]), 1) + self.assertTrue(conn in pool.pools[addr]) + # Release + pool.release(conn) + self.assertEqual(len(pool.pools[addr]), 0) + # Acquire + created, conn = pool.acquire(*addr) + self.assertTrue(created) + self.assertEqual(mock_tcp_server_connection.call_count, 2) + mock_conn.is_reusable.assert_not_called() diff --git a/tests/core/test_connection.py b/tests/core/test_connection.py index 3cd63ad129..905ab56d2b 100644 --- a/tests/core/test_connection.py +++ b/tests/core/test_connection.py @@ -22,8 +22,10 @@ class TestTcpConnection(unittest.TestCase): class TcpConnectionToTest(TcpConnection): - def __init__(self, conn: Optional[Union[ssl.SSLSocket, socket.socket]] = None, - tag: int = tcpConnectionTypes.CLIENT) -> None: + def __init__( + self, conn: Optional[Union[ssl.SSLSocket, socket.socket]] = None, + tag: int = tcpConnectionTypes.CLIENT, + ) -> None: super().__init__(tag) self._conn = conn @@ -65,48 +67,60 @@ def testFlushReturnsIfNoBuffer(self) -> None: @mock.patch('socket.socket') def testTcpServerEstablishesIPv6Connection( - self, mock_socket: mock.Mock) -> None: + self, mock_socket: mock.Mock, + ) -> None: conn = TcpServerConnection( - str(DEFAULT_IPV6_HOSTNAME), DEFAULT_PORT) + str(DEFAULT_IPV6_HOSTNAME), DEFAULT_PORT, + ) conn.connect() mock_socket.assert_called() mock_socket.return_value.connect.assert_called_with( - (str(DEFAULT_IPV6_HOSTNAME), DEFAULT_PORT, 0, 0)) + (str(DEFAULT_IPV6_HOSTNAME), DEFAULT_PORT, 0, 0), + ) @mock.patch('proxy.core.connection.server.new_socket_connection') def testTcpServerIgnoresDoubleConnectSilently( self, - mock_new_socket_connection: mock.Mock) -> None: + mock_new_socket_connection: mock.Mock, + ) -> None: conn = TcpServerConnection( - str(DEFAULT_IPV6_HOSTNAME), DEFAULT_PORT) + str(DEFAULT_IPV6_HOSTNAME), DEFAULT_PORT, + ) conn.connect() conn.connect() mock_new_socket_connection.assert_called_once() @mock.patch('socket.socket') def testTcpServerEstablishesIPv4Connection( - self, mock_socket: mock.Mock) -> None: + self, mock_socket: mock.Mock, + ) -> None: conn = TcpServerConnection( - str(DEFAULT_IPV4_HOSTNAME), DEFAULT_PORT) + str(DEFAULT_IPV4_HOSTNAME), DEFAULT_PORT, + ) conn.connect() mock_socket.assert_called() mock_socket.return_value.connect.assert_called_with( - (str(DEFAULT_IPV4_HOSTNAME), DEFAULT_PORT)) + (str(DEFAULT_IPV4_HOSTNAME), DEFAULT_PORT), + ) @mock.patch('proxy.core.connection.server.new_socket_connection') def testTcpServerConnectionProperty( self, - mock_new_socket_connection: mock.Mock) -> None: + mock_new_socket_connection: mock.Mock, + ) -> None: conn = TcpServerConnection( - str(DEFAULT_IPV6_HOSTNAME), DEFAULT_PORT) + str(DEFAULT_IPV6_HOSTNAME), DEFAULT_PORT, + ) conn.connect() self.assertEqual( conn.connection, - mock_new_socket_connection.return_value) + mock_new_socket_connection.return_value, + ) def testTcpServerRaisesTcpConnectionUninitializedException(self) -> None: conn = TcpServerConnection( - str(DEFAULT_IPV6_HOSTNAME), DEFAULT_PORT) + str(DEFAULT_IPV6_HOSTNAME), DEFAULT_PORT, + ) with self.assertRaises(TcpConnectionUninitializedException): _ = conn.connection diff --git a/tests/core/test_event_dispatcher.py b/tests/core/test_event_dispatcher.py index bb17a709ba..74e7b7187e 100644 --- a/tests/core/test_event_dispatcher.py +++ b/tests/core/test_event_dispatcher.py @@ -8,15 +8,16 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -import multiprocessing import os import threading import unittest import queue +import multiprocessing + +from multiprocessing import connection from unittest import mock -from proxy.common.types import DictQueueType from proxy.core.event import EventDispatcher, EventQueue, eventNames @@ -24,68 +25,91 @@ class TestEventDispatcher(unittest.TestCase): def setUp(self) -> None: self.dispatcher_shutdown = threading.Event() - self.event_queue = EventQueue(multiprocessing.Manager().Queue()) + self.manager = multiprocessing.Manager() + self.event_queue = EventQueue(self.manager.Queue()) self.dispatcher = EventDispatcher( shutdown=self.dispatcher_shutdown, - event_queue=self.event_queue) + event_queue=self.event_queue, + ) def tearDown(self) -> None: self.dispatcher_shutdown.set() + self.manager.shutdown() def test_empties_queue(self) -> None: self.event_queue.publish( request_id='1234', event_name=eventNames.WORK_STARTED, event_payload={'hello': 'events'}, - publisher_id=self.__class__.__name__ + publisher_id=self.__class__.__name__, ) self.dispatcher.run_once() with self.assertRaises(queue.Empty): self.dispatcher.run_once() @mock.patch('time.time') - def subscribe(self, mock_time: mock.Mock) -> DictQueueType: + def subscribe(self, mock_time: mock.Mock) -> connection.Connection: mock_time.return_value = 1234567 - q = multiprocessing.Manager().Queue() - self.event_queue.subscribe(sub_id='1234', channel=q) + relay_recv, relay_send = multiprocessing.Pipe() + self.event_queue.subscribe(sub_id='1234', channel=relay_send) + # consume the subscribe event self.dispatcher.run_once() + # assert subscribed ack + self.assertEqual( + relay_recv.recv(), { + 'event_name': eventNames.SUBSCRIBED, + }, + ) + # publish event self.event_queue.publish( request_id='1234', event_name=eventNames.WORK_STARTED, event_payload={'hello': 'events'}, - publisher_id=self.__class__.__name__ + publisher_id=self.__class__.__name__, ) + # consume self.dispatcher.run_once() - self.assertEqual(q.get(), { - 'request_id': '1234', - 'process_id': os.getpid(), - 'thread_id': threading.get_ident(), - 'event_timestamp': 1234567, - 'event_name': eventNames.WORK_STARTED, - 'event_payload': {'hello': 'events'}, - 'publisher_id': self.__class__.__name__, - }) - return q + # assert recv + r = relay_recv.recv() + print(r) + self.assertEqual( + r, { + 'request_id': '1234', + 'process_id': os.getpid(), + 'thread_id': threading.get_ident(), + 'event_timestamp': 1234567, + 'event_name': eventNames.WORK_STARTED, + 'event_payload': {'hello': 'events'}, + 'publisher_id': self.__class__.__name__, + }, + ) + return relay_recv def test_subscribe(self) -> None: self.subscribe() def test_unsubscribe(self) -> None: - q = self.subscribe() + relay_recv = self.subscribe() self.event_queue.unsubscribe('1234') self.dispatcher.run_once() + # assert unsubscribe ack + self.assertEqual( + relay_recv.recv(), { + 'event_name': eventNames.UNSUBSCRIBED, + }, + ) self.event_queue.publish( request_id='1234', event_name=eventNames.WORK_STARTED, event_payload={'hello': 'events'}, - publisher_id=self.__class__.__name__ + publisher_id=self.__class__.__name__, ) self.dispatcher.run_once() - with self.assertRaises(queue.Empty): - q.get(timeout=0.1) + with self.assertRaises(EOFError): + relay_recv.recv() - def test_unsubscribe_on_broken_pipe_error(self) -> None: - pass + # def test_unsubscribe_on_broken_pipe_error(self) -> None: + # pass - def test_run(self) -> None: - pass + # def test_run(self) -> None: + # pass diff --git a/tests/core/test_event_manager.py b/tests/core/test_event_manager.py new file mode 100644 index 0000000000..5f532a95fe --- /dev/null +++ b/tests/core/test_event_manager.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import unittest + +from unittest import mock + +from proxy.core.event import EventManager + + +class TestEventManager(unittest.TestCase): + + @mock.patch('proxy.core.event.manager.EventQueue') + @mock.patch('proxy.core.event.manager.EventDispatcher') + @mock.patch('proxy.core.event.manager.multiprocessing.Queue') + @mock.patch('proxy.core.event.manager.threading.Event') + @mock.patch('proxy.core.event.manager.threading.Thread') + def test_setup_and_teardown( + self, + mock_thread: mock.Mock, + mock_event: mock.Mock, + mock_queue: mock.Mock, + mock_dispatcher: mock.Mock, + mock_event_queue: mock.Mock, + ) -> None: + with EventManager() as _: + mock_queue.assert_called_once() + mock_event.assert_called_once() + mock_thread.assert_called_once_with( + target=mock_dispatcher.return_value.run, + ) + mock_thread.return_value.start.assert_called_once() + mock_event_queue.assert_called_once_with(mock_queue.return_value) + mock_dispatcher.assert_called_once_with( + shutdown=mock_event.return_value, + event_queue=mock_event_queue.return_value, + ) + mock_event.return_value.set.assert_called_once() + mock_thread.return_value.join.assert_called_once() diff --git a/tests/core/test_event_queue.py b/tests/core/test_event_queue.py index 18f7527163..174d573244 100644 --- a/tests/core/test_event_queue.py +++ b/tests/core/test_event_queue.py @@ -17,41 +17,54 @@ from proxy.core.event import EventQueue, eventNames -MANAGER = multiprocessing.Manager() - class TestCoreEvent(unittest.TestCase): + def setUp(self) -> None: + self.manager = multiprocessing.Manager() + + def tearDown(self) -> None: + self.manager.shutdown() + @mock.patch('time.time') def test_publish(self, mock_time: mock.Mock) -> None: mock_time.return_value = 1234567 - evq = EventQueue(MANAGER.Queue()) + evq = EventQueue(self.manager.Queue()) evq.publish( request_id='1234', event_name=eventNames.WORK_STARTED, event_payload={'hello': 'events'}, - publisher_id=self.__class__.__name__ + publisher_id=self.__class__.__name__, + ) + self.assertEqual( + evq.queue.get(), { + 'request_id': '1234', + 'process_id': os.getpid(), + 'thread_id': threading.get_ident(), + 'event_timestamp': 1234567, + 'event_name': eventNames.WORK_STARTED, + 'event_payload': {'hello': 'events'}, + 'publisher_id': self.__class__.__name__, + }, ) - self.assertEqual(evq.queue.get(), { - 'request_id': '1234', - 'process_id': os.getpid(), - 'thread_id': threading.get_ident(), - 'event_timestamp': 1234567, - 'event_name': eventNames.WORK_STARTED, - 'event_payload': {'hello': 'events'}, - 'publisher_id': self.__class__.__name__, - }) def test_subscribe(self) -> None: - evq = EventQueue(MANAGER.Queue()) - q = multiprocessing.Manager().Queue() - evq.subscribe('1234', q) + evq = EventQueue(self.manager.Queue()) + _, relay_send = multiprocessing.Pipe() + print(relay_send) + evq.subscribe('1234', relay_send) ev = evq.queue.get() self.assertEqual(ev['event_name'], eventNames.SUBSCRIBE) self.assertEqual(ev['event_payload']['sub_id'], '1234') + # Not the same as the one sent over multiprocessing.Queue + # will be another descriptor. We must ideally use + # send_handle during subscription too. + # + # self.assertEqual(ev['event_payload'] + # ['conn'].fileno(), relay_send.fileno()) def test_unsubscribe(self) -> None: - evq = EventQueue(MANAGER.Queue()) + evq = EventQueue(self.manager.Queue()) evq.unsubscribe('1234') ev = evq.queue.get() self.assertEqual(ev['event_name'], eventNames.UNSUBSCRIBE) diff --git a/tests/core/test_event_subscriber.py b/tests/core/test_event_subscriber.py index 30e67b39df..3ce8dcb844 100644 --- a/tests/core/test_event_subscriber.py +++ b/tests/core/test_event_subscriber.py @@ -9,6 +9,7 @@ :license: BSD, see LICENSE for more details. """ import os +import queue import threading import unittest import multiprocessing @@ -23,38 +24,48 @@ class TestEventSubscriber(unittest.TestCase): + def setUp(self) -> None: + self.manager = multiprocessing.Manager() + self.event_queue = EventQueue(self.manager.Queue()) + + def tearDown(self) -> None: + self.manager.shutdown() + @mock.patch('time.time') def test_event_subscriber(self, mock_time: mock.Mock) -> None: mock_time.return_value = 1234567 self.dispatcher_shutdown = threading.Event() - self.event_queue = EventQueue(multiprocessing.Manager().Queue()) self.dispatcher = EventDispatcher( shutdown=self.dispatcher_shutdown, - event_queue=self.event_queue) - self.subscriber = EventSubscriber(self.event_queue) - - self.subscriber.subscribe(self.callback) + event_queue=self.event_queue, + ) + self.subscriber = EventSubscriber(self.event_queue, self.callback) + self.subscriber.setup() self.dispatcher.run_once() self.event_queue.publish( request_id='1234', event_name=eventNames.WORK_STARTED, event_payload={'hello': 'events'}, - publisher_id=self.__class__.__name__ + publisher_id=self.__class__.__name__, ) self.dispatcher.run_once() - self.subscriber.unsubscribe() self.dispatcher.run_once() + self.subscriber.shutdown(do_unsubscribe=False) + with self.assertRaises(queue.Empty): + self.dispatcher.run_once() self.dispatcher_shutdown.set() def callback(self, ev: Dict[str, Any]) -> None: - self.assertEqual(ev, { - 'request_id': '1234', - 'process_id': os.getpid(), - 'thread_id': PUBLISHER_ID, - 'event_timestamp': 1234567, - 'event_name': eventNames.WORK_STARTED, - 'event_payload': {'hello': 'events'}, - 'publisher_id': self.__class__.__name__, - }) + self.assertEqual( + ev, { + 'request_id': '1234', + 'process_id': os.getpid(), + 'thread_id': PUBLISHER_ID, + 'event_timestamp': 1234567, + 'event_name': eventNames.WORK_STARTED, + 'event_payload': {'hello': 'events'}, + 'publisher_id': self.__class__.__name__, + }, + ) diff --git a/tests/core/test_listener.py b/tests/core/test_listener.py new file mode 100644 index 0000000000..5ca29e58f9 --- /dev/null +++ b/tests/core/test_listener.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import os +import socket +import tempfile +import unittest + +from unittest import mock + +import pytest + +from proxy.core.acceptor import Listener +from proxy.common._compat import IS_WINDOWS # noqa: WPS436 +from proxy.common.flag import FlagParser + + +class TestListener(unittest.TestCase): + + @mock.patch('socket.socket') + def test_setup_and_teardown(self, mock_socket: mock.Mock) -> None: + sock = mock_socket.return_value + flags = FlagParser.initialize(port=0) + listener = Listener(flags) + listener.setup() + mock_socket.assert_called_with( + socket.AF_INET6 if flags.hostname.version == 6 else socket.AF_INET, + socket.SOCK_STREAM, + ) + self.assertEqual(sock.setsockopt.call_count, 2) + self.assertEqual( + sock.setsockopt.call_args_list[0][0], + (socket.SOL_SOCKET, socket.SO_REUSEADDR, 1), + ) + self.assertEqual( + sock.setsockopt.call_args_list[1][0], + (socket.IPPROTO_TCP, socket.TCP_NODELAY, 1), + ) + sock.bind.assert_called_with( + (str(flags.hostname), 0), + ) + sock.listen.assert_called_with(flags.backlog) + sock.setblocking.assert_called_with(False) + + listener.shutdown() + sock.close.assert_called_once() + + # FIXME: Ignore is necessary for as long as pytest hasn't figured out + # FIXME: typing for their fixtures. + # Refs: + # * https://github.com/pytest-dev/pytest/issues/7469#issuecomment-918345196 + # * https://github.com/pytest-dev/pytest/issues/3342 + @pytest.mark.skipif( + IS_WINDOWS, + reason='AF_UNIX not available on Windows', + ) # type: ignore[misc] + @mock.patch('os.remove') + @mock.patch('socket.socket') + def test_unix_path_listener(self, mock_socket: mock.Mock, mock_remove: mock.Mock) -> None: + sock = mock_socket.return_value + sock_path = os.path.join(tempfile.gettempdir(), 'proxy.sock') + flags = FlagParser.initialize(unix_socket_path=sock_path) + listener = Listener(flags) + listener.setup() + + mock_socket.assert_called_with( + socket.AF_UNIX, + socket.SOCK_STREAM, + ) + self.assertEqual(sock.setsockopt.call_count, 2) + self.assertEqual( + sock.setsockopt.call_args_list[0][0], + (socket.SOL_SOCKET, socket.SO_REUSEADDR, 1), + ) + self.assertEqual( + sock.setsockopt.call_args_list[1][0], + (socket.IPPROTO_TCP, socket.TCP_NODELAY, 1), + ) + sock.bind.assert_called_with(sock_path) + sock.listen.assert_called_with(flags.backlog) + sock.setblocking.assert_called_with(False) + + listener.shutdown() + mock_remove.assert_called_once_with(sock_path) + sock.close.assert_called_once() diff --git a/tests/http/exceptions/__init__.py b/tests/http/exceptions/__init__.py new file mode 100644 index 0000000000..232621f0b5 --- /dev/null +++ b/tests/http/exceptions/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" diff --git a/tests/http/exceptions/test_http_proxy_auth_failed.py b/tests/http/exceptions/test_http_proxy_auth_failed.py new file mode 100644 index 0000000000..9b4feb1bc5 --- /dev/null +++ b/tests/http/exceptions/test_http_proxy_auth_failed.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import pytest +import selectors + +from pytest_mock import MockerFixture + +from proxy.common.flag import FlagParser +from proxy.http.exception.proxy_auth_failed import ProxyAuthenticationFailed +from proxy.http import HttpProtocolHandler +from proxy.core.connection import TcpClientConnection +from proxy.common.utils import build_http_request + +from ...test_assertions import Assertions + + +class TestHttpProxyAuthFailed(Assertions): + + @pytest.fixture(autouse=True) # type: ignore[misc] + def _setUp(self, mocker: MockerFixture) -> None: + self.mock_fromfd = mocker.patch('socket.fromfd') + self.mock_selector = mocker.patch('selectors.DefaultSelector') + self.mock_server_conn = mocker.patch( + 'proxy.http.proxy.server.TcpServerConnection', + ) + + self.fileno = 10 + self._addr = ('127.0.0.1', 54382) + self.flags = FlagParser.initialize( + ["--basic-auth", "user:pass"], threaded=True, + ) + self._conn = self.mock_fromfd.return_value + self.protocol_handler = HttpProtocolHandler( + TcpClientConnection(self._conn, self._addr), + flags=self.flags, + ) + self.protocol_handler.initialize() + + @pytest.mark.asyncio # type: ignore[misc] + async def test_proxy_auth_fails_without_cred(self) -> None: + self._conn.recv.return_value = build_http_request( + b'GET', b'http://upstream.host/not-found.html', + headers={ + b'Host': b'upstream.host', + }, + ) + self.mock_selector.return_value.select.side_effect = [ + [( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=selectors.EVENT_READ, + data=None, + ), + selectors.EVENT_READ, + )], + ] + await self.protocol_handler._run_once() + self.mock_server_conn.assert_not_called() + self.assertEqual(self.protocol_handler.work.has_buffer(), True) + self.assertEqual( + self.protocol_handler.work.buffer[0], ProxyAuthenticationFailed.RESPONSE_PKT, + ) + self._conn.send.assert_not_called() + + @pytest.mark.asyncio # type: ignore[misc] + async def test_proxy_auth_fails_with_invalid_cred(self) -> None: + self._conn.recv.return_value = build_http_request( + b'GET', b'http://upstream.host/not-found.html', + headers={ + b'Host': b'upstream.host', + b'Proxy-Authorization': b'Basic hello', + }, + ) + self.mock_selector.return_value.select.side_effect = [ + [( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=selectors.EVENT_READ, + data=None, + ), + selectors.EVENT_READ, + )], + ] + await self.protocol_handler._run_once() + self.mock_server_conn.assert_not_called() + self.assertEqual(self.protocol_handler.work.has_buffer(), True) + self.assertEqual( + self.protocol_handler.work.buffer[0], ProxyAuthenticationFailed.RESPONSE_PKT, + ) + self._conn.send.assert_not_called() + + @pytest.mark.asyncio # type: ignore[misc] + async def test_proxy_auth_works_with_valid_cred(self) -> None: + self._conn.recv.return_value = build_http_request( + b'GET', b'http://upstream.host/not-found.html', + headers={ + b'Host': b'upstream.host', + b'Proxy-Authorization': b'Basic dXNlcjpwYXNz', + }, + ) + self.mock_selector.return_value.select.side_effect = [ + [( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=selectors.EVENT_READ, + data=None, + ), + selectors.EVENT_READ, + )], + ] + await self.protocol_handler._run_once() + self.mock_server_conn.assert_called_once() + self.assertEqual(self.protocol_handler.work.has_buffer(), False) + + @pytest.mark.asyncio # type: ignore[misc] + async def test_proxy_auth_works_with_mixed_case_basic_string(self) -> None: + self._conn.recv.return_value = build_http_request( + b'GET', b'http://upstream.host/not-found.html', + headers={ + b'Host': b'upstream.host', + b'Proxy-Authorization': b'bAsIc dXNlcjpwYXNz', + }, + ) + self.mock_selector.return_value.select.side_effect = [ + [( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=selectors.EVENT_READ, + data=None, + ), + selectors.EVENT_READ, + )], + ] + await self.protocol_handler._run_once() + self.mock_server_conn.assert_called_once() + self.assertEqual(self.protocol_handler.work.has_buffer(), False) diff --git a/tests/http/test_http_request_rejected.py b/tests/http/exceptions/test_http_request_rejected.py similarity index 74% rename from tests/http/test_http_request_rejected.py rename to tests/http/exceptions/test_http_request_rejected.py index 59eac81c3b..de8f736bc4 100644 --- a/tests/http/test_http_request_rejected.py +++ b/tests/http/exceptions/test_http_request_rejected.py @@ -10,11 +10,11 @@ """ import unittest +from proxy.http import httpStatusCodes from proxy.http.parser import HttpParser, httpParserTypes from proxy.http.exception import HttpRequestRejected from proxy.common.constants import CRLF from proxy.common.utils import build_http_response -from proxy.http.codes import httpStatusCodes class TestHttpRequestRejected(unittest.TestCase): @@ -28,15 +28,22 @@ def test_empty_response(self) -> None: def test_status_code_response(self) -> None: e = HttpRequestRejected(status_code=200, reason=b'OK') - self.assertEqual(e.response(self.request), CRLF.join([ - b'HTTP/1.1 200 OK', - CRLF - ])) + self.assertEqual( + e.response(self.request), CRLF.join([ + b'HTTP/1.1 200 OK', + CRLF, + ]), + ) def test_body_response(self) -> None: e = HttpRequestRejected( status_code=httpStatusCodes.NOT_FOUND, reason=b'NOT FOUND', - body=b'Nothing here') + body=b'Nothing here', + ) self.assertEqual( e.response(self.request), - build_http_response(httpStatusCodes.NOT_FOUND, reason=b'NOT FOUND', body=b'Nothing here')) + build_http_response( + httpStatusCodes.NOT_FOUND, + reason=b'NOT FOUND', body=b'Nothing here', + ), + ) diff --git a/tests/http/test_chunk_parser.py b/tests/http/test_chunk_parser.py index 94b71afb6d..68d10781b8 100644 --- a/tests/http/test_chunk_parser.py +++ b/tests/http/test_chunk_parser.py @@ -10,7 +10,7 @@ """ import unittest -from proxy.http.chunk_parser import chunkParserStates, ChunkParser +from proxy.http.parser import chunkParserStates, ChunkParser class TestChunkParser(unittest.TestCase): @@ -19,16 +19,18 @@ def setUp(self) -> None: self.parser = ChunkParser() def test_chunk_parse_basic(self) -> None: - self.parser.parse(b''.join([ - b'4\r\n', - b'Wiki\r\n', - b'5\r\n', - b'pedia\r\n', - b'E\r\n', - b' in\r\n\r\nchunks.\r\n', - b'0\r\n', - b'\r\n' - ])) + self.parser.parse( + b''.join([ + b'4\r\n', + b'Wiki\r\n', + b'5\r\n', + b'pedia\r\n', + b'E\r\n', + b' in\r\n\r\nchunks.\r\n', + b'0\r\n', + b'\r\n', + ]), + ) self.assertEqual(self.parser.chunk, b'') self.assertEqual(self.parser.size, None) self.assertEqual(self.parser.body, b'Wikipedia in\r\n\r\nchunks.') @@ -42,42 +44,48 @@ def test_chunk_parse_issue_27(self) -> None: self.assertEqual(self.parser.body, b'') self.assertEqual( self.parser.state, - chunkParserStates.WAITING_FOR_SIZE) + chunkParserStates.WAITING_FOR_SIZE, + ) self.parser.parse(b'\r\n') self.assertEqual(self.parser.chunk, b'') self.assertEqual(self.parser.size, 3) self.assertEqual(self.parser.body, b'') self.assertEqual( self.parser.state, - chunkParserStates.WAITING_FOR_DATA) + chunkParserStates.WAITING_FOR_DATA, + ) self.parser.parse(b'abc') self.assertEqual(self.parser.chunk, b'') self.assertEqual(self.parser.size, None) self.assertEqual(self.parser.body, b'abc') self.assertEqual( self.parser.state, - chunkParserStates.WAITING_FOR_SIZE) + chunkParserStates.WAITING_FOR_SIZE, + ) self.parser.parse(b'\r\n') self.assertEqual(self.parser.chunk, b'') self.assertEqual(self.parser.size, None) self.assertEqual(self.parser.body, b'abc') self.assertEqual( self.parser.state, - chunkParserStates.WAITING_FOR_SIZE) + chunkParserStates.WAITING_FOR_SIZE, + ) self.parser.parse(b'4\r\n') self.assertEqual(self.parser.chunk, b'') self.assertEqual(self.parser.size, 4) self.assertEqual(self.parser.body, b'abc') self.assertEqual( self.parser.state, - chunkParserStates.WAITING_FOR_DATA) + chunkParserStates.WAITING_FOR_DATA, + ) self.parser.parse(b'defg\r\n0') self.assertEqual(self.parser.chunk, b'0') self.assertEqual(self.parser.size, None) self.assertEqual(self.parser.body, b'abcdefg') self.assertEqual( self.parser.state, - chunkParserStates.WAITING_FOR_SIZE) + chunkParserStates.WAITING_FOR_SIZE, + ) self.parser.parse(b'\r\n\r\n') self.assertEqual(self.parser.chunk, b'') self.assertEqual(self.parser.size, None) @@ -87,4 +95,5 @@ def test_chunk_parse_issue_27(self) -> None: def test_to_chunks(self) -> None: self.assertEqual( b'f\r\n{"key":"value"}\r\n0\r\n\r\n', - ChunkParser.to_chunks(b'{"key":"value"}')) + ChunkParser.to_chunks(b'{"key":"value"}'), + ) diff --git a/tests/http/test_http_parser.py b/tests/http/test_http_parser.py index 253f2b7f7b..8d97a8fa6b 100644 --- a/tests/http/test_http_parser.py +++ b/tests/http/test_http_parser.py @@ -10,10 +10,11 @@ """ import unittest -from proxy.common.constants import CRLF -from proxy.common.utils import build_http_request, find_http_line, build_http_response, build_http_header, bytes_ -from proxy.http.methods import httpMethods -from proxy.http.codes import httpStatusCodes +from proxy.common.constants import CRLF, HTTP_1_0 +from proxy.common.utils import build_http_request, build_http_response, build_http_header +from proxy.common.utils import find_http_line, bytes_ + +from proxy.http import httpStatusCodes, httpMethods from proxy.http.parser import HttpParser, httpParserTypes, httpParserStates @@ -22,71 +23,161 @@ class TestHttpParser(unittest.TestCase): def setUp(self) -> None: self.parser = HttpParser(httpParserTypes.REQUEST_PARSER) + def test_issue_127(self) -> None: + with self.assertRaises(ValueError): + self.parser.parse(CRLF) + + with self.assertRaises(ValueError): + raw = b'qwqrqw!@!#@!#ad adfad\r\n' + while True: + self.parser.parse(raw) + + def test_issue_398(self) -> None: + p = HttpParser(httpParserTypes.RESPONSE_PARSER) + p.parse(HTTP_1_0 + b' 200 OK' + CRLF) + self.assertEqual(p.version, HTTP_1_0) + self.assertEqual(p.code, b'200') + self.assertEqual(p.reason, b'OK') + self.assertEqual(p.state, httpParserStates.LINE_RCVD) + p.parse( + b'CP=CAO PSA OUR' + CRLF + + b'Cache-Control:private,max-age=0;' + CRLF + + b'X-Frame-Options:SAMEORIGIN' + CRLF + + b'X-Content-Type-Options:nosniff' + CRLF + + b'X-XSS-Protection:1; mode=block' + CRLF + + b'Content-Security-Policy:default-src \'self\' \'unsafe-inline\' \'unsafe-eval\'' + CRLF + + b'Strict-Transport-Security:max-age=2592000; includeSubdomains' + CRLF + + b'Set-Cookie: lang=eng; path=/;HttpOnly;' + CRLF + + b'Content-type:text/html;charset=UTF-8;' + CRLF + CRLF + + b'', + ) + self.assertEqual(p.body, b'') + self.assertEqual(p.state, httpParserStates.RCVING_BODY) + def test_urlparse(self) -> None: self.parser.parse(b'CONNECT httpbin.org:443 HTTP/1.1\r\n') + self.assertTrue(self.parser.is_https_tunnel()) self.assertEqual(self.parser.host, b'httpbin.org') self.assertEqual(self.parser.port, 443) + self.assertNotEqual(self.parser.state, httpParserStates.COMPLETE) + + def test_urlparse_on_invalid_connect_request(self) -> None: + self.parser.parse(b'CONNECT / HTTP/1.0\r\n\r\n') + self.assertTrue(self.parser.is_https_tunnel()) + self.assertEqual(self.parser.host, None) + self.assertEqual(self.parser.port, 443) + self.assertEqual(self.parser.state, httpParserStates.COMPLETE) + + def test_unicode_character_domain_connect(self) -> None: + self.parser.parse(bytes_('CONNECT รงรงรง.org:443 HTTP/1.1\r\n')) + self.assertTrue(self.parser.is_https_tunnel()) + self.assertEqual(self.parser.host, bytes_('รงรงรง.org')) + self.assertEqual(self.parser.port, 443) + + def test_invalid_ipv6_in_request_line(self) -> None: + self.parser.parse( + bytes_('CONNECT 2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF:443 HTTP/1.1\r\n'), + ) + self.assertTrue(self.parser.is_https_tunnel()) + self.assertEqual( + self.parser.host, bytes_( + '[2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF]', + ), + ) + self.assertEqual(self.parser.port, 443) + + def test_valid_ipv6_in_request_line(self) -> None: + self.parser.parse( + bytes_( + 'CONNECT [2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF]:443 HTTP/1.1\r\n', + ), + ) + self.assertTrue(self.parser.is_https_tunnel()) + self.assertEqual( + self.parser.host, bytes_( + '[2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF]', + ), + ) + self.assertEqual(self.parser.port, 443) def test_build_request(self) -> None: self.assertEqual( build_http_request( - b'GET', b'http://localhost:12345', b'HTTP/1.1'), + b'GET', b'http://localhost:12345', b'HTTP/1.1', + ), CRLF.join([ b'GET http://localhost:12345 HTTP/1.1', - CRLF - ])) + CRLF, + ]), + ) self.assertEqual( - build_http_request(b'GET', b'http://localhost:12345', b'HTTP/1.1', - headers={b'key': b'value'}), + build_http_request( + b'GET', b'http://localhost:12345', b'HTTP/1.1', + headers={b'key': b'value'}, + ), CRLF.join([ b'GET http://localhost:12345 HTTP/1.1', b'key: value', - CRLF - ])) + CRLF, + ]), + ) self.assertEqual( - build_http_request(b'GET', b'http://localhost:12345', b'HTTP/1.1', - headers={b'key': b'value'}, - body=b'Hello from py'), + build_http_request( + b'GET', b'http://localhost:12345', b'HTTP/1.1', + headers={b'key': b'value'}, + body=b'Hello from py', + ), CRLF.join([ b'GET http://localhost:12345 HTTP/1.1', b'key: value', - CRLF - ]) + b'Hello from py') + CRLF, + ]) + b'Hello from py', + ) def test_build_response(self) -> None: self.assertEqual( build_http_response( - 200, reason=b'OK', protocol_version=b'HTTP/1.1'), + 200, reason=b'OK', protocol_version=b'HTTP/1.1', + ), CRLF.join([ b'HTTP/1.1 200 OK', - CRLF - ])) + CRLF, + ]), + ) self.assertEqual( - build_http_response(200, reason=b'OK', protocol_version=b'HTTP/1.1', - headers={b'key': b'value'}), + build_http_response( + 200, reason=b'OK', protocol_version=b'HTTP/1.1', + headers={b'key': b'value'}, + ), CRLF.join([ b'HTTP/1.1 200 OK', b'key: value', - CRLF - ])) + CRLF, + ]), + ) def test_build_response_adds_content_length_header(self) -> None: body = b'Hello world!!!' self.assertEqual( - build_http_response(200, reason=b'OK', protocol_version=b'HTTP/1.1', - headers={b'key': b'value'}, - body=body), + build_http_response( + 200, reason=b'OK', protocol_version=b'HTTP/1.1', + headers={b'key': b'value'}, + body=body, + ), CRLF.join([ b'HTTP/1.1 200 OK', b'key: value', b'Content-Length: ' + bytes_(len(body)), - CRLF - ]) + body) + CRLF, + ]) + body, + ) def test_build_header(self) -> None: self.assertEqual( build_http_header( - b'key', b'value'), b'key: value') + b'key', b'value', + ), b'key: value', + ) def test_header_raises(self) -> None: with self.assertRaises(KeyError): @@ -99,20 +190,27 @@ def test_has_header(self) -> None: def test_set_host_port_raises(self) -> None: with self.assertRaises(KeyError): - self.parser.set_line_attributes() + self.parser._set_line_attributes() def test_find_line(self) -> None: self.assertEqual( find_http_line( - b'CONNECT python.org:443 HTTP/1.0\r\n\r\n'), - (b'CONNECT python.org:443 HTTP/1.0', - CRLF)) + b'CONNECT python.org:443 HTTP/1.0\r\n\r\n', + ), + ( + b'CONNECT python.org:443 HTTP/1.0', + CRLF, + ), + ) def test_find_line_returns_None(self) -> None: self.assertEqual( find_http_line(b'CONNECT python.org:443 HTTP/1.0'), - (None, - b'CONNECT python.org:443 HTTP/1.0')) + ( + None, + b'CONNECT python.org:443 HTTP/1.0', + ), + ) def test_connect_request_with_crlf_as_separate_chunk(self) -> None: """See https://github.com/abhinavsingh/py/issues/70 for background.""" @@ -126,31 +224,34 @@ def test_get_full_parse(self) -> None: raw = CRLF.join([ b'GET %s HTTP/1.1', b'Host: %s', - CRLF + CRLF, ]) - pkt = raw % (b'https://example.com/path/dir/?a=b&c=d#p=q', - b'example.com') + pkt = raw % ( + b'https://example.com/path/dir/?a=b&c=d#p=q', + b'example.com', + ) self.parser.parse(pkt) self.assertEqual(self.parser.total_size, len(pkt)) - self.assertEqual(self.parser.build_path(), b'/path/dir/?a=b&c=d#p=q') + assert self.parser._url and self.parser._url.remainder + self.assertEqual(self.parser._url.remainder, b'/path/dir/?a=b&c=d#p=q') self.assertEqual(self.parser.method, b'GET') - assert self.parser.url - self.assertEqual(self.parser.url.hostname, b'example.com') - self.assertEqual(self.parser.url.port, None) + self.assertEqual(self.parser._url.hostname, b'example.com') + self.assertEqual(self.parser._url.port, None) self.assertEqual(self.parser.version, b'HTTP/1.1') self.assertEqual(self.parser.state, httpParserStates.COMPLETE) self.assertEqual( - self.parser.headers[b'host'], (b'Host', b'example.com')) + self.parser.headers[b'host'], (b'Host', b'example.com'), + ) self.parser.del_headers([b'host']) self.parser.add_headers([(b'Host', b'example.com')]) self.assertEqual( raw % - (b'/path/dir/?a=b&c=d#p=q', - b'example.com'), - self.parser.build()) - - def test_build_url_none(self) -> None: - self.assertEqual(self.parser.build_path(), b'/None') + ( + b'/path/dir/?a=b&c=d#p=q', + b'example.com', + ), + self.parser.build(), + ) def test_line_rcvd_to_rcving_headers_state_change(self) -> None: pkt = b'GET http://localhost HTTP/1.1' @@ -159,56 +260,67 @@ def test_line_rcvd_to_rcving_headers_state_change(self) -> None: self.assert_state_change_with_crlf( httpParserStates.INITIALIZED, httpParserStates.LINE_RCVD, - httpParserStates.COMPLETE) + httpParserStates.COMPLETE, + ) def test_get_partial_parse1(self) -> None: pkt = CRLF.join([ - b'GET http://localhost:8080 HTTP/1.1' + b'GET http://localhost:8080 HTTP/1.1', ]) self.parser.parse(pkt) self.assertEqual(self.parser.total_size, len(pkt)) self.assertEqual(self.parser.method, None) - self.assertEqual(self.parser.url, None) + self.assertEqual(self.parser._url, None) self.assertEqual(self.parser.version, None) self.assertEqual( self.parser.state, - httpParserStates.INITIALIZED) + httpParserStates.INITIALIZED, + ) self.parser.parse(CRLF) self.assertEqual(self.parser.total_size, len(pkt) + len(CRLF)) self.assertEqual(self.parser.method, b'GET') - assert self.parser.url - self.assertEqual(self.parser.url.hostname, b'localhost') - self.assertEqual(self.parser.url.port, 8080) + assert self.parser._url + self.assertEqual(self.parser._url.hostname, b'localhost') + self.assertEqual(self.parser._url.port, 8080) self.assertEqual(self.parser.version, b'HTTP/1.1') self.assertEqual(self.parser.state, httpParserStates.LINE_RCVD) host_hdr = b'Host: localhost:8080' self.parser.parse(host_hdr) - self.assertEqual(self.parser.total_size, - len(pkt) + len(CRLF) + len(host_hdr)) - self.assertDictEqual(self.parser.headers, dict()) + self.assertEqual( + self.parser.total_size, + len(pkt) + len(CRLF) + len(host_hdr), + ) + self.assertDictEqual(self.parser.headers, {}) self.assertEqual(self.parser.buffer, b'Host: localhost:8080') self.assertEqual(self.parser.state, httpParserStates.LINE_RCVD) self.parser.parse(CRLF * 2) - self.assertEqual(self.parser.total_size, len(pkt) + - (3 * len(CRLF)) + len(host_hdr)) + self.assertEqual( + self.parser.total_size, len(pkt) + + (3 * len(CRLF)) + len(host_hdr), + ) self.assertEqual( self.parser.headers[b'host'], - (b'Host', - b'localhost:8080')) + ( + b'Host', + b'localhost:8080', + ), + ) self.assertEqual(self.parser.state, httpParserStates.COMPLETE) def test_get_partial_parse2(self) -> None: - self.parser.parse(CRLF.join([ - b'GET http://localhost:8080 HTTP/1.1', - b'Host: ' - ])) + self.parser.parse( + CRLF.join([ + b'GET http://localhost:8080 HTTP/1.1', + b'Host: ', + ]), + ) self.assertEqual(self.parser.method, b'GET') - assert self.parser.url - self.assertEqual(self.parser.url.hostname, b'localhost') - self.assertEqual(self.parser.url.port, 8080) + assert self.parser._url + self.assertEqual(self.parser._url.hostname, b'localhost') + self.assertEqual(self.parser._url.port, 8080) self.assertEqual(self.parser.version, b'HTTP/1.1') self.assertEqual(self.parser.buffer, b'Host: ') self.assertEqual(self.parser.state, httpParserStates.LINE_RCVD) @@ -216,20 +328,29 @@ def test_get_partial_parse2(self) -> None: self.parser.parse(b'localhost:8080' + CRLF) self.assertEqual( self.parser.headers[b'host'], - (b'Host', - b'localhost:8080')) + ( + b'Host', + b'localhost:8080', + ), + ) self.assertEqual(self.parser.buffer, b'') self.assertEqual( self.parser.state, - httpParserStates.RCVING_HEADERS) + httpParserStates.RCVING_HEADERS, + ) self.parser.parse(b'Content-Type: text/plain' + CRLF) self.assertEqual(self.parser.buffer, b'') self.assertEqual( - self.parser.headers[b'content-type'], (b'Content-Type', b'text/plain')) + self.parser.headers[b'content-type'], ( + b'Content-Type', + b'text/plain', + ), + ) self.assertEqual( self.parser.state, - httpParserStates.RCVING_HEADERS) + httpParserStates.RCVING_HEADERS, + ) self.parser.parse(CRLF) self.assertEqual(self.parser.state, httpParserStates.COMPLETE) @@ -240,27 +361,33 @@ def test_post_full_parse(self) -> None: b'Host: localhost', b'Content-Length: 7', b'Content-Type: application/x-www-form-urlencoded' + CRLF, - b'a=b&c=d' + b'a=b&c=d', ]) self.parser.parse(raw % b'http://localhost') self.assertEqual(self.parser.method, b'POST') - assert self.parser.url - self.assertEqual(self.parser.url.hostname, b'localhost') - self.assertEqual(self.parser.url.port, None) + assert self.parser._url + self.assertEqual(self.parser._url.hostname, b'localhost') + self.assertEqual(self.parser._url.port, None) self.assertEqual(self.parser.version, b'HTTP/1.1') - self.assertEqual(self.parser.headers[b'content-type'], - (b'Content-Type', b'application/x-www-form-urlencoded')) - self.assertEqual(self.parser.headers[b'content-length'], - (b'Content-Length', b'7')) + self.assertEqual( + self.parser.headers[b'content-type'], + (b'Content-Type', b'application/x-www-form-urlencoded'), + ) + self.assertEqual( + self.parser.headers[b'content-length'], + (b'Content-Length', b'7'), + ) self.assertEqual(self.parser.body, b'a=b&c=d') self.assertEqual(self.parser.buffer, b'') self.assertEqual(self.parser.state, httpParserStates.COMPLETE) self.assertEqual(len(self.parser.build()), len(raw % b'/')) - def assert_state_change_with_crlf(self, - initial_state: int, - next_state: int, - final_state: int) -> None: + def assert_state_change_with_crlf( + self, + initial_state: int, + next_state: int, + final_state: int, + ) -> None: self.assertEqual(self.parser.state, initial_state) self.parser.parse(CRLF) self.assertEqual(self.parser.state, next_state) @@ -268,26 +395,30 @@ def assert_state_change_with_crlf(self, self.assertEqual(self.parser.state, final_state) def test_post_partial_parse(self) -> None: - self.parser.parse(CRLF.join([ - b'POST http://localhost HTTP/1.1', - b'Host: localhost', - b'Content-Length: 7', - b'Content-Type: application/x-www-form-urlencoded' - ])) + self.parser.parse( + CRLF.join([ + b'POST http://localhost HTTP/1.1', + b'Host: localhost', + b'Content-Length: 7', + b'Content-Type: application/x-www-form-urlencoded', + ]), + ) self.assertEqual(self.parser.method, b'POST') - assert self.parser.url - self.assertEqual(self.parser.url.hostname, b'localhost') - self.assertEqual(self.parser.url.port, None) + assert self.parser._url + self.assertEqual(self.parser._url.hostname, b'localhost') + self.assertEqual(self.parser._url.port, None) self.assertEqual(self.parser.version, b'HTTP/1.1') self.assert_state_change_with_crlf( httpParserStates.RCVING_HEADERS, httpParserStates.RCVING_HEADERS, - httpParserStates.HEADERS_COMPLETE) + httpParserStates.HEADERS_COMPLETE, + ) self.parser.parse(b'a=b') self.assertEqual( self.parser.state, - httpParserStates.RCVING_BODY) + httpParserStates.RCVING_BODY, + ) self.assertEqual(self.parser.body, b'a=b') self.assertEqual(self.parser.buffer, b'') @@ -321,12 +452,14 @@ def test_request_parse_without_content_length(self) -> None: See https://github.com/abhinavsingh/py/issues/20 for details. """ - self.parser.parse(CRLF.join([ - b'POST http://localhost HTTP/1.1', - b'Host: localhost', - b'Content-Type: application/x-www-form-urlencoded', - CRLF - ])) + self.parser.parse( + CRLF.join([ + b'POST http://localhost HTTP/1.1', + b'Host: localhost', + b'Content-Type: application/x-www-form-urlencoded', + CRLF, + ]), + ) self.assertEqual(self.parser.method, b'POST') self.assertEqual(self.parser.state, httpParserStates.COMPLETE) @@ -350,33 +483,38 @@ def test_response_parse_without_content_length(self) -> None: self.assertEqual(self.parser.code, b'200') self.assertEqual(self.parser.version, b'HTTP/1.0') self.assertEqual(self.parser.state, httpParserStates.LINE_RCVD) - self.parser.parse(CRLF.join([ - b'Server: BaseHTTP/0.3 Python/2.7.10', - b'Date: Thu, 13 Dec 2018 16:24:09 GMT', - CRLF - ])) + self.parser.parse( + CRLF.join([ + b'Server: BaseHTTP/0.3 Python/2.7.10', + b'Date: Thu, 13 Dec 2018 16:24:09 GMT', + CRLF, + ]), + ) self.assertEqual( self.parser.state, - httpParserStates.COMPLETE) + httpParserStates.COMPLETE, + ) def test_response_parse(self) -> None: self.parser.type = httpParserTypes.RESPONSE_PARSER - self.parser.parse(b''.join([ - b'HTTP/1.1 301 Moved Permanently\r\n', - b'Location: http://www.google.com/\r\n', - b'Content-Type: text/html; charset=UTF-8\r\n', - b'Date: Wed, 22 May 2013 14:07:29 GMT\r\n', - b'Expires: Fri, 21 Jun 2013 14:07:29 GMT\r\n', - b'Cache-Control: public, max-age=2592000\r\n', - b'Server: gws\r\n', - b'Content-Length: 219\r\n', - b'X-XSS-Protection: 1; mode=block\r\n', - b'X-Frame-Options: SAMEORIGIN\r\n\r\n', - b'\n' + - b'301 Moved', - b'\n

301 Moved

\nThe document has moved\n' + - b'here.\r\n\r\n' - ])) + self.parser.parse( + b''.join([ + b'HTTP/1.1 301 Moved Permanently\r\n', + b'Location: http://www.google.com/\r\n', + b'Content-Type: text/html; charset=UTF-8\r\n', + b'Date: Wed, 22 May 2013 14:07:29 GMT\r\n', + b'Expires: Fri, 21 Jun 2013 14:07:29 GMT\r\n', + b'Cache-Control: public, max-age=2592000\r\n', + b'Server: gws\r\n', + b'Content-Length: 219\r\n', + b'X-XSS-Protection: 1; mode=block\r\n', + b'X-Frame-Options: SAMEORIGIN\r\n\r\n', + b'\n' + + b'301 Moved', + b'\n

301 Moved

\nThe document has moved\n' + + b'here.\r\n\r\n', + ]), + ) self.assertEqual(self.parser.code, b'301') self.assertEqual(self.parser.reason, b'Moved Permanently') self.assertEqual(self.parser.version, b'HTTP/1.1') @@ -384,63 +522,77 @@ def test_response_parse(self) -> None: self.parser.body, b'\n' + b'301 Moved\n

301 Moved

\nThe document has moved\n' + - b'here.\r\n\r\n') - self.assertEqual(self.parser.headers[b'content-length'], - (b'Content-Length', b'219')) + b'here.\r\n\r\n', + ) + self.assertEqual( + self.parser.headers[b'content-length'], + (b'Content-Length', b'219'), + ) self.assertEqual(self.parser.state, httpParserStates.COMPLETE) def test_response_partial_parse(self) -> None: self.parser.type = httpParserTypes.RESPONSE_PARSER - self.parser.parse(b''.join([ - b'HTTP/1.1 301 Moved Permanently\r\n', - b'Location: http://www.google.com/\r\n', - b'Content-Type: text/html; charset=UTF-8\r\n', - b'Date: Wed, 22 May 2013 14:07:29 GMT\r\n', - b'Expires: Fri, 21 Jun 2013 14:07:29 GMT\r\n', - b'Cache-Control: public, max-age=2592000\r\n', - b'Server: gws\r\n', - b'Content-Length: 219\r\n', - b'X-XSS-Protection: 1; mode=block\r\n', - b'X-Frame-Options: SAMEORIGIN\r\n' - ])) - self.assertEqual(self.parser.headers[b'x-frame-options'], - (b'X-Frame-Options', b'SAMEORIGIN')) + self.parser.parse( + b''.join([ + b'HTTP/1.1 301 Moved Permanently\r\n', + b'Location: http://www.google.com/\r\n', + b'Content-Type: text/html; charset=UTF-8\r\n', + b'Date: Wed, 22 May 2013 14:07:29 GMT\r\n', + b'Expires: Fri, 21 Jun 2013 14:07:29 GMT\r\n', + b'Cache-Control: public, max-age=2592000\r\n', + b'Server: gws\r\n', + b'Content-Length: 219\r\n', + b'X-XSS-Protection: 1; mode=block\r\n', + b'X-Frame-Options: SAMEORIGIN\r\n', + ]), + ) + self.assertEqual( + self.parser.headers[b'x-frame-options'], + (b'X-Frame-Options', b'SAMEORIGIN'), + ) self.assertEqual( self.parser.state, - httpParserStates.RCVING_HEADERS) + httpParserStates.RCVING_HEADERS, + ) self.parser.parse(b'\r\n') self.assertEqual( self.parser.state, - httpParserStates.HEADERS_COMPLETE) + httpParserStates.HEADERS_COMPLETE, + ) self.parser.parse( b'\n' + - b'301 Moved') + b'301 Moved', + ) self.assertEqual( self.parser.state, - httpParserStates.RCVING_BODY) + httpParserStates.RCVING_BODY, + ) self.parser.parse( b'\n

301 Moved

\nThe document has moved\n' + - b'here.\r\n\r\n') + b'here.\r\n\r\n', + ) self.assertEqual(self.parser.state, httpParserStates.COMPLETE) def test_chunked_response_parse(self) -> None: self.parser.type = httpParserTypes.RESPONSE_PARSER - self.parser.parse(b''.join([ - b'HTTP/1.1 200 OK\r\n', - b'Content-Type: application/json\r\n', - b'Date: Wed, 22 May 2013 15:08:15 GMT\r\n', - b'Server: gunicorn/0.16.1\r\n', - b'transfer-encoding: chunked\r\n', - b'Connection: keep-alive\r\n\r\n', - b'4\r\n', - b'Wiki\r\n', - b'5\r\n', - b'pedia\r\n', - b'E\r\n', - b' in\r\n\r\nchunks.\r\n', - b'0\r\n', - b'\r\n' - ])) + self.parser.parse( + b''.join([ + b'HTTP/1.1 200 OK\r\n', + b'Content-Type: application/json\r\n', + b'Date: Wed, 22 May 2013 15:08:15 GMT\r\n', + b'Server: gunicorn/0.16.1\r\n', + b'transfer-encoding: chunked\r\n', + b'Connection: keep-alive\r\n\r\n', + b'4\r\n', + b'Wiki\r\n', + b'5\r\n', + b'pedia\r\n', + b'E\r\n', + b' in\r\n\r\nchunks.\r\n', + b'0\r\n', + b'\r\n', + ]), + ) self.assertEqual(self.parser.body, b'Wikipedia in\r\n\r\nchunks.') self.assertEqual(self.parser.state, httpParserStates.COMPLETE) @@ -448,7 +600,7 @@ def test_pipelined_response_parse(self) -> None: response = build_http_response( httpStatusCodes.OK, reason=b'OK', headers={ - b'Content-Length': b'15' + b'Content-Length': b'15', }, body=b'{"key":"value"}', ) @@ -461,7 +613,7 @@ def test_pipelined_chunked_response_parse(self) -> None: b'Transfer-Encoding': b'chunked', b'Content-Type': b'application/json', }, - body=b'f\r\n{"key":"value"}\r\n0\r\n\r\n' + body=b'f\r\n{"key":"value"}\r\n0\r\n\r\n', ) self.assert_pipeline_response(response) @@ -480,52 +632,69 @@ def assert_pipeline_response(self, response: bytes) -> None: self.assertEqual(parser.buffer, b'') def test_chunked_request_parse(self) -> None: - self.parser.parse(build_http_request( - httpMethods.POST, b'http://example.org/', - headers={ - b'Transfer-Encoding': b'chunked', - b'Content-Type': b'application/json', - }, - body=b'f\r\n{"key":"value"}\r\n0\r\n\r\n')) + self.parser.parse( + build_http_request( + httpMethods.POST, + b'http://example.org/', + headers={ + b'Transfer-Encoding': b'chunked', + b'Content-Type': b'application/json', + }, + body=b'f\r\n{"key":"value"}\r\n0\r\n\r\n', + ), + ) self.assertEqual(self.parser.body, b'{"key":"value"}') self.assertEqual(self.parser.state, httpParserStates.COMPLETE) - self.assertEqual(self.parser.build(), build_http_request( - httpMethods.POST, b'/', - headers={ - b'Transfer-Encoding': b'chunked', - b'Content-Type': b'application/json', - }, - body=b'f\r\n{"key":"value"}\r\n0\r\n\r\n')) + self.assertEqual( + self.parser.build(), build_http_request( + httpMethods.POST, + b'/', + headers={ + b'Transfer-Encoding': b'chunked', + b'Content-Type': b'application/json', + }, + body=b'f\r\n{"key":"value"}\r\n0\r\n\r\n', + ), + ) def test_is_http_1_1_keep_alive(self) -> None: - self.parser.parse(build_http_request( - httpMethods.GET, b'/' - )) + self.parser.parse( + build_http_request( + httpMethods.GET, b'/', + ), + ) self.assertTrue(self.parser.is_http_1_1_keep_alive()) def test_is_http_1_1_keep_alive_with_non_close_connection_header( - self) -> None: - self.parser.parse(build_http_request( - httpMethods.GET, b'/', - headers={ - b'Connection': b'keep-alive', - } - )) + self, + ) -> None: + self.parser.parse( + build_http_request( + httpMethods.GET, b'/', + headers={ + b'Connection': b'keep-alive', + }, + ), + ) self.assertTrue(self.parser.is_http_1_1_keep_alive()) def test_is_not_http_1_1_keep_alive_with_close_header(self) -> None: - self.parser.parse(build_http_request( - httpMethods.GET, b'/', - headers={ - b'Connection': b'close', - } - )) + self.parser.parse( + build_http_request( + httpMethods.GET, b'/', + headers={ + b'Connection': b'close', + }, + ), + ) self.assertFalse(self.parser.is_http_1_1_keep_alive()) def test_is_not_http_1_1_keep_alive_for_http_1_0(self) -> None: - self.parser.parse(build_http_request( - httpMethods.GET, b'/', protocol_version=b'HTTP/1.0', - )) + self.parser.parse( + build_http_request( + httpMethods.GET, b'/', protocol_version=b'HTTP/1.0', + ), + ) self.assertFalse(self.parser.is_http_1_1_keep_alive()) def test_paramiko_doc(self) -> None: @@ -536,3 +705,50 @@ def test_paramiko_doc(self) -> None: self.parser = HttpParser(httpParserTypes.RESPONSE_PARSER) self.parser.parse(response) self.assertEqual(self.parser.state, httpParserStates.COMPLETE) + + def test_request_factory(self) -> None: + r = HttpParser.request( + b'POST http://localhost:12345 HTTP/1.1' + CRLF + + b'key: value' + CRLF + + b'Content-Length: 13' + CRLF + CRLF + + b'Hello from py', + ) + self.assertEqual(r.host, b'localhost') + self.assertEqual(r.port, 12345) + self.assertEqual(r.path, None) + self.assertEqual(r.header(b'key'), b'value') + self.assertEqual(r.header(b'KEY'), b'value') + self.assertEqual(r.header(b'content-length'), b'13') + self.assertEqual(r.body, b'Hello from py') + + def test_response_factory(self) -> None: + r = HttpParser.response( + b'HTTP/1.1 200 OK\r\nkey: value\r\n\r\n', + ) + self.assertEqual(r.code, b'200') + self.assertEqual(r.reason, b'OK') + self.assertEqual(r.header(b'key'), b'value') + + def test_proxy_protocol(self) -> None: + r = HttpParser.request( + b'PROXY TCP4 192.168.0.1 192.168.0.11 56324 443' + CRLF + + b'GET / HTTP/1.1' + CRLF + + b'Host: 192.168.0.11' + CRLF + CRLF, + enable_proxy_protocol=True, + ) + self.assertTrue(r.protocol is not None) + assert r.protocol and r.protocol.version and \ + r.protocol.family and \ + r.protocol.source and \ + r.protocol.destination + self.assertEqual(r.protocol.version, 1) + self.assertEqual(r.protocol.family, b'TCP4') + self.assertEqual(r.protocol.source, (b'192.168.0.1', 56324)) + self.assertEqual(r.protocol.destination, (b'192.168.0.11', 443)) + + def test_proxy_protocol_not_for_response_parser(self) -> None: + with self.assertRaises(AssertionError): + HttpParser( + httpParserTypes.RESPONSE_PARSER, + enable_proxy_protocol=True, + ) diff --git a/tests/http/test_http_proxy.py b/tests/http/test_http_proxy.py index 1be5af9657..b1bc964059 100644 --- a/tests/http/test_http_proxy.py +++ b/tests/http/test_http_proxy.py @@ -8,88 +8,145 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -import unittest +import pytest import selectors -from unittest import mock + +from pytest_mock import MockerFixture from proxy.common.constants import DEFAULT_HTTP_PORT -from proxy.proxy import Proxy +from proxy.common.flag import FlagParser from proxy.core.connection import TcpClientConnection from proxy.http.proxy import HttpProxyPlugin -from proxy.http.handler import HttpProtocolHandler +from proxy.http import HttpProtocolHandler from proxy.http.exception import HttpProtocolException from proxy.common.utils import build_http_request -class TestHttpProxyPlugin(unittest.TestCase): +class TestHttpProxyPlugin: - @mock.patch('selectors.DefaultSelector') - @mock.patch('socket.fromfd') - def setUp(self, - mock_fromfd: mock.Mock, - mock_selector: mock.Mock) -> None: - self.mock_fromfd = mock_fromfd - self.mock_selector = mock_selector + @pytest.fixture(autouse=True) # type: ignore[misc] + def _setUp(self, mocker: MockerFixture) -> None: + self.mock_server_conn = mocker.patch( + 'proxy.http.proxy.server.TcpServerConnection', + ) + self.mock_selector = mocker.patch('selectors.DefaultSelector') + self.mock_fromfd = mocker.patch('socket.fromfd') self.fileno = 10 self._addr = ('127.0.0.1', 54382) - self.flags = Proxy.initialize() - self.plugin = mock.MagicMock() + self.flags = FlagParser.initialize(threaded=True) + self.plugin = mocker.MagicMock() self.flags.plugins = { b'HttpProtocolHandlerPlugin': [HttpProxyPlugin], - b'HttpProxyBasePlugin': [self.plugin] + b'HttpProxyBasePlugin': [self.plugin], } - self._conn = mock_fromfd.return_value + self._conn = self.mock_fromfd.return_value self.protocol_handler = HttpProtocolHandler( TcpClientConnection(self._conn, self._addr), - flags=self.flags) + flags=self.flags, + ) self.protocol_handler.initialize() def test_proxy_plugin_initialized(self) -> None: self.plugin.assert_called() - @mock.patch('proxy.http.proxy.server.TcpServerConnection') - def test_proxy_plugin_on_and_before_upstream_connection( - self, - mock_server_conn: mock.Mock) -> None: + @pytest.mark.asyncio # type: ignore[misc] + async def test_proxy_plugin_on_and_before_upstream_connection(self) -> None: + self.plugin.return_value.write_to_descriptors.return_value = False + self.plugin.return_value.read_from_descriptors.return_value = False self.plugin.return_value.before_upstream_connection.side_effect = lambda r: r self.plugin.return_value.handle_client_request.side_effect = lambda r: r + self.plugin.return_value.resolve_dns.return_value = None, None self._conn.recv.return_value = build_http_request( b'GET', b'http://upstream.host/not-found.html', headers={ - b'Host': b'upstream.host' - }) + b'Host': b'upstream.host', + }, + ) self.mock_selector.return_value.select.side_effect = [ - [(selectors.SelectorKey( - fileobj=self._conn, - fd=self._conn.fileno, - events=selectors.EVENT_READ, - data=None), selectors.EVENT_READ)], ] - - self.protocol_handler.run_once() - mock_server_conn.assert_called_with('upstream.host', DEFAULT_HTTP_PORT) + [( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=selectors.EVENT_READ, + data=None, + ), + selectors.EVENT_READ, + )], + ] + + await self.protocol_handler._run_once() + + self.mock_server_conn.assert_called_with( + 'upstream.host', DEFAULT_HTTP_PORT, + ) self.plugin.return_value.before_upstream_connection.assert_called() self.plugin.return_value.handle_client_request.assert_called() - @mock.patch('proxy.http.proxy.server.TcpServerConnection') - def test_proxy_plugin_before_upstream_connection_can_teardown( - self, - mock_server_conn: mock.Mock) -> None: + @pytest.mark.asyncio # type: ignore[misc] + async def test_proxy_plugin_before_upstream_connection_can_teardown(self) -> None: + self.plugin.return_value.write_to_descriptors.return_value = False + self.plugin.return_value.read_from_descriptors.return_value = False self.plugin.return_value.before_upstream_connection.side_effect = HttpProtocolException() self._conn.recv.return_value = build_http_request( b'GET', b'http://upstream.host/not-found.html', headers={ - b'Host': b'upstream.host' - }) + b'Host': b'upstream.host', + }, + ) self.mock_selector.return_value.select.side_effect = [ - [(selectors.SelectorKey( - fileobj=self._conn, - fd=self._conn.fileno, - events=selectors.EVENT_READ, - data=None), selectors.EVENT_READ)], ] + [( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=selectors.EVENT_READ, + data=None, + ), + selectors.EVENT_READ, + )], + ] - self.protocol_handler.run_once() + await self.protocol_handler._run_once() + self.mock_server_conn.assert_not_called() self.plugin.return_value.before_upstream_connection.assert_called() - mock_server_conn.assert_not_called() + + def test_proxy_plugin_plugins_can_teardown_from_write_to_descriptors(self) -> None: + pass + + def test_proxy_plugin_retries_on_ssl_want_write_error(self) -> None: + pass + + def test_proxy_plugin_broken_pipe_error_on_write_will_teardown(self) -> None: + pass + + def test_proxy_plugin_plugins_can_teardown_from_read_from_descriptors(self) -> None: + pass + + def test_proxy_plugin_retries_on_ssl_want_read_error(self) -> None: + pass + + def test_proxy_plugin_timeout_error_on_read_will_teardown(self) -> None: + pass + + def test_proxy_plugin_invokes_handle_pipeline_response(self) -> None: + pass + + def test_proxy_plugin_invokes_on_access_log(self) -> None: + pass + + def test_proxy_plugin_skips_server_teardown_when_client_closes_and_server_never_initialized(self) -> None: + pass + + def test_proxy_plugin_invokes_handle_client_data(self) -> None: + pass + + def test_proxy_plugin_handles_pipeline_response(self) -> None: + pass + + def test_proxy_plugin_invokes_resolve_dns(self) -> None: + pass + + def test_proxy_plugin_require_both_host_port_to_connect(self) -> None: + pass diff --git a/tests/http/test_http_proxy_tls_interception.py b/tests/http/test_http_proxy_tls_interception.py index ddb7987018..1fcfa71720 100644 --- a/tests/http/test_http_proxy_tls_interception.py +++ b/tests/http/test_http_proxy_tls_interception.py @@ -8,54 +8,45 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ +import ssl import uuid -import unittest import socket -import ssl +import pytest import selectors from typing import Any +from pytest_mock import MockerFixture from unittest import mock +from proxy.common.constants import DEFAULT_CA_FILE from proxy.core.connection import TcpClientConnection, TcpServerConnection -from proxy.http.handler import HttpProtocolHandler +from proxy.http import HttpProtocolHandler, httpMethods from proxy.http.proxy import HttpProxyPlugin -from proxy.http.methods import httpMethods from proxy.common.utils import build_http_request, bytes_ -from proxy.proxy import Proxy - - -class TestHttpProxyTlsInterception(unittest.TestCase): - - @mock.patch('ssl.wrap_socket') - @mock.patch('ssl.create_default_context') - @mock.patch('proxy.http.proxy.server.TcpServerConnection') - @mock.patch('proxy.http.proxy.server.gen_public_key') - @mock.patch('proxy.http.proxy.server.gen_csr') - @mock.patch('proxy.http.proxy.server.sign_csr') - @mock.patch('selectors.DefaultSelector') - @mock.patch('socket.fromfd') - def test_e2e( - self, - mock_fromfd: mock.Mock, - mock_selector: mock.Mock, - mock_sign_csr: mock.Mock, - mock_gen_csr: mock.Mock, - mock_gen_public_key: mock.Mock, - mock_server_conn: mock.Mock, - mock_ssl_context: mock.Mock, - mock_ssl_wrap: mock.Mock) -> None: +from proxy.common.flag import FlagParser + +from ..test_assertions import Assertions + + +class TestHttpProxyTlsInterception(Assertions): + + @pytest.mark.asyncio # type: ignore[misc] + async def test_e2e(self, mocker: MockerFixture) -> None: host, port = uuid.uuid4().hex, 443 netloc = '{0}:{1}'.format(host, port) - self.mock_fromfd = mock_fromfd - self.mock_selector = mock_selector - self.mock_sign_csr = mock_sign_csr - self.mock_gen_csr = mock_gen_csr - self.mock_gen_public_key = mock_gen_public_key - self.mock_server_conn = mock_server_conn - self.mock_ssl_context = mock_ssl_context - self.mock_ssl_wrap = mock_ssl_wrap + self.mock_fromfd = mocker.patch('socket.fromfd') + self.mock_selector = mocker.patch('selectors.DefaultSelector') + self.mock_sign_csr = mocker.patch('proxy.http.proxy.server.sign_csr') + self.mock_gen_csr = mocker.patch('proxy.http.proxy.server.gen_csr') + self.mock_gen_public_key = mocker.patch( + 'proxy.http.proxy.server.gen_public_key', + ) + self.mock_server_conn = mocker.patch( + 'proxy.http.proxy.server.TcpServerConnection', + ) + self.mock_ssl_context = mocker.patch('ssl.create_default_context') + self.mock_ssl_wrap = mocker.patch('ssl.wrap_socket') self.mock_sign_csr.return_value = True self.mock_gen_csr.return_value = True @@ -74,17 +65,19 @@ def mock_connection() -> Any: # Do not mock the original wrap method self.mock_server_conn.return_value.wrap.side_effect = \ lambda x, y: TcpServerConnection.wrap( - self.mock_server_conn.return_value, x, y) + self.mock_server_conn.return_value, x, y, + ) type(self.mock_server_conn.return_value).connection = \ mock.PropertyMock(side_effect=mock_connection) self.fileno = 10 self._addr = ('127.0.0.1', 54382) - self.flags = Proxy.initialize( + self.flags = FlagParser.initialize( ca_cert_file='ca-cert.pem', ca_key_file='ca-key.pem', - ca_signing_key_file='ca-signing-key.pem' + ca_signing_key_file='ca-signing-key.pem', + threaded=True, ) self.plugin = mock.MagicMock() self.proxy_plugin = mock.MagicMock() @@ -92,10 +85,11 @@ def mock_connection() -> Any: b'HttpProtocolHandlerPlugin': [self.plugin, HttpProxyPlugin], b'HttpProxyBasePlugin': [self.proxy_plugin], } - self._conn = mock_fromfd.return_value + self._conn = self.mock_fromfd.return_value self.protocol_handler = HttpProtocolHandler( TcpClientConnection(self._conn, self._addr), - flags=self.flags) + flags=self.flags, + ) self.protocol_handler.initialize() self.plugin.assert_called() @@ -105,87 +99,110 @@ def mock_connection() -> Any: self.assertEqual(self.proxy_plugin.call_args[0][1], self.flags) self.assertEqual( self.proxy_plugin.call_args[0][2].connection, - self._conn) + self._conn, + ) connect_request = build_http_request( httpMethods.CONNECT, bytes_(netloc), headers={ b'Host': bytes_(netloc), - }) + }, + ) self._conn.recv.return_value = connect_request # Prepare mocked HttpProtocolHandlerPlugin + async def asyncReturnBool(val: bool) -> bool: + return val self.plugin.return_value.get_descriptors.return_value = ([], []) - self.plugin.return_value.write_to_descriptors.return_value = False - self.plugin.return_value.read_from_descriptors.return_value = False + self.plugin.return_value.write_to_descriptors.return_value = asyncReturnBool(False) + self.plugin.return_value.read_from_descriptors.return_value = asyncReturnBool(False) self.plugin.return_value.on_client_data.side_effect = lambda raw: raw self.plugin.return_value.on_request_complete.return_value = False self.plugin.return_value.on_response_chunk.side_effect = lambda chunk: chunk self.plugin.return_value.on_client_connection_close.return_value = None # Prepare mocked HttpProxyBasePlugin + self.proxy_plugin.return_value.write_to_descriptors.return_value = False + self.proxy_plugin.return_value.read_from_descriptors.return_value = False self.proxy_plugin.return_value.before_upstream_connection.side_effect = lambda r: r self.proxy_plugin.return_value.handle_client_request.side_effect = lambda r: r + self.proxy_plugin.return_value.resolve_dns.return_value = None, None self.mock_selector.return_value.select.side_effect = [ - [(selectors.SelectorKey( - fileobj=self._conn, - fd=self._conn.fileno, - events=selectors.EVENT_READ, - data=None), selectors.EVENT_READ)], ] - - self.protocol_handler.run_once() + [( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=selectors.EVENT_READ, + data=None, + ), + selectors.EVENT_READ, + )], + ] + + await self.protocol_handler._run_once() # Assert our mocked plugins invocations self.plugin.return_value.get_descriptors.assert_called() self.plugin.return_value.write_to_descriptors.assert_called_with([]) self.plugin.return_value.on_client_data.assert_called_with( - connect_request) + connect_request, + ) self.plugin.return_value.on_request_complete.assert_called() self.plugin.return_value.read_from_descriptors.assert_called_with([ - self._conn]) + self._conn.fileno(), + ]) self.proxy_plugin.return_value.before_upstream_connection.assert_called() self.proxy_plugin.return_value.handle_client_request.assert_called() self.mock_server_conn.assert_called_with(host, port) self.mock_server_conn.return_value.connection.setblocking.assert_called_with( - False) + False, + ) self.mock_ssl_context.assert_called_with( - ssl.Purpose.SERVER_AUTH, cafile=None) + ssl.Purpose.SERVER_AUTH, cafile=str(DEFAULT_CA_FILE), + ) # self.assertEqual(self.mock_ssl_context.return_value.options, # ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1 | # ssl.OP_NO_TLSv1_1) self.assertEqual(plain_connection.setblocking.call_count, 2) self.mock_ssl_context.return_value.wrap_socket.assert_called_with( - plain_connection, server_hostname=host) + plain_connection, server_hostname=host, + ) self.assertEqual(self.mock_sign_csr.call_count, 1) self.assertEqual(self.mock_gen_csr.call_count, 1) self.assertEqual(self.mock_gen_public_key.call_count, 1) self.assertEqual(ssl_connection.setblocking.call_count, 1) self.assertEqual( self.mock_server_conn.return_value._conn, - ssl_connection) + ssl_connection, + ) self._conn.send.assert_called_with( - HttpProxyPlugin.PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT) + HttpProxyPlugin.PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT, + ) assert self.flags.ca_cert_dir is not None self.mock_ssl_wrap.assert_called_with( self._conn, server_side=True, keyfile=self.flags.ca_signing_key_file, certfile=HttpProxyPlugin.generated_cert_file_path( - self.flags.ca_cert_dir, host), - ssl_version=ssl.PROTOCOL_TLS + self.flags.ca_cert_dir, host, + ), + ssl_version=ssl.PROTOCOL_TLS, ) self.assertEqual(self._conn.setblocking.call_count, 2) self.assertEqual( - self.protocol_handler.client.connection, - self.mock_ssl_wrap.return_value) + self.protocol_handler.work.connection, + self.mock_ssl_wrap.return_value, + ) # Assert connection references for all other plugins is updated self.assertEqual( self.plugin.return_value.client._conn, - self.mock_ssl_wrap.return_value) + self.mock_ssl_wrap.return_value, + ) self.assertEqual( self.proxy_plugin.return_value.client._conn, - self.mock_ssl_wrap.return_value) + self.mock_ssl_wrap.return_value, + ) diff --git a/tests/http/test_protocol_handler.py b/tests/http/test_protocol_handler.py index a79e6b4ad8..ca6cce944e 100644 --- a/tests/http/test_protocol_handler.py +++ b/tests/http/test_protocol_handler.py @@ -8,14 +8,16 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -import unittest -import selectors import base64 +import pytest +import selectors -from typing import cast from unittest import mock +from pytest_mock import MockerFixture +from typing import cast, Any -from proxy.proxy import Proxy +from proxy.common.plugins import Plugins +from proxy.common.flag import FlagParser from proxy.common.version import __version__ from proxy.common.utils import bytes_ from proxy.common.constants import CRLF, PLUGIN_HTTP_PROXY, PLUGIN_PROXY_AUTH, PLUGIN_WEB_SERVER @@ -24,51 +26,144 @@ from proxy.http.proxy import HttpProxyPlugin from proxy.http.parser import httpParserStates, httpParserTypes from proxy.http.exception import ProxyAuthenticationFailed, ProxyConnectionFailed -from proxy.http.handler import HttpProtocolHandler +from proxy.http import HttpProtocolHandler + +from ..test_assertions import Assertions -class TestHttpProtocolHandler(unittest.TestCase): +def mock_selector_for_client_read(self: Any) -> None: + self.mock_selector.return_value.select.return_value = [ + ( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=selectors.EVENT_READ, + data=None, + ), + selectors.EVENT_READ, + ), + ] + + +class TestHttpProtocolHandlerWithoutServerMock(Assertions): + + @pytest.fixture(autouse=True) # type: ignore[misc] + def _setUp(self, mocker: MockerFixture) -> None: + self.mock_fromfd = mocker.patch('socket.fromfd') + self.mock_selector = mocker.patch('selectors.DefaultSelector') - @mock.patch('selectors.DefaultSelector') - @mock.patch('socket.fromfd') - def setUp(self, - mock_fromfd: mock.Mock, - mock_selector: mock.Mock) -> None: self.fileno = 10 self._addr = ('127.0.0.1', 54382) - self._conn = mock_fromfd.return_value + self._conn = self.mock_fromfd.return_value self.http_server_port = 65535 - self.flags = Proxy.initialize() - self.flags.plugins = Proxy.load_plugins([ + self.flags = FlagParser.initialize(threaded=True) + self.flags.plugins = Plugins.load([ bytes_(PLUGIN_HTTP_PROXY), bytes_(PLUGIN_WEB_SERVER), ]) - self.mock_selector = mock_selector self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), flags=self.flags) + TcpClientConnection(self._conn, self._addr), + flags=self.flags, + ) + self.protocol_handler.initialize() + + @pytest.mark.asyncio # type: ignore[misc] + async def test_proxy_connection_failed(self) -> None: + mock_selector_for_client_read(self) + self._conn.recv.return_value = CRLF.join([ + b'GET http://unknown.domain HTTP/1.1', + b'Host: unknown.domain', + CRLF, + ]) + await self.protocol_handler._run_once() + self.assertEqual( + self.protocol_handler.work.buffer[0], + ProxyConnectionFailed.RESPONSE_PKT, + ) + + @pytest.mark.asyncio # type: ignore[misc] + async def test_proxy_authentication_failed(self) -> None: + self._conn = self.mock_fromfd.return_value + mock_selector_for_client_read(self) + flags = FlagParser.initialize( + auth_code=base64.b64encode(b'user:pass'), + threaded=True, + ) + flags.plugins = Plugins.load([ + bytes_(PLUGIN_HTTP_PROXY), + bytes_(PLUGIN_WEB_SERVER), + bytes_(PLUGIN_PROXY_AUTH), + ]) + self.protocol_handler = HttpProtocolHandler( + TcpClientConnection(self._conn, self._addr), flags=flags, + ) self.protocol_handler.initialize() + self._conn.recv.return_value = CRLF.join([ + b'GET http://abhinavsingh.com HTTP/1.1', + b'Host: abhinavsingh.com', + CRLF, + ]) + await self.protocol_handler._run_once() + self.assertEqual( + self.protocol_handler.work.buffer[0], + ProxyAuthenticationFailed.RESPONSE_PKT, + ) + + +class TestHttpProtocolHandler(Assertions): + + @pytest.fixture(autouse=True) # type: ignore[misc] + def _setUp(self, mocker: MockerFixture) -> None: + self.mock_fromfd = mocker.patch('socket.fromfd') + self.mock_selector = mocker.patch('selectors.DefaultSelector') + self.mock_server_connection = mocker.patch( + 'proxy.http.proxy.server.TcpServerConnection', + ) + + self.fileno = 10 + self._addr = ('127.0.0.1', 54382) + self._conn = self.mock_fromfd.return_value + + self.http_server_port = 65535 + self.flags = FlagParser.initialize(threaded=True) + self.flags.plugins = Plugins.load([ + bytes_(PLUGIN_HTTP_PROXY), + bytes_(PLUGIN_WEB_SERVER), + ]) - @mock.patch('proxy.http.proxy.server.TcpServerConnection') - def test_http_get(self, mock_server_connection: mock.Mock) -> None: - server = mock_server_connection.return_value + self.protocol_handler = HttpProtocolHandler( + TcpClientConnection(self._conn, self._addr), + flags=self.flags, + ) + self.protocol_handler.initialize() + + @pytest.mark.asyncio # type: ignore[misc] + async def test_http_get(self) -> None: + server = self.mock_server_connection.return_value server.connect.return_value = True server.buffer_size.return_value = 0 - self.mock_selector_for_client_read_read_server_write( - self.mock_selector, server) + + self.mock_selector_for_client_read_and_server_write(server) # Send request line assert self.http_server_port is not None - self._conn.recv.return_value = (b'GET http://localhost:%d HTTP/1.1' % - self.http_server_port) + CRLF - self.protocol_handler.run_once() + self._conn.recv.return_value = ( + b'GET http://localhost:%d HTTP/1.1' % + self.http_server_port + ) + CRLF + + await self.protocol_handler._run_once() + self.assertEqual( self.protocol_handler.request.state, - httpParserStates.LINE_RCVD) + httpParserStates.LINE_RCVD, + ) self.assertNotEqual( self.protocol_handler.request.state, - httpParserStates.COMPLETE) + httpParserStates.COMPLETE, + ) # Send headers and blank line, thus completing HTTP request assert self.http_server_port is not None @@ -77,34 +172,41 @@ def test_http_get(self, mock_server_connection: mock.Mock) -> None: b'Host: localhost:%d' % self.http_server_port, b'Accept: */*', b'Proxy-Connection: Keep-Alive', - CRLF + CRLF, ]) - self.assert_data_queued(mock_server_connection, server) - self.protocol_handler.run_once() + await self.assert_data_queued(server) + await self.protocol_handler._run_once() server.flush.assert_called_once() - def assert_tunnel_response( - self, mock_server_connection: mock.Mock, server: mock.Mock) -> None: - self.protocol_handler.run_once() + async def assert_tunnel_response( + self, + server: mock.Mock, + ) -> None: + await self.protocol_handler._run_once() self.assertTrue( - cast(HttpProxyPlugin, self.protocol_handler.plugins['HttpProxyPlugin']).server is not None) + cast( + HttpProxyPlugin, + self.protocol_handler.plugins['HttpProxyPlugin'], + ).upstream is not None, + ) self.assertEqual( - self.protocol_handler.client.buffer[0], - HttpProxyPlugin.PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT) - mock_server_connection.assert_called_once() + self.protocol_handler.work.buffer[0], + HttpProxyPlugin.PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT, + ) + self.mock_server_connection.assert_called_once() server.connect.assert_called_once() server.queue.assert_not_called() server.closed = False parser = HttpParser(httpParserTypes.RESPONSE_PARSER) - parser.parse(self.protocol_handler.client.buffer[0].tobytes()) + parser.parse(self.protocol_handler.work.buffer[0].tobytes()) self.assertEqual(parser.state, httpParserStates.COMPLETE) assert parser.code is not None self.assertEqual(int(parser.code), 200) - @mock.patch('proxy.http.proxy.server.TcpServerConnection') - def test_http_tunnel(self, mock_server_connection: mock.Mock) -> None: - server = mock_server_connection.return_value + @pytest.mark.asyncio # type: ignore[misc] + async def test_http_tunnel(self) -> None: + server = self.mock_server_connection.return_value server.connect.return_value = True def has_buffer() -> bool: @@ -112,26 +214,50 @@ def has_buffer() -> bool: server.has_buffer.side_effect = has_buffer self.mock_selector.return_value.select.side_effect = [ - [(selectors.SelectorKey( - fileobj=self._conn, - fd=self._conn.fileno, - events=selectors.EVENT_READ, - data=None), selectors.EVENT_READ), ], - [(selectors.SelectorKey( - fileobj=self._conn, - fd=self._conn.fileno, - events=0, - data=None), selectors.EVENT_WRITE), ], - [(selectors.SelectorKey( - fileobj=self._conn, - fd=self._conn.fileno, - events=selectors.EVENT_READ, - data=None), selectors.EVENT_READ), ], - [(selectors.SelectorKey( - fileobj=server.connection, - fd=server.connection.fileno, - events=0, - data=None), selectors.EVENT_WRITE), ], + [ + ( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=selectors.EVENT_READ, + data=None, + ), + selectors.EVENT_READ, + ), + ], + [ + ( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=0, + data=None, + ), + selectors.EVENT_WRITE, + ), + ], + [ + ( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=selectors.EVENT_READ, + data=None, + ), + selectors.EVENT_READ, + ), + ], + [ + ( + selectors.SelectorKey( + fileobj=server.connection.fileno(), + fd=server.connection.fileno(), + events=0, + data=None, + ), + selectors.EVENT_WRITE, + ), + ], ] assert self.http_server_port is not None @@ -140,95 +266,55 @@ def has_buffer() -> bool: b'Host: localhost:%d' % self.http_server_port, b'User-Agent: proxy.py/%s' % bytes_(__version__), b'Proxy-Connection: Keep-Alive', - CRLF + CRLF, ]) - self.assert_tunnel_response(mock_server_connection, server) + await self.assert_tunnel_response(server) # Dispatch tunnel established response to client - self.protocol_handler.run_once() - self.assert_data_queued_to_server(server) + await self.protocol_handler._run_once() + await self.assert_data_queued_to_server(server) - self.protocol_handler.run_once() + await self.protocol_handler._run_once() self.assertEqual(server.queue.call_count, 1) server.flush.assert_called_once() - def test_proxy_connection_failed(self) -> None: - self.mock_selector_for_client_read(self.mock_selector) - self._conn.recv.return_value = CRLF.join([ - b'GET http://unknown.domain HTTP/1.1', - b'Host: unknown.domain', - CRLF - ]) - self.protocol_handler.run_once() - self.assertEqual( - self.protocol_handler.client.buffer[0], - ProxyConnectionFailed.RESPONSE_PKT) + @pytest.mark.asyncio # type: ignore[misc] + async def test_authenticated_proxy_http_get(self) -> None: + self._conn = self.mock_fromfd.return_value + mock_selector_for_client_read(self) - @mock.patch('selectors.DefaultSelector') - @mock.patch('socket.fromfd') - def test_proxy_authentication_failed( - self, - mock_fromfd: mock.Mock, - mock_selector: mock.Mock) -> None: - self._conn = mock_fromfd.return_value - self.mock_selector_for_client_read(mock_selector) - flags = Proxy.initialize( - auth_code=base64.b64encode(b'user:pass')) - flags.plugins = Proxy.load_plugins([ - bytes_(PLUGIN_HTTP_PROXY), - bytes_(PLUGIN_WEB_SERVER), - bytes_(PLUGIN_PROXY_AUTH), - ]) - self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), flags=flags) - self.protocol_handler.initialize() - self._conn.recv.return_value = CRLF.join([ - b'GET http://abhinavsingh.com HTTP/1.1', - b'Host: abhinavsingh.com', - CRLF - ]) - self.protocol_handler.run_once() - self.assertEqual( - self.protocol_handler.client.buffer[0], - ProxyAuthenticationFailed.RESPONSE_PKT) - - @mock.patch('selectors.DefaultSelector') - @mock.patch('socket.fromfd') - @mock.patch('proxy.http.proxy.server.TcpServerConnection') - def test_authenticated_proxy_http_get( - self, mock_server_connection: mock.Mock, - mock_fromfd: mock.Mock, - mock_selector: mock.Mock) -> None: - self._conn = mock_fromfd.return_value - self.mock_selector_for_client_read(mock_selector) - - server = mock_server_connection.return_value + server = self.mock_server_connection.return_value server.connect.return_value = True server.buffer_size.return_value = 0 - flags = Proxy.initialize( - auth_code=base64.b64encode(b'user:pass')) - flags.plugins = Proxy.load_plugins([ + flags = FlagParser.initialize( + auth_code=base64.b64encode(b'user:pass'), + threaded=True, + ) + flags.plugins = Plugins.load([ bytes_(PLUGIN_HTTP_PROXY), bytes_(PLUGIN_WEB_SERVER), ]) self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), flags=flags) + TcpClientConnection(self._conn, self._addr), flags=flags, + ) self.protocol_handler.initialize() assert self.http_server_port is not None self._conn.recv.return_value = b'GET http://localhost:%d HTTP/1.1' % self.http_server_port - self.protocol_handler.run_once() + await self.protocol_handler._run_once() self.assertEqual( self.protocol_handler.request.state, - httpParserStates.INITIALIZED) + httpParserStates.INITIALIZED, + ) self._conn.recv.return_value = CRLF - self.protocol_handler.run_once() + await self.protocol_handler._run_once() self.assertEqual( self.protocol_handler.request.state, - httpParserStates.LINE_RCVD) + httpParserStates.LINE_RCVD, + ) assert self.http_server_port is not None self._conn.recv.return_value = CRLF.join([ @@ -237,33 +323,30 @@ def test_authenticated_proxy_http_get( b'Accept: */*', b'Proxy-Connection: Keep-Alive', b'Proxy-Authorization: Basic dXNlcjpwYXNz', - CRLF + CRLF, ]) - self.assert_data_queued(mock_server_connection, server) - - @mock.patch('selectors.DefaultSelector') - @mock.patch('socket.fromfd') - @mock.patch('proxy.http.proxy.server.TcpServerConnection') - def test_authenticated_proxy_http_tunnel( - self, mock_server_connection: mock.Mock, - mock_fromfd: mock.Mock, - mock_selector: mock.Mock) -> None: - server = mock_server_connection.return_value + await self.assert_data_queued(server) + + @pytest.mark.asyncio # type: ignore[misc] + async def test_authenticated_proxy_http_tunnel(self) -> None: + server = self.mock_server_connection.return_value server.connect.return_value = True server.buffer_size.return_value = 0 - self._conn = mock_fromfd.return_value - self.mock_selector_for_client_read_read_server_write( - mock_selector, server) - - flags = Proxy.initialize( - auth_code=base64.b64encode(b'user:pass')) - flags.plugins = Proxy.load_plugins([ + self._conn = self.mock_fromfd.return_value + self.mock_selector_for_client_read_and_server_write(server) + + flags = FlagParser.initialize( + auth_code=base64.b64encode(b'user:pass'), + threaded=True, + ) + flags.plugins = Plugins.load([ bytes_(PLUGIN_HTTP_PROXY), - bytes_(PLUGIN_WEB_SERVER) + bytes_(PLUGIN_WEB_SERVER), ]) self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), flags=flags) + TcpClientConnection(self._conn, self._addr), flags=flags, + ) self.protocol_handler.initialize() assert self.http_server_port is not None @@ -273,42 +356,63 @@ def test_authenticated_proxy_http_tunnel( b'User-Agent: proxy.py/%s' % bytes_(__version__), b'Proxy-Connection: Keep-Alive', b'Proxy-Authorization: Basic dXNlcjpwYXNz', - CRLF + CRLF, ]) - self.assert_tunnel_response(mock_server_connection, server) - self.protocol_handler.client.flush() - self.assert_data_queued_to_server(server) + await self.assert_tunnel_response(server) + self.protocol_handler.work.flush() + await self.assert_data_queued_to_server(server) - self.protocol_handler.run_once() + await self.protocol_handler._run_once() server.flush.assert_called_once() - def mock_selector_for_client_read_read_server_write( - self, mock_selector: mock.Mock, server: mock.Mock) -> None: - mock_selector.return_value.select.side_effect = [ - [(selectors.SelectorKey( - fileobj=self._conn, - fd=self._conn.fileno, - events=selectors.EVENT_READ, - data=None), selectors.EVENT_READ), ], - [(selectors.SelectorKey( - fileobj=self._conn, - fd=self._conn.fileno, - events=0, - data=None), selectors.EVENT_READ), ], - [(selectors.SelectorKey( - fileobj=server.connection, - fd=server.connection.fileno, - events=0, - data=None), selectors.EVENT_WRITE), ], + def mock_selector_for_client_read_and_server_write( + self, server: mock.Mock, + ) -> None: + self.mock_selector.return_value.select.side_effect = [ + [ + ( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=selectors.EVENT_READ, + data=None, + ), + selectors.EVENT_READ, + ), + ], + [ + ( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=0, + data=None, + ), + selectors.EVENT_READ, + ), + ], + [ + ( + selectors.SelectorKey( + fileobj=server.connection.fileno(), + fd=server.connection.fileno(), + events=0, + data=None, + ), + selectors.EVENT_WRITE, + ), + ], ] - def assert_data_queued( - self, mock_server_connection: mock.Mock, server: mock.Mock) -> None: - self.protocol_handler.run_once() + async def assert_data_queued( + self, server: mock.Mock, + ) -> None: + await self.protocol_handler._run_once() self.assertEqual( self.protocol_handler.request.state, - httpParserStates.COMPLETE) - mock_server_connection.assert_called_once() + httpParserStates.COMPLETE, + ) + self.mock_server_connection.assert_called_once() server.connect.assert_called_once() server.closed = False assert self.http_server_port is not None @@ -318,35 +422,29 @@ def assert_data_queued( b'Host: localhost:%d' % self.http_server_port, b'Accept: */*', b'Via: 1.1 proxy.py v%s' % bytes_(__version__), - CRLF + CRLF, ]) - server.queue.assert_called_once_with(pkt) + server.queue.assert_called_once() + self.assertEqual(server.queue.call_args_list[0][0][0].tobytes(), pkt) server.buffer_size.return_value = len(pkt) - def assert_data_queued_to_server(self, server: mock.Mock) -> None: + async def assert_data_queued_to_server(self, server: mock.Mock) -> None: assert self.http_server_port is not None self.assertEqual( self._conn.send.call_args[0][0], - HttpProxyPlugin.PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT) + HttpProxyPlugin.PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT, + ) pkt = CRLF.join([ b'GET / HTTP/1.1', b'Host: localhost:%d' % self.http_server_port, b'User-Agent: proxy.py/%s' % bytes_(__version__), - CRLF + CRLF, ]) self._conn.recv.return_value = pkt - self.protocol_handler.run_once() + await self.protocol_handler._run_once() server.queue.assert_called_once_with(pkt) server.buffer_size.return_value = len(pkt) server.flush.assert_not_called() - - def mock_selector_for_client_read(self, mock_selector: mock.Mock) -> None: - mock_selector.return_value.select.return_value = [( - selectors.SelectorKey( - fileobj=self._conn, - fd=self._conn.fileno, - events=selectors.EVENT_READ, - data=None), selectors.EVENT_READ), ] diff --git a/tests/http/test_proxy_protocol.py b/tests/http/test_proxy_protocol.py new file mode 100644 index 0000000000..b6701abfb7 --- /dev/null +++ b/tests/http/test_proxy_protocol.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import unittest + +from proxy.http.parser import ProxyProtocol, PROXY_PROTOCOL_V2_SIGNATURE + + +class TestProxyProtocol(unittest.TestCase): + + def setUp(self) -> None: + self.protocol = ProxyProtocol() + + def test_v1(self) -> None: + self.protocol.parse(b'PROXY TCP6 ::1 ::1 64665 8899') + self.assertEqual(self.protocol.version, 1) + self.assertEqual(self.protocol.family, b'TCP6') + self.assertEqual(self.protocol.source, (b'::1', 64665)) + self.assertEqual(self.protocol.destination, (b'::1', 8899)) + + def test_v1_example_from_spec(self) -> None: + self.protocol.parse(b'PROXY TCP4 192.168.0.1 192.168.0.11 56324 443') + self.assertEqual(self.protocol.version, 1) + self.assertEqual(self.protocol.family, b'TCP4') + self.assertEqual(self.protocol.source, (b'192.168.0.1', 56324)) + self.assertEqual(self.protocol.destination, (b'192.168.0.11', 443)) + + def test_v1_worst_case_ipv4_from_spec(self) -> None: + self.protocol.parse( + b'PROXY TCP4 255.255.255.255 255.255.255.255 65535 65535', + ) + self.assertEqual(self.protocol.version, 1) + self.assertEqual(self.protocol.family, b'TCP4') + self.assertEqual(self.protocol.source, (b'255.255.255.255', 65535)) + self.assertEqual( + self.protocol.destination, + (b'255.255.255.255', 65535), + ) + + def test_v1_worst_case_ipv6_from_spec(self) -> None: + self.protocol.parse( + b'PROXY TCP6 ffff:f...f:ffff ffff:f...f:ffff 65535 65535', + ) + self.assertEqual(self.protocol.version, 1) + self.assertEqual(self.protocol.family, b'TCP6') + self.assertEqual(self.protocol.source, (b'ffff:f...f:ffff', 65535)) + self.assertEqual( + self.protocol.destination, + (b'ffff:f...f:ffff', 65535), + ) + + def test_v1_worst_case_unknown_from_spec(self) -> None: + self.protocol.parse( + b'PROXY UNKNOWN ffff:f...f:ffff ffff:f...f:ffff 65535 65535', + ) + self.assertEqual(self.protocol.version, 1) + self.assertEqual(self.protocol.family, b'UNKNOWN') + self.assertEqual(self.protocol.source, (b'ffff:f...f:ffff', 65535)) + self.assertEqual( + self.protocol.destination, + (b'ffff:f...f:ffff', 65535), + ) + + def test_v1_unknown_with_no_src_dst(self) -> None: + self.protocol.parse(b'PROXY UNKNOWN') + self.assertEqual(self.protocol.version, 1) + self.assertEqual(self.protocol.family, b'UNKNOWN') + self.assertEqual(self.protocol.source, None) + self.assertEqual(self.protocol.destination, None) + + def test_v2_not_implemented(self) -> None: + with self.assertRaises(NotImplementedError): + self.protocol.parse(PROXY_PROTOCOL_V2_SIGNATURE) + self.assertEqual(self.protocol.version, 2) + + def test_unknown_value_error(self) -> None: + with self.assertRaises(ValueError): + self.protocol.parse(PROXY_PROTOCOL_V2_SIGNATURE[:10]) + self.assertEqual(self.protocol.version, None) diff --git a/tests/http/test_url.py b/tests/http/test_url.py new file mode 100644 index 0000000000..de8ec0e71a --- /dev/null +++ b/tests/http/test_url.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import unittest + +from proxy.http import Url + + +class TestUrl(unittest.TestCase): + + def test_url_str(self) -> None: + url = Url.from_bytes(b'localhost') + self.assertEqual(str(url), 'localhost') + url = Url.from_bytes(b'/') + self.assertEqual(str(url), '/') + url = Url.from_bytes(b'http://httpbin.org/get') + self.assertEqual(str(url), 'http://httpbin.org/get') + url = Url.from_bytes(b'httpbin.org:443') + self.assertEqual(str(url), 'httpbin.org:443') + url = Url.from_bytes('รฅโˆซรง.com'.encode('utf-8')) + self.assertEqual(str(url), 'รฅโˆซรง.com') + url = Url.from_bytes(b'https://example.com/path/dir/?a=b&c=d#p=q') + self.assertEqual(str(url), 'https://example.com/path/dir/?a=b&c=d#p=q') + url = Url.from_bytes(b'http://localhost:12345/v1/users/') + self.assertEqual(str(url), 'http://localhost:12345/v1/users/') + + def test_just_domain_name_url(self) -> None: + url = Url.from_bytes(b'localhost') + self.assertEqual(url.scheme, None) + self.assertEqual(url.hostname, b'localhost') + self.assertEqual(url.port, None) + self.assertEqual(url.remainder, None) + + def test_web_server_url(self) -> None: + url = Url.from_bytes(b'/') + self.assertEqual(url.scheme, None) + self.assertEqual(url.hostname, None) + self.assertEqual(url.port, None) + self.assertEqual(url.remainder, b'/') + + def test_http_proxy_url(self) -> None: + url = Url.from_bytes(b'http://httpbin.org/get') + self.assertEqual(url.scheme, b'http') + self.assertEqual(url.hostname, b'httpbin.org') + self.assertEqual(url.port, None) + self.assertEqual(url.remainder, b'/get') + + def test_https_connect_url(self) -> None: + url = Url.from_bytes(b'httpbin.org:443') + self.assertEqual(url.scheme, None) + self.assertEqual(url.hostname, b'httpbin.org') + self.assertEqual(url.port, 443) + self.assertEqual(url.remainder, None) + + def test_https_connect_with_ipv6_url(self) -> None: + url = Url.from_bytes(b'[::]:443') + self.assertEqual(url.scheme, None) + self.assertEqual(url.hostname, b'[::]') + self.assertEqual(url.port, 443) + self.assertEqual(url.remainder, None) + + def test_https_connect_with_ipv6_malformed_url(self) -> None: + url = Url.from_bytes(b':::443') + self.assertEqual(url.scheme, None) + self.assertEqual(url.hostname, b'[::]') + self.assertEqual(url.port, 443) + self.assertEqual(url.remainder, None) + + def test_http_ipv6_url(self) -> None: + url = Url.from_bytes(b'http://[::]') + self.assertEqual(url.scheme, b'http') + self.assertEqual(url.hostname, b'[::]') + self.assertEqual(url.port, None) + self.assertEqual(url.remainder, None) + + def test_http_ipv6_with_port_url(self) -> None: + url = Url.from_bytes(b'http://[::]:443') + self.assertEqual(url.scheme, b'http') + self.assertEqual(url.hostname, b'[::]') + self.assertEqual(url.port, 443) + self.assertEqual(url.remainder, None) + + def test_unicode_url(self) -> None: + url = Url.from_bytes('รฅโˆซรง.com'.encode('utf-8')) + self.assertEqual(url.scheme, None) + self.assertEqual(url.hostname, 'รฅโˆซรง.com'.encode('utf-8')) + self.assertEqual(url.port, None) + self.assertEqual(url.remainder, None) + + def test_full_url(self) -> None: + url = Url.from_bytes(b'https://example.com/path/dir/?a=b&c=d#p=q') + self.assertEqual(url.scheme, b'https') + self.assertEqual(url.hostname, b'example.com') + self.assertEqual(url.port, None) + self.assertEqual(url.remainder, b'/path/dir/?a=b&c=d#p=q') + + def test_no_trailing_slash_url(self) -> None: + url = Url.from_bytes(b'http://localhost:12345') + self.assertEqual(url.scheme, b'http') + self.assertEqual(url.hostname, b'localhost') + self.assertEqual(url.port, 12345) + self.assertEqual(url.remainder, None) + + def test_trailing_slash_url(self) -> None: + url = Url.from_bytes(b'http://localhost:12345/v1/users/') + self.assertEqual(url.scheme, b'http') + self.assertEqual(url.hostname, b'localhost') + self.assertEqual(url.port, 12345) + self.assertEqual(url.remainder, b'/v1/users/') diff --git a/tests/http/test_web_server.py b/tests/http/test_web_server.py index cdb3592633..54a1d49060 100644 --- a/tests/http/test_web_server.py +++ b/tests/http/test_web_server.py @@ -8,254 +8,362 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -import gzip import os +import gzip +import pytest import tempfile -import unittest import selectors -from unittest import mock -from proxy.proxy import Proxy +from typing import Any +from pytest_mock import MockerFixture +# from unittest import mock + +from proxy.common.plugins import Plugins +from proxy.common.flag import FlagParser from proxy.core.connection import TcpClientConnection -from proxy.http.handler import HttpProtocolHandler -from proxy.http.parser import httpParserStates -from proxy.common.utils import build_http_response, build_http_request, bytes_, text_ +from proxy.http import HttpProtocolHandler +from proxy.http.parser import HttpParser, httpParserStates, httpParserTypes +from proxy.common.utils import build_http_response, build_http_request, bytes_ from proxy.common.constants import CRLF, PLUGIN_HTTP_PROXY, PLUGIN_PAC_FILE, PLUGIN_WEB_SERVER, PROXY_PY_DIR from proxy.http.server import HttpWebServerPlugin +from ..test_assertions import Assertions -class TestWebServerPlugin(unittest.TestCase): - @mock.patch('selectors.DefaultSelector') - @mock.patch('socket.fromfd') - def setUp(self, mock_fromfd: mock.Mock, mock_selector: mock.Mock) -> None: - self.fileno = 10 - self._addr = ('127.0.0.1', 54382) - self._conn = mock_fromfd.return_value - self.mock_selector = mock_selector - self.flags = Proxy.initialize() - self.flags.plugins = Proxy.load_plugins([ - bytes_(PLUGIN_HTTP_PROXY), - bytes_(PLUGIN_WEB_SERVER), - ]) - self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), - flags=self.flags) - self.protocol_handler.initialize() +PAC_FILE_PATH = os.path.join( + os.path.dirname(PROXY_PY_DIR), + 'helper', + 'proxy.pac', +) - @mock.patch('selectors.DefaultSelector') - @mock.patch('socket.fromfd') - def test_pac_file_served_from_disk( - self, mock_fromfd: mock.Mock, mock_selector: mock.Mock) -> None: - pac_file = os.path.join( - os.path.dirname(PROXY_PY_DIR), - 'helper', - 'proxy.pac') - self._conn = mock_fromfd.return_value - self.mock_selector_for_client_read(mock_selector) - self.init_and_make_pac_file_request(pac_file) - self.protocol_handler.run_once() - self.assertEqual( - self.protocol_handler.request.state, - httpParserStates.COMPLETE) - with open(pac_file, 'rb') as f: - self._conn.send.called_once_with(build_http_response( - 200, reason=b'OK', headers={ - b'Content-Type': b'application/x-ns-proxy-autoconfig', - b'Connection': b'close' - }, body=f.read() - )) +PAC_FILE_CONTENT = b'function FindProxyForURL(url, host) { return "PROXY localhost:8899; DIRECT"; }' - @mock.patch('selectors.DefaultSelector') - @mock.patch('socket.fromfd') - def test_pac_file_served_from_buffer( - self, mock_fromfd: mock.Mock, mock_selector: mock.Mock) -> None: - self._conn = mock_fromfd.return_value - self.mock_selector_for_client_read(mock_selector) - pac_file_content = b'function FindProxyForURL(url, host) { return "PROXY localhost:8899; DIRECT"; }' - self.init_and_make_pac_file_request(text_(pac_file_content)) - self.protocol_handler.run_once() - self.assertEqual( - self.protocol_handler.request.state, - httpParserStates.COMPLETE) - self._conn.send.called_once_with(build_http_response( - 200, reason=b'OK', headers={ - b'Content-Type': b'application/x-ns-proxy-autoconfig', - b'Connection': b'close' - }, body=pac_file_content - )) - @mock.patch('selectors.DefaultSelector') - @mock.patch('socket.fromfd') - def test_default_web_server_returns_404( - self, mock_fromfd: mock.Mock, mock_selector: mock.Mock) -> None: - self._conn = mock_fromfd.return_value - mock_selector.return_value.select.return_value = [( +def test_on_client_connection_called_on_teardown(mocker: MockerFixture) -> None: + plugin = mocker.MagicMock() + mock_fromfd = mocker.patch('socket.fromfd') + flags = FlagParser.initialize(threaded=True) + flags.plugins = {b'HttpProtocolHandlerPlugin': [plugin]} + _conn = mock_fromfd.return_value + _addr = ('127.0.0.1', 54382) + protocol_handler = HttpProtocolHandler( + TcpClientConnection(_conn, _addr), + flags=flags, + ) + protocol_handler.initialize() + plugin.assert_called() + mock_run_once = mocker.patch.object(protocol_handler, '_run_once') + mock_run_once.return_value = True + protocol_handler.run() + assert _conn.closed + plugin.return_value.on_client_connection_close.assert_called() + + +def mock_selector_for_client_read(self: Any) -> None: + self.mock_selector.return_value.select.return_value = [ + ( selectors.SelectorKey( - fileobj=self._conn, - fd=self._conn.fileno, + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), events=selectors.EVENT_READ, - data=None), selectors.EVENT_READ), ] - flags = Proxy.initialize() - flags.plugins = Proxy.load_plugins([ + data=None, + ), + selectors.EVENT_READ, + ), + ] + + # @mock.patch('socket.fromfd') + # def test_on_client_connection_called_on_teardown( + # self, mock_fromfd: mock.Mock, + # ) -> None: + # flags = FlagParser.initialize(threaded=True) + # plugin = mock.MagicMock() + # flags.plugins = {b'HttpProtocolHandlerPlugin': [plugin]} + # self._conn = mock_fromfd.return_value + # self.protocol_handler = HttpProtocolHandler( + # TcpClientConnection(self._conn, self._addr), + # flags=flags, + # ) + # self.protocol_handler.initialize() + # plugin.assert_called() + # with mock.patch.object(self.protocol_handler, '_run_once') as mock_run_once: + # mock_run_once.return_value = True + # self.protocol_handler.run() + # self.assertTrue(self._conn.closed) + # plugin.return_value.on_client_connection_close.assert_called() + + # @mock.patch('socket.fromfd') + # def test_on_client_connection_called_on_teardown( + # self, mock_fromfd: mock.Mock, + # ) -> None: + # flags = FlagParser.initialize(threaded=True) + # plugin = mock.MagicMock() + # flags.plugins = {b'HttpProtocolHandlerPlugin': [plugin]} + # self._conn = mock_fromfd.return_value + # self.protocol_handler = HttpProtocolHandler( + # TcpClientConnection(self._conn, self._addr), + # flags=flags, + # ) + # self.protocol_handler.initialize() + # plugin.assert_called() + # with mock.patch.object(self.protocol_handler, '_run_once') as mock_run_once: + # mock_run_once.return_value = True + # self.protocol_handler.run() + # self.assertTrue(self._conn.closed) + # plugin.return_value.on_client_connection_close.assert_called() + + # @mock.patch('socket.fromfd') + # def test_on_client_connection_called_on_teardown( + # self, mock_fromfd: mock.Mock, + # ) -> None: + # flags = FlagParser.initialize(threaded=True) + # plugin = mock.MagicMock() + # flags.plugins = {b'HttpProtocolHandlerPlugin': [plugin]} + # self._conn = mock_fromfd.return_value + # self.protocol_handler = HttpProtocolHandler( + # TcpClientConnection(self._conn, self._addr), + # flags=flags, + # ) + # self.protocol_handler.initialize() + # plugin.assert_called() + # with mock.patch.object(self.protocol_handler, '_run_once') as mock_run_once: + # mock_run_once.return_value = True + # self.protocol_handler.run() + # self.assertTrue(self._conn.closed) + # plugin.return_value.on_client_connection_close.assert_called() + + +class TestWebServerPluginWithPacFilePlugin(Assertions): + + @pytest.fixture( + autouse=True, params=[ + PAC_FILE_PATH, + PAC_FILE_CONTENT, + ], + ) # type: ignore[misc] + def _setUp(self, request: Any, mocker: MockerFixture) -> None: + self.mock_fromfd = mocker.patch('socket.fromfd') + self.mock_selector = mocker.patch('selectors.DefaultSelector') + self.fileno = 10 + self._addr = ('127.0.0.1', 54382) + self._conn = self.mock_fromfd.return_value + self.pac_file = request.param + if isinstance(self.pac_file, str): + with open(self.pac_file, 'rb') as f: + self.expected_response = f.read() + else: + self.expected_response = PAC_FILE_CONTENT + self.flags = FlagParser.initialize( + pac_file=self.pac_file, threaded=True, + ) + self.flags.plugins = Plugins.load([ bytes_(PLUGIN_HTTP_PROXY), bytes_(PLUGIN_WEB_SERVER), + bytes_(PLUGIN_PAC_FILE), ]) self.protocol_handler = HttpProtocolHandler( TcpClientConnection(self._conn, self._addr), - flags=flags) + flags=self.flags, + ) self.protocol_handler.initialize() self._conn.recv.return_value = CRLF.join([ - b'GET /hello HTTP/1.1', + b'GET / HTTP/1.1', CRLF, ]) - self.protocol_handler.run_once() + mock_selector_for_client_read(self) + + @pytest.mark.asyncio # type: ignore[misc] + async def test_pac_file_served_from_disk(self) -> None: + await self.protocol_handler._run_once() self.assertEqual( self.protocol_handler.request.state, - httpParserStates.COMPLETE) - self.assertEqual( - self.protocol_handler.client.buffer[0], - HttpWebServerPlugin.DEFAULT_404_RESPONSE) + httpParserStates.COMPLETE, + ) + self._conn.send.called_once_with( + build_http_response( + 200, reason=b'OK', headers={ + b'Content-Type': b'application/x-ns-proxy-autoconfig', + b'Connection': b'close', + }, body=self.expected_response, + ), + ) - @unittest.skipIf(os.environ.get('GITHUB_ACTIONS', False), - 'Disabled on GitHub actions because this test is flaky on GitHub infrastructure.') - @mock.patch('selectors.DefaultSelector') - @mock.patch('socket.fromfd') - def test_static_web_server_serves( - self, mock_fromfd: mock.Mock, mock_selector: mock.Mock) -> None: - # Setup a static directory - static_server_dir = os.path.join(tempfile.gettempdir(), 'static') - index_file_path = os.path.join(static_server_dir, 'index.html') - html_file_content = b'''

Proxy.py Testing

''' - os.makedirs(static_server_dir, exist_ok=True) - with open(index_file_path, 'wb') as f: - f.write(html_file_content) - self._conn = mock_fromfd.return_value - self._conn.recv.return_value = build_http_request( - b'GET', b'/index.html') +class TestStaticWebServerPlugin(Assertions): - mock_selector.return_value.select.side_effect = [ - [(selectors.SelectorKey( - fileobj=self._conn, - fd=self._conn.fileno, - events=selectors.EVENT_READ, - data=None), selectors.EVENT_READ)], - [(selectors.SelectorKey( - fileobj=self._conn, - fd=self._conn.fileno, - events=selectors.EVENT_WRITE, - data=None), selectors.EVENT_WRITE)], ] - - flags = Proxy.initialize( + @pytest.fixture(autouse=True) # type: ignore[misc] + def _setUp(self, mocker: MockerFixture) -> None: + self.mock_fromfd = mocker.patch('socket.fromfd') + self.mock_selector = mocker.patch('selectors.DefaultSelector') + self.fileno = 10 + self._addr = ('127.0.0.1', 54382) + self._conn = self.mock_fromfd.return_value + # Setup a static directory + self.static_server_dir = os.path.join(tempfile.gettempdir(), 'static') + self.index_file_path = os.path.join( + self.static_server_dir, 'index.html', + ) + self.html_file_content = b'''

Proxy.py Testing

''' + os.makedirs(self.static_server_dir, exist_ok=True) + with open(self.index_file_path, 'wb') as f: + f.write(self.html_file_content) + # + flags = FlagParser.initialize( enable_static_server=True, - static_server_dir=static_server_dir) - flags.plugins = Proxy.load_plugins([ + static_server_dir=self.static_server_dir, + threaded=True, + ) + flags.plugins = Plugins.load([ bytes_(PLUGIN_HTTP_PROXY), bytes_(PLUGIN_WEB_SERVER), ]) - self.protocol_handler = HttpProtocolHandler( TcpClientConnection(self._conn, self._addr), - flags=flags) + flags=flags, + ) self.protocol_handler.initialize() - self.protocol_handler.run_once() - self.protocol_handler.run_once() + @pytest.mark.asyncio # type: ignore[misc] + async def test_static_web_server_serves(self) -> None: + self._conn.recv.return_value = build_http_request( + b'GET', b'/index.html', + ) + self.mock_selector.return_value.select.side_effect = [ + [( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=selectors.EVENT_READ, + data=None, + ), + selectors.EVENT_READ, + )], + [( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=selectors.EVENT_WRITE, + data=None, + ), + selectors.EVENT_WRITE, + )], + ] + await self.protocol_handler._run_once() + await self.protocol_handler._run_once() - self.assertEqual(mock_selector.return_value.select.call_count, 2) + self.assertEqual(self.mock_selector.return_value.select.call_count, 2) self.assertEqual(self._conn.send.call_count, 1) - encoded_html_file_content = gzip.compress(html_file_content) - self.assertEqual(self._conn.send.call_args[0][0], build_http_response( - 200, reason=b'OK', headers={ - b'Content-Type': b'text/html', - b'Cache-Control': b'max-age=86400', - b'Content-Encoding': b'gzip', - b'Connection': b'close', - b'Content-Length': bytes_(len(encoded_html_file_content)), - }, - body=encoded_html_file_content - )) + encoded_html_file_content = gzip.compress(self.html_file_content) - @mock.patch('selectors.DefaultSelector') - @mock.patch('socket.fromfd') - def test_static_web_server_serves_404( - self, - mock_fromfd: mock.Mock, - mock_selector: mock.Mock) -> None: - self._conn = mock_fromfd.return_value - self._conn.recv.return_value = build_http_request( - b'GET', b'/not-found.html') + # parse response and verify + response = HttpParser(httpParserTypes.RESPONSE_PARSER) + response.parse(self._conn.send.call_args[0][0]) + self.assertEqual(response.code, b'200') + self.assertEqual(response.header(b'content-type'), b'text/html') + self.assertEqual(response.header(b'cache-control'), b'max-age=86400') + self.assertEqual(response.header(b'content-encoding'), b'gzip') + self.assertEqual(response.header(b'connection'), b'close') + self.assertEqual( + response.header(b'content-length'), + bytes_(len(encoded_html_file_content)), + ) + assert response.body + self.assertEqual( + gzip.decompress(response.body), + self.html_file_content, + ) - mock_selector.return_value.select.side_effect = [ - [(selectors.SelectorKey( - fileobj=self._conn, - fd=self._conn.fileno, - events=selectors.EVENT_READ, - data=None), selectors.EVENT_READ)], - [(selectors.SelectorKey( - fileobj=self._conn, - fd=self._conn.fileno, - events=selectors.EVENT_WRITE, - data=None), selectors.EVENT_WRITE)], ] + @pytest.mark.asyncio # type: ignore[misc] + async def test_static_web_server_serves_404(self) -> None: + self._conn.recv.return_value = build_http_request( + b'GET', b'/not-found.html', + ) + self.mock_selector.return_value.select.side_effect = [ + [( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=selectors.EVENT_READ, + data=None, + ), + selectors.EVENT_READ, + )], + [( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=selectors.EVENT_WRITE, + data=None, + ), + selectors.EVENT_WRITE, + )], + ] - flags = Proxy.initialize(enable_static_server=True) - flags.plugins = Proxy.load_plugins([ - bytes_(PLUGIN_HTTP_PROXY), - bytes_(PLUGIN_WEB_SERVER), - ]) + await self.protocol_handler._run_once() + await self.protocol_handler._run_once() - self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), - flags=flags) - self.protocol_handler.initialize() + self.assertEqual(self.mock_selector.return_value.select.call_count, 2) + self.assertEqual(self._conn.send.call_count, 1) + self.assertEqual( + self._conn.send.call_args[0][0], + HttpWebServerPlugin.DEFAULT_404_RESPONSE, + ) - self.protocol_handler.run_once() - self.protocol_handler.run_once() - self.assertEqual(mock_selector.return_value.select.call_count, 2) - self.assertEqual(self._conn.send.call_count, 1) - self.assertEqual(self._conn.send.call_args[0][0], - HttpWebServerPlugin.DEFAULT_404_RESPONSE) +class TestWebServerPlugin(Assertions): - @mock.patch('socket.fromfd') - def test_on_client_connection_called_on_teardown( - self, mock_fromfd: mock.Mock) -> None: - flags = Proxy.initialize() - plugin = mock.MagicMock() - flags.plugins = {b'HttpProtocolHandlerPlugin': [plugin]} - self._conn = mock_fromfd.return_value + @pytest.fixture(autouse=True) # type: ignore[misc] + def _setUp(self, mocker: MockerFixture) -> None: + self.mock_fromfd = mocker.patch('socket.fromfd') + self.mock_selector = mocker.patch('selectors.DefaultSelector') + self.fileno = 10 + self._addr = ('127.0.0.1', 54382) + self._conn = self.mock_fromfd.return_value + self.flags = FlagParser.initialize(threaded=True) + self.flags.plugins = Plugins.load([ + bytes_(PLUGIN_HTTP_PROXY), + bytes_(PLUGIN_WEB_SERVER), + ]) self.protocol_handler = HttpProtocolHandler( TcpClientConnection(self._conn, self._addr), - flags=flags) + flags=self.flags, + ) self.protocol_handler.initialize() - plugin.assert_called() - with mock.patch.object(self.protocol_handler, 'run_once') as mock_run_once: - mock_run_once.return_value = True - self.protocol_handler.run() - self.assertTrue(self._conn.closed) - plugin.return_value.on_client_connection_close.assert_called() - def init_and_make_pac_file_request(self, pac_file: str) -> None: - flags = Proxy.initialize(pac_file=pac_file) - flags.plugins = Proxy.load_plugins([ + @pytest.mark.asyncio # type: ignore[misc] + async def test_default_web_server_returns_404(self) -> None: + self._conn = self.mock_fromfd.return_value + self.mock_selector.return_value.select.return_value = [ + ( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=selectors.EVENT_READ, + data=None, + ), + selectors.EVENT_READ, + ), + ] + flags = FlagParser.initialize(threaded=True) + flags.plugins = Plugins.load([ bytes_(PLUGIN_HTTP_PROXY), bytes_(PLUGIN_WEB_SERVER), - bytes_(PLUGIN_PAC_FILE), ]) self.protocol_handler = HttpProtocolHandler( TcpClientConnection(self._conn, self._addr), - flags=flags) + flags=flags, + ) self.protocol_handler.initialize() self._conn.recv.return_value = CRLF.join([ - b'GET / HTTP/1.1', + b'GET /hello HTTP/1.1', CRLF, ]) - - def mock_selector_for_client_read(self, mock_selector: mock.Mock) -> None: - mock_selector.return_value.select.return_value = [( - selectors.SelectorKey( - fileobj=self._conn, - fd=self._conn.fileno, - events=selectors.EVENT_READ, - data=None), selectors.EVENT_READ), ] + await self.protocol_handler._run_once() + self.assertEqual( + self.protocol_handler.request.state, + httpParserStates.COMPLETE, + ) + self.assertEqual( + self.protocol_handler.work.buffer[0], + HttpWebServerPlugin.DEFAULT_404_RESPONSE, + ) diff --git a/tests/http/test_websocket_client.py b/tests/http/test_websocket_client.py index faf18b2ac6..ef500c5b97 100644 --- a/tests/http/test_websocket_client.py +++ b/tests/http/test_websocket_client.py @@ -21,18 +21,21 @@ class TestWebsocketClient(unittest.TestCase): @mock.patch('proxy.http.websocket.client.socket.gethostbyname') @mock.patch('base64.b64encode') @mock.patch('proxy.http.websocket.client.new_socket_connection') - def test_handshake(self, mock_connect: mock.Mock, - mock_b64encode: mock.Mock, - mock_gethostbyname: mock.Mock) -> None: + def test_handshake( + self, mock_connect: mock.Mock, + mock_b64encode: mock.Mock, + mock_gethostbyname: mock.Mock, + ) -> None: key = b'MySecretKey' mock_b64encode.return_value = key mock_gethostbyname.return_value = '127.0.0.1' mock_connect.return_value.recv.return_value = \ build_websocket_handshake_response( - WebsocketFrame.key_to_accept(key)) + WebsocketFrame.key_to_accept(key), + ) client = WebsocketClient(b'localhost', DEFAULT_PORT) mock_connect.return_value.send.assert_not_called() client.handshake() mock_connect.return_value.send.assert_called_with( - build_websocket_handshake_request(key) + build_websocket_handshake_request(key), ) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py new file mode 100644 index 0000000000..9df24fa8d1 --- /dev/null +++ b/tests/integration/test_integration.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. + + Test the simplest proxy use scenario for smoke. +""" +from pathlib import Path +from subprocess import check_output, Popen +from typing import Generator, Any + +import pytest + +from proxy.common.utils import get_available_port +from proxy.common._compat import IS_WINDOWS # noqa: WPS436 + + +# FIXME: Ignore is necessary for as long as pytest hasn't figured out +# FIXME: typing for their fixtures. +# Refs: +# * https://github.com/pytest-dev/pytest/issues/7469#issuecomment-918345196 +# * https://github.com/pytest-dev/pytest/issues/3342 +@pytest.fixture # type: ignore[misc] +def proxy_py_subprocess(request: Any) -> Generator[int, None, None]: + """Instantiate proxy.py in a subprocess for testing. + + NOTE: Doesn't waits for the proxy to startup. + Ensure instance check in your tests. + + After the testing is over, tear it down. + """ + port = get_available_port() + proxy_cmd = ( + 'python', '-m', 'proxy', + '--hostname', '127.0.0.1', + '--port', str(port), + '--enable-web-server', + ) + tuple(request.param.split()) + proxy_proc = Popen(proxy_cmd) + try: + yield port + finally: + proxy_proc.terminate() + proxy_proc.wait() + + +# FIXME: Ignore is necessary for as long as pytest hasn't figured out +# FIXME: typing for their fixtures. +# Refs: +# * https://github.com/pytest-dev/pytest/issues/7469#issuecomment-918345196 +# * https://github.com/pytest-dev/pytest/issues/3342 +@pytest.mark.smoke # type: ignore[misc] +@pytest.mark.parametrize( + 'proxy_py_subprocess', + ( + ('--threadless'), + ('--threadless --local-executor'), + ('--threaded'), + ), + indirect=True, +) # type: ignore[misc] +@pytest.mark.xfail( + IS_WINDOWS, + reason='OSError: [WinError 193] %1 is not a valid Win32 application', + raises=OSError, +) # type: ignore[misc] +def test_curl(proxy_py_subprocess: int) -> None: + """An acceptance test with using ``curl`` through proxy.py.""" + this_test_module = Path(__file__) + shell_script_test = this_test_module.with_suffix('.sh') + check_output([str(shell_script_test), str(proxy_py_subprocess)]) diff --git a/tests/integration/main.sh b/tests/integration/test_integration.sh similarity index 71% rename from tests/integration/main.sh rename to tests/integration/test_integration.sh index d75d9781de..b31b8452bf 100755 --- a/tests/integration/main.sh +++ b/tests/integration/test_integration.sh @@ -1,7 +1,7 @@ #!/bin/bash # TODO: Option to also shutdown proxy.py after -# integration testing is done. Atleast on +# integration testing is done. At least on # macOS and ubuntu, pkill and kill commands # will do the job. # @@ -9,9 +9,16 @@ # to clean up any background process including # proxy.py +PROXY_PY_PORT=$1 +if [[ -z "$PROXY_PY_PORT" ]]; then + echo "PROXY_PY_PORT required as argument." + exit 1 +fi + # Wait for server to come up +WAIT_FOR_PROXY="lsof -i TCP:$PROXY_PY_PORT | wc -l | tr -d ' '" while true; do - if [[ $(lsof -i TCP:8899 | wc -l | tr -d ' ') == 0 ]]; then + if [[ $WAIT_FOR_PORT == 0 ]]; then echo "Waiting for proxy..." sleep 1 else @@ -24,8 +31,8 @@ while true; do curl -v \ --max-time 1 \ --connect-timeout 1 \ - -x localhost:8899 \ - http://localhost:8899/ 2>/dev/null + -x 127.0.0.1:$PROXY_PY_PORT \ + http://127.0.0.1:$PROXY_PY_PORT/ 2>/dev/null if [[ $? == 0 ]]; then break fi @@ -41,7 +48,7 @@ done # detect if we have internet access. If we do, # then use httpbin.org for integration testing. curl -v \ - -x localhost:8899 \ + -x 127.0.0.1:$PROXY_PY_PORT \ http://httpbin.org/get if [[ $? != 0 ]]; then echo "http request failed" @@ -49,7 +56,7 @@ if [[ $? != 0 ]]; then fi curl -v \ - -x localhost:8899 \ + -x 127.0.0.1:$PROXY_PY_PORT \ https://httpbin.org/get if [[ $? != 0 ]]; then echo "https request failed" @@ -57,8 +64,8 @@ if [[ $? != 0 ]]; then fi curl -v \ - -x localhost:8899 \ - http://localhost:8899/ + -x 127.0.0.1:$PROXY_PY_PORT \ + http://127.0.0.1:$PROXY_PY_PORT/ if [[ $? != 0 ]]; then echo "http request to built in webserver failed" exit 1 diff --git a/tests/plugin/test_http_proxy_plugins.py b/tests/plugin/test_http_proxy_plugins.py index 28e85896ec..7102c3ef7a 100644 --- a/tests/plugin/test_http_proxy_plugins.py +++ b/tests/plugin/test_http_proxy_plugins.py @@ -8,57 +8,76 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -import unittest -import selectors import json +import pytest +import selectors -from urllib import parse as urlparse +from pathlib import Path from unittest import mock -from typing import cast +from typing import cast, Any +from urllib import parse as urlparse +from pytest_mock import MockerFixture -from proxy.proxy import Proxy +from proxy.common.flag import FlagParser from proxy.core.connection import TcpClientConnection -from proxy.http.handler import HttpProtocolHandler +from proxy.http import HttpProtocolHandler +from proxy.http import httpStatusCodes from proxy.http.proxy import HttpProxyPlugin from proxy.common.utils import build_http_request, bytes_, build_http_response from proxy.common.constants import PROXY_AGENT_HEADER_VALUE, DEFAULT_HTTP_PORT -from proxy.http.codes import httpStatusCodes - from proxy.plugin import ProposedRestApiPlugin, RedirectToCustomServerPlugin from .utils import get_plugin_by_test_name +from ..test_assertions import Assertions + + +class TestHttpProxyPluginExamples(Assertions): -class TestHttpProxyPluginExamples(unittest.TestCase): + @pytest.fixture(autouse=True) # type: ignore[misc] + def _setUp(self, request: Any, mocker: MockerFixture) -> None: + self.mock_fromfd = mocker.patch('socket.fromfd') + self.mock_selector = mocker.patch('selectors.DefaultSelector') + self.mock_server_conn = mocker.patch( + 'proxy.http.proxy.server.TcpServerConnection', + ) - @mock.patch('selectors.DefaultSelector') - @mock.patch('socket.fromfd') - def setUp(self, - mock_fromfd: mock.Mock, - mock_selector: mock.Mock) -> None: self.fileno = 10 self._addr = ('127.0.0.1', 54382) - self.flags = Proxy.initialize() + adblock_json_path = Path( + __file__, + ).parent.parent.parent / "proxy" / "plugin" / "adblock.json" + self.flags = FlagParser.initialize( + input_args=[ + "--filtered-url-regex-config", + str(adblock_json_path), + ], + threaded=True, + ) self.plugin = mock.MagicMock() - self.mock_fromfd = mock_fromfd - self.mock_selector = mock_selector - - plugin = get_plugin_by_test_name(self._testMethodName) + plugin = get_plugin_by_test_name(request.param) self.flags.plugins = { b'HttpProtocolHandlerPlugin': [HttpProxyPlugin], b'HttpProxyBasePlugin': [plugin], } - self._conn = mock_fromfd.return_value + self._conn = self.mock_fromfd.return_value self.protocol_handler = HttpProtocolHandler( TcpClientConnection(self._conn, self._addr), - flags=self.flags) + flags=self.flags, + ) self.protocol_handler.initialize() - @mock.patch('proxy.http.proxy.server.TcpServerConnection') - def test_modify_post_data_plugin( - self, mock_server_conn: mock.Mock) -> None: + @pytest.mark.asyncio # type: ignore[misc] + @pytest.mark.parametrize( + "_setUp", + ( + ('test_modify_post_data_plugin'), + ), + indirect=True, + ) # type: ignore[misc] + async def test_modify_post_data_plugin(self) -> None: original = b'{"key": "value"}' modified = b'{"key": "modified"}' @@ -69,18 +88,25 @@ def test_modify_post_data_plugin( b'Content-Type': b'application/x-www-form-urlencoded', b'Content-Length': bytes_(len(original)), }, - body=original + body=original, ) self.mock_selector.return_value.select.side_effect = [ - [(selectors.SelectorKey( - fileobj=self._conn, - fd=self._conn.fileno, - events=selectors.EVENT_READ, - data=None), selectors.EVENT_READ)], ] + [( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=selectors.EVENT_READ, + data=None, + ), + selectors.EVENT_READ, + )], + ] - self.protocol_handler.run_once() - mock_server_conn.assert_called_with('httpbin.org', DEFAULT_HTTP_PORT) - mock_server_conn.return_value.queue.assert_called_with( + await self.protocol_handler._run_once() + self.mock_server_conn.assert_called_with( + 'httpbin.org', DEFAULT_HTTP_PORT, + ) + self.mock_server_conn.return_value.queue.assert_called_with( build_http_request( b'POST', b'/post', headers={ @@ -89,113 +115,157 @@ def test_modify_post_data_plugin( b'Content-Type': b'application/json', b'Via': b'1.1 %s' % PROXY_AGENT_HEADER_VALUE, }, - body=modified - ) + body=modified, + ), ) - @mock.patch('proxy.http.proxy.server.TcpServerConnection') - def test_proposed_rest_api_plugin( - self, mock_server_conn: mock.Mock) -> None: + @pytest.mark.asyncio # type: ignore[misc] + @pytest.mark.parametrize( + "_setUp", + ( + ('test_proposed_rest_api_plugin'), + ), + indirect=True, + ) # type: ignore[misc] + async def test_proposed_rest_api_plugin(self) -> None: path = b'/v1/users/' self._conn.recv.return_value = build_http_request( b'GET', b'http://%s%s' % ( - ProposedRestApiPlugin.API_SERVER, path), + ProposedRestApiPlugin.API_SERVER, path, + ), headers={ b'Host': ProposedRestApiPlugin.API_SERVER, - } + }, ) self.mock_selector.return_value.select.side_effect = [ - [(selectors.SelectorKey( - fileobj=self._conn, - fd=self._conn.fileno, - events=selectors.EVENT_READ, - data=None), selectors.EVENT_READ)], ] - self.protocol_handler.run_once() + [( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=selectors.EVENT_READ, + data=None, + ), + selectors.EVENT_READ, + )], + ] + await self.protocol_handler._run_once() - mock_server_conn.assert_not_called() + self.mock_server_conn.assert_not_called() self.assertEqual( - self.protocol_handler.client.buffer[0].tobytes(), + self.protocol_handler.work.buffer[0].tobytes(), build_http_response( httpStatusCodes.OK, reason=b'OK', headers={b'Content-Type': b'application/json'}, body=bytes_( json.dumps( - ProposedRestApiPlugin.REST_API_SPEC[path])) - )) + ProposedRestApiPlugin.REST_API_SPEC[path], + ), + ), + ), + ) - @mock.patch('proxy.http.proxy.server.TcpServerConnection') - def test_redirect_to_custom_server_plugin( - self, mock_server_conn: mock.Mock) -> None: + @pytest.mark.asyncio # type: ignore[misc] + @pytest.mark.parametrize( + "_setUp", + ( + ('test_redirect_to_custom_server_plugin'), + ), + indirect=True, + ) # type: ignore[misc] + async def test_redirect_to_custom_server_plugin(self) -> None: request = build_http_request( b'GET', b'http://example.org/get', headers={ b'Host': b'example.org', - } + }, ) self._conn.recv.return_value = request self.mock_selector.return_value.select.side_effect = [ - [(selectors.SelectorKey( - fileobj=self._conn, - fd=self._conn.fileno, - events=selectors.EVENT_READ, - data=None), selectors.EVENT_READ)], ] - self.protocol_handler.run_once() + [( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=selectors.EVENT_READ, + data=None, + ), + selectors.EVENT_READ, + )], + ] + await self.protocol_handler._run_once() upstream = urlparse.urlsplit( - RedirectToCustomServerPlugin.UPSTREAM_SERVER) - mock_server_conn.assert_called_with('localhost', 8899) - mock_server_conn.return_value.queue.assert_called_with( + RedirectToCustomServerPlugin.UPSTREAM_SERVER, + ) + self.mock_server_conn.assert_called_with('localhost', 8899) + self.mock_server_conn.return_value.queue.assert_called_with( build_http_request( b'GET', upstream.path, headers={ b'Host': upstream.netloc, b'Via': b'1.1 %s' % PROXY_AGENT_HEADER_VALUE, - } - ) + }, + ), ) - @mock.patch('proxy.http.proxy.server.TcpServerConnection') - def test_filter_by_upstream_host_plugin( - self, mock_server_conn: mock.Mock) -> None: + @pytest.mark.asyncio # type: ignore[misc] + @pytest.mark.parametrize( + "_setUp", + ( + ('test_filter_by_upstream_host_plugin'), + ), + indirect=True, + ) # type: ignore[misc] + async def test_filter_by_upstream_host_plugin(self) -> None: request = build_http_request( - b'GET', b'http://google.com/', + b'GET', b'http://facebook.com/', headers={ - b'Host': b'google.com', - } + b'Host': b'facebook.com', + }, ) self._conn.recv.return_value = request self.mock_selector.return_value.select.side_effect = [ - [(selectors.SelectorKey( - fileobj=self._conn, - fd=self._conn.fileno, - events=selectors.EVENT_READ, - data=None), selectors.EVENT_READ)], ] - self.protocol_handler.run_once() + [( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=selectors.EVENT_READ, + data=None, + ), + selectors.EVENT_READ, + )], + ] + await self.protocol_handler._run_once() - mock_server_conn.assert_not_called() + self.mock_server_conn.assert_not_called() self.assertEqual( - self.protocol_handler.client.buffer[0].tobytes(), + self.protocol_handler.work.buffer[0].tobytes(), build_http_response( status_code=httpStatusCodes.I_AM_A_TEAPOT, reason=b'I\'m a tea pot', headers={ - b'Connection': b'close' + b'Connection': b'close', }, - ) + ), ) - @mock.patch('proxy.http.proxy.server.TcpServerConnection') - def test_man_in_the_middle_plugin( - self, mock_server_conn: mock.Mock) -> None: + @pytest.mark.asyncio # type: ignore[misc] + @pytest.mark.parametrize( + "_setUp", + ( + ('test_man_in_the_middle_plugin'), + ), + indirect=True, + ) # type: ignore[misc] + async def test_man_in_the_middle_plugin(self) -> None: request = build_http_request( b'GET', b'http://super.secure/', headers={ b'Host': b'super.secure', - } + }, ) self._conn.recv.return_value = request - server = mock_server_conn.return_value + server = self.mock_server_conn.return_value server.connect.return_value = True def has_buffer() -> bool: @@ -208,76 +278,104 @@ def closed() -> bool: type(server).closed = mock.PropertyMock(side_effect=closed) self.mock_selector.return_value.select.side_effect = [ - [(selectors.SelectorKey( - fileobj=self._conn, - fd=self._conn.fileno, - events=selectors.EVENT_READ, - data=None), selectors.EVENT_READ)], - [(selectors.SelectorKey( - fileobj=server.connection, - fd=server.connection.fileno, - events=selectors.EVENT_WRITE, - data=None), selectors.EVENT_WRITE)], - [(selectors.SelectorKey( - fileobj=server.connection, - fd=server.connection.fileno, - events=selectors.EVENT_READ, - data=None), selectors.EVENT_READ)], ] + [( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=selectors.EVENT_READ, + data=None, + ), + selectors.EVENT_READ, + )], + [( + selectors.SelectorKey( + fileobj=server.connection.fileno(), + fd=server.connection.fileno(), + events=selectors.EVENT_WRITE, + data=None, + ), + selectors.EVENT_WRITE, + )], + [( + selectors.SelectorKey( + fileobj=server.connection.fileno(), + fd=server.connection.fileno(), + events=selectors.EVENT_READ, + data=None, + ), + selectors.EVENT_READ, + )], + ] # Client read - self.protocol_handler.run_once() - mock_server_conn.assert_called_with('super.secure', DEFAULT_HTTP_PORT) + await self.protocol_handler._run_once() + self.mock_server_conn.assert_called_with( + 'super.secure', DEFAULT_HTTP_PORT, + ) server.connect.assert_called_once() queued_request = \ build_http_request( b'GET', b'/', headers={ b'Host': b'super.secure', - b'Via': b'1.1 %s' % PROXY_AGENT_HEADER_VALUE - } + b'Via': b'1.1 %s' % PROXY_AGENT_HEADER_VALUE, + }, ) server.queue.assert_called_once_with(queued_request) # Server write - self.protocol_handler.run_once() + await self.protocol_handler._run_once() server.flush.assert_called_once() # Server read server.recv.return_value = \ build_http_response( httpStatusCodes.OK, - reason=b'OK', body=b'Original Response From Upstream') - self.protocol_handler.run_once() + reason=b'OK', body=b'Original Response From Upstream', + ) + await self.protocol_handler._run_once() self.assertEqual( - self.protocol_handler.client.buffer[0].tobytes(), + self.protocol_handler.work.buffer[0].tobytes(), build_http_response( httpStatusCodes.OK, - reason=b'OK', body=b'Hello from man in the middle') + reason=b'OK', body=b'Hello from man in the middle', + ), ) - @mock.patch('proxy.http.proxy.server.TcpServerConnection') - def test_filter_by_url_regex_plugin( - self, mock_server_conn: mock.Mock) -> None: + @pytest.mark.asyncio # type: ignore[misc] + @pytest.mark.parametrize( + "_setUp", + ( + ('test_filter_by_url_regex_plugin'), + ), + indirect=True, + ) # type: ignore[misc] + async def test_filter_by_url_regex_plugin(self) -> None: request = build_http_request( b'GET', b'http://www.facebook.com/tr/', headers={ b'Host': b'www.facebook.com', - } + }, ) self._conn.recv.return_value = request self.mock_selector.return_value.select.side_effect = [ - [(selectors.SelectorKey( - fileobj=self._conn, - fd=self._conn.fileno, - events=selectors.EVENT_READ, - data=None), selectors.EVENT_READ)], ] - self.protocol_handler.run_once() + [( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=selectors.EVENT_READ, + data=None, + ), + selectors.EVENT_READ, + )], + ] + await self.protocol_handler._run_once() self.assertEqual( - self.protocol_handler.client.buffer[0].tobytes(), + self.protocol_handler.work.buffer[0].tobytes(), build_http_response( status_code=httpStatusCodes.NOT_FOUND, reason=b'Blocked', headers={b'Connection': b'close'}, - ) + ), ) diff --git a/tests/plugin/test_http_proxy_plugins_with_tls_interception.py b/tests/plugin/test_http_proxy_plugins_with_tls_interception.py index 39311b22aa..232b0dd954 100644 --- a/tests/plugin/test_http_proxy_plugins_with_tls_interception.py +++ b/tests/plugin/test_http_proxy_plugins_with_tls_interception.py @@ -8,53 +8,43 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -import unittest +import ssl import socket +import pytest import selectors -import ssl -from unittest import mock +from pytest_mock import MockerFixture from typing import Any, cast -from proxy.proxy import Proxy -from proxy.common.utils import bytes_ -from proxy.common.utils import build_http_request, build_http_response +from proxy.common.flag import FlagParser +from proxy.common.utils import bytes_, build_http_request, build_http_response from proxy.core.connection import TcpClientConnection, TcpServerConnection -from proxy.http.codes import httpStatusCodes -from proxy.http.methods import httpMethods -from proxy.http.handler import HttpProtocolHandler + +from proxy.http import httpMethods, httpStatusCodes, HttpProtocolHandler from proxy.http.proxy import HttpProxyPlugin +from proxy.http.parser import HttpParser from .utils import get_plugin_by_test_name +from ..test_assertions import Assertions + -class TestHttpProxyPluginExamplesWithTlsInterception(unittest.TestCase): - - @mock.patch('ssl.wrap_socket') - @mock.patch('ssl.create_default_context') - @mock.patch('proxy.http.proxy.server.TcpServerConnection') - @mock.patch('proxy.http.proxy.server.gen_public_key') - @mock.patch('proxy.http.proxy.server.gen_csr') - @mock.patch('proxy.http.proxy.server.sign_csr') - @mock.patch('selectors.DefaultSelector') - @mock.patch('socket.fromfd') - def setUp(self, - mock_fromfd: mock.Mock, - mock_selector: mock.Mock, - mock_sign_csr: mock.Mock, - mock_gen_csr: mock.Mock, - mock_gen_public_key: mock.Mock, - mock_server_conn: mock.Mock, - mock_ssl_context: mock.Mock, - mock_ssl_wrap: mock.Mock) -> None: - self.mock_fromfd = mock_fromfd - self.mock_selector = mock_selector - self.mock_sign_csr = mock_sign_csr - self.mock_gen_csr = mock_gen_csr - self.mock_gen_public_key = mock_gen_public_key - self.mock_server_conn = mock_server_conn - self.mock_ssl_context = mock_ssl_context - self.mock_ssl_wrap = mock_ssl_wrap +class TestHttpProxyPluginExamplesWithTlsInterception(Assertions): + + @pytest.fixture(autouse=True) # type: ignore[misc] + def _setUp(self, request: Any, mocker: MockerFixture) -> None: + self.mock_fromfd = mocker.patch('socket.fromfd') + self.mock_selector = mocker.patch('selectors.DefaultSelector') + self.mock_sign_csr = mocker.patch('proxy.http.proxy.server.sign_csr') + self.mock_gen_csr = mocker.patch('proxy.http.proxy.server.gen_csr') + self.mock_gen_public_key = mocker.patch( + 'proxy.http.proxy.server.gen_public_key', + ) + self.mock_server_conn = mocker.patch( + 'proxy.http.proxy.server.TcpServerConnection', + ) + self.mock_ssl_context = mocker.patch('ssl.create_default_context') + self.mock_ssl_wrap = mocker.patch('ssl.wrap_socket') self.mock_sign_csr.return_value = True self.mock_gen_csr.return_value = True @@ -62,29 +52,32 @@ def setUp(self, self.fileno = 10 self._addr = ('127.0.0.1', 54382) - self.flags = Proxy.initialize( + self.flags = FlagParser.initialize( ca_cert_file='ca-cert.pem', ca_key_file='ca-key.pem', - ca_signing_key_file='ca-signing-key.pem',) - self.plugin = mock.MagicMock() + ca_signing_key_file='ca-signing-key.pem', + threaded=True, + ) + self.plugin = mocker.MagicMock() - plugin = get_plugin_by_test_name(self._testMethodName) + plugin = get_plugin_by_test_name(request.param) self.flags.plugins = { b'HttpProtocolHandlerPlugin': [HttpProxyPlugin], b'HttpProxyBasePlugin': [plugin], } - self._conn = mock.MagicMock(spec=socket.socket) - mock_fromfd.return_value = self._conn + self._conn = mocker.MagicMock(spec=socket.socket) + self.mock_fromfd.return_value = self._conn self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), flags=self.flags) + TcpClientConnection(self._conn, self._addr), flags=self.flags, + ) self.protocol_handler.initialize() self.server = self.mock_server_conn.return_value - self.server_ssl_connection = mock.MagicMock(spec=ssl.SSLSocket) + self.server_ssl_connection = mocker.MagicMock(spec=ssl.SSLSocket) self.mock_ssl_context.return_value.wrap_socket.return_value = self.server_ssl_connection - self.client_ssl_connection = mock.MagicMock(spec=ssl.SSLSocket) + self.client_ssl_connection = mocker.MagicMock(spec=ssl.SSLSocket) self.mock_ssl_wrap.return_value = self.client_ssl_connection def has_buffer() -> bool: @@ -103,32 +96,51 @@ def mock_connection() -> Any: lambda x, y: TcpServerConnection.wrap(self.server, x, y) self.server.has_buffer.side_effect = has_buffer - type(self.server).closed = mock.PropertyMock(side_effect=closed) + type(self.server).closed = mocker.PropertyMock(side_effect=closed) type( - self.server).connection = mock.PropertyMock( - side_effect=mock_connection) + self.server, + ).connection = mocker.PropertyMock( + side_effect=mock_connection, + ) self.mock_selector.return_value.select.side_effect = [ - [(selectors.SelectorKey( - fileobj=self._conn, - fd=self._conn.fileno, - events=selectors.EVENT_READ, - data=None), selectors.EVENT_READ)], - [(selectors.SelectorKey( - fileobj=self.client_ssl_connection, - fd=self.client_ssl_connection.fileno, - events=selectors.EVENT_READ, - data=None), selectors.EVENT_READ)], - [(selectors.SelectorKey( - fileobj=self.server_ssl_connection, - fd=self.server_ssl_connection.fileno, - events=selectors.EVENT_WRITE, - data=None), selectors.EVENT_WRITE)], - [(selectors.SelectorKey( - fileobj=self.server_ssl_connection, - fd=self.server_ssl_connection.fileno, - events=selectors.EVENT_READ, - data=None), selectors.EVENT_READ)], ] + [( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=selectors.EVENT_READ, + data=None, + ), + selectors.EVENT_READ, + )], + [( + selectors.SelectorKey( + fileobj=self.client_ssl_connection.fileno(), + fd=self.client_ssl_connection.fileno(), + events=selectors.EVENT_READ, + data=None, + ), + selectors.EVENT_READ, + )], + [( + selectors.SelectorKey( + fileobj=self.server_ssl_connection.fileno(), + fd=self.server_ssl_connection.fileno(), + events=selectors.EVENT_WRITE, + data=None, + ), + selectors.EVENT_WRITE, + )], + [( + selectors.SelectorKey( + fileobj=self.server_ssl_connection.fileno(), + fd=self.server_ssl_connection.fileno(), + events=selectors.EVENT_READ, + data=None, + ), + selectors.EVENT_READ, + )], + ] # Connect def send(raw: bytes) -> int: @@ -136,9 +148,19 @@ def send(raw: bytes) -> int: self._conn.send.side_effect = send self._conn.recv.return_value = build_http_request( - httpMethods.CONNECT, b'uni.corn:443' + httpMethods.CONNECT, b'uni.corn:443', ) - self.protocol_handler.run_once() + + @pytest.mark.asyncio # type: ignore[misc] + @pytest.mark.parametrize( + '_setUp', + ( + ('test_modify_post_data_plugin'), + ), + indirect=True, + ) # type: ignore[misc] + async def test_modify_post_data_plugin(self) -> None: + await self.protocol_handler._run_once() self.assertEqual(self.mock_sign_csr.call_count, 1) self.assertEqual(self.mock_gen_csr.call_count, 1) @@ -147,65 +169,98 @@ def send(raw: bytes) -> int: self.mock_server_conn.assert_called_once_with('uni.corn', 443) self.server.connect.assert_called() self.assertEqual( - self.protocol_handler.client.connection, - self.client_ssl_connection) + self.protocol_handler.work.connection, + self.client_ssl_connection, + ) self.assertEqual(self.server.connection, self.server_ssl_connection) self._conn.send.assert_called_with( - HttpProxyPlugin.PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT + HttpProxyPlugin.PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT, ) - self.assertFalse(self.protocol_handler.client.has_buffer()) + self.assertFalse(self.protocol_handler.work.has_buffer()) - def test_modify_post_data_plugin(self) -> None: + # original = b'{"key": "value"}' modified = b'{"key": "modified"}' self.client_ssl_connection.recv.return_value = build_http_request( b'POST', b'/', headers={ b'Host': b'uni.corn', - b'Content-Type': b'application/x-www-form-urlencoded', b'Content-Length': bytes_(len(original)), + b'Content-Type': b'application/x-www-form-urlencoded', }, - body=original + body=original, ) - self.protocol_handler.run_once() - self.server.queue.assert_called_with( - build_http_request( - b'POST', b'/', - headers={ - b'Host': b'uni.corn', - b'Content-Length': bytes_(len(modified)), - b'Content-Type': b'application/json', - }, - body=modified - ) + await self.protocol_handler._run_once() + self.server.queue.assert_called_once() + # pkt = build_http_request( + # b'POST', b'/', + # headers={ + # b'Host': b'uni.corn', + # b'Content-Length': bytes_(len(modified)), + # b'Content-Type': b'application/json', + # }, + # body=modified, + # ) + response = HttpParser.response( + self.server.queue.call_args_list[0][0][0].tobytes(), ) + self.assertEqual(response.body, modified) - def test_man_in_the_middle_plugin(self) -> None: + @pytest.mark.asyncio # type: ignore[misc] + @pytest.mark.parametrize( + '_setUp', + ( + ('test_man_in_the_middle_plugin'), + ), + indirect=True, + ) # type: ignore[misc] + async def test_man_in_the_middle_plugin(self) -> None: + await self.protocol_handler._run_once() + + self.assertEqual(self.mock_sign_csr.call_count, 1) + self.assertEqual(self.mock_gen_csr.call_count, 1) + self.assertEqual(self.mock_gen_public_key.call_count, 1) + + self.mock_server_conn.assert_called_once_with('uni.corn', 443) + self.server.connect.assert_called() + self.assertEqual( + self.protocol_handler.work.connection, + self.client_ssl_connection, + ) + self.assertEqual(self.server.connection, self.server_ssl_connection) + self._conn.send.assert_called_with( + HttpProxyPlugin.PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT, + ) + self.assertFalse(self.protocol_handler.work.has_buffer()) + # request = build_http_request( b'GET', b'/', headers={ b'Host': b'uni.corn', - } + }, ) self.client_ssl_connection.recv.return_value = request # Client read - self.protocol_handler.run_once() + await self.protocol_handler._run_once() self.server.queue.assert_called_once_with(request) # Server write - self.protocol_handler.run_once() + await self.protocol_handler._run_once() self.server.flush.assert_called_once() # Server read - self.server.recv.return_value = \ + self.server.recv.return_value = memoryview( build_http_response( httpStatusCodes.OK, - reason=b'OK', body=b'Original Response From Upstream') - self.protocol_handler.run_once() + reason=b'OK', body=b'Original Response From Upstream', + ), + ) + await self.protocol_handler._run_once() self.assertEqual( - self.protocol_handler.client.buffer[0].tobytes(), + self.protocol_handler.work.buffer[0].tobytes(), build_http_response( httpStatusCodes.OK, - reason=b'OK', body=b'Hello from man in the middle') + reason=b'OK', body=b'Hello from man in the middle', + ), ) diff --git a/tests/test_assertions.py b/tests/test_assertions.py new file mode 100644 index 0000000000..85d6ca6f3f --- /dev/null +++ b/tests/test_assertions.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +from typing import Any + + +class Assertions: + + def assertTrue(self, obj: Any) -> None: + assert obj + + def assertFalse(self, obj: Any) -> None: + assert not obj + + def assertEqual(self, obj1: Any, obj2: Any) -> None: + assert obj1 == obj2 + + def assertNotEqual(self, obj1: Any, obj2: Any) -> None: + assert obj1 != obj2 diff --git a/tests/test_circular_imports.py b/tests/test_circular_imports.py new file mode 100644 index 0000000000..4e6f8d6f77 --- /dev/null +++ b/tests/test_circular_imports.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + โšกโšกโšก Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. + + Tests for circular imports in all local packages and modules. + + This ensures all internal packages can be imported right away without + any need to import some other module before doing so. + + This module is based on an idea that pytest uses for self-testing: + * https://github.com/sanitizers/octomachinery/blob/be18b54/tests/circular_imports_test.py + * https://github.com/pytest-dev/pytest/blob/d18c75b/testing/test_meta.py + * https://twitter.com/codewithanthony/status/1229445110510735361 +""" +from itertools import chain +from pathlib import Path +from types import ModuleType +from typing import Generator, List + +import os +import pkgutil +import subprocess +import sys +import pytest + +import proxy + + +def _find_all_importables(pkg: ModuleType) -> List[str]: + """Find all importables in the project. + + Return them in order. + """ + return sorted( + set( + chain.from_iterable( + _discover_path_importables(Path(p), pkg.__name__) + # FIXME: Unignore after upgrading to `mypy > 0.910`. The fix + # FIXME: is in the `master` branch of upstream since Aug 4, + # FIXME: 2021 but has not yet been included in any releases. + # Refs: + # * https://github.com/python/mypy/issues/1422 + # * https://github.com/python/mypy/pull/9454 + for p in pkg.__path__ # type: ignore[attr-defined] + ), + ), + ) + + +def _discover_path_importables( + pkg_pth: Path, pkg_name: str, +) -> Generator[str, None, None]: + """Yield all importables under a given path and package.""" + for dir_path, _d, file_names in os.walk(pkg_pth): + pkg_dir_path = Path(dir_path) + + if pkg_dir_path.parts[-1] == '__pycache__': + continue + + if all(Path(_).suffix != '.py' for _ in file_names): + continue + + rel_pt = pkg_dir_path.relative_to(pkg_pth) + pkg_pref = '.'.join((pkg_name,) + rel_pt.parts) + yield from ( + pkg_path + for _, pkg_path, _ in pkgutil.walk_packages( + (str(pkg_dir_path),), prefix=f'{pkg_pref}.', + ) + ) + + +# FIXME: Ignore is necessary for as long as pytest hasn't figured out their +# FIXME: typing for the `parametrize` mark. +# Refs: +# * https://github.com/pytest-dev/pytest/issues/7469#issuecomment-918345196 +# * https://github.com/pytest-dev/pytest/issues/3342 +@pytest.mark.parametrize( # type: ignore[misc] + 'import_path', + _find_all_importables(proxy), +) +def test_no_warnings(import_path: str) -> None: + """Verify that exploding importables doesn't explode. + + This is seeking for any import errors including ones caused + by circular imports. + """ + imp_cmd = ( + sys.executable, + '-W', 'error', + '-c', f'import {import_path!s}', + ) + + subprocess.check_call(imp_cmd) diff --git a/tests/test_main.py b/tests/test_main.py index 0e171eb04f..28874e822f 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -7,37 +7,37 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + eventing """ -import unittest -import tempfile import os +import tempfile +import unittest from unittest import mock -from typing import List -from proxy.proxy import main, Proxy +from proxy.proxy import main, entry_point from proxy.common.utils import bytes_ -from proxy.http.handler import HttpProtocolHandler -from proxy.common.constants import DEFAULT_LOG_LEVEL, DEFAULT_LOG_FILE, DEFAULT_LOG_FORMAT, DEFAULT_BASIC_AUTH +from proxy.common.constants import DEFAULT_ENABLE_DASHBOARD, DEFAULT_LOCAL_EXECUTOR, DEFAULT_LOG_LEVEL, DEFAULT_LOG_FILE from proxy.common.constants import DEFAULT_TIMEOUT, DEFAULT_DEVTOOLS_WS_PATH, DEFAULT_DISABLE_HTTP_PROXY from proxy.common.constants import DEFAULT_ENABLE_STATIC_SERVER, DEFAULT_ENABLE_EVENTS, DEFAULT_ENABLE_DEVTOOLS from proxy.common.constants import DEFAULT_ENABLE_WEB_SERVER, DEFAULT_THREADLESS, DEFAULT_CERT_FILE, DEFAULT_KEY_FILE from proxy.common.constants import DEFAULT_CA_CERT_FILE, DEFAULT_CA_KEY_FILE, DEFAULT_CA_SIGNING_KEY_FILE -from proxy.common.constants import DEFAULT_PAC_FILE, DEFAULT_PLUGINS, DEFAULT_PID_FILE, DEFAULT_PORT +from proxy.common.constants import DEFAULT_PAC_FILE, DEFAULT_PLUGINS, DEFAULT_PID_FILE, DEFAULT_PORT, DEFAULT_BASIC_AUTH from proxy.common.constants import DEFAULT_NUM_WORKERS, DEFAULT_OPEN_FILE_LIMIT, DEFAULT_IPV6_HOSTNAME -from proxy.common.constants import DEFAULT_SERVER_RECVBUF_SIZE, DEFAULT_CLIENT_RECVBUF_SIZE, PY2_DEPRECATION_MESSAGE -from proxy.common.version import __version__ - - -def get_temp_file(name: str) -> str: - return os.path.join(tempfile.gettempdir(), name) +from proxy.common.constants import DEFAULT_SERVER_RECVBUF_SIZE, DEFAULT_CLIENT_RECVBUF_SIZE, DEFAULT_WORK_KLASS +from proxy.common.constants import PLUGIN_INSPECT_TRAFFIC, PLUGIN_DASHBOARD, PLUGIN_DEVTOOLS_PROTOCOL, PLUGIN_WEB_SERVER +from proxy.common.constants import PLUGIN_HTTP_PROXY, DEFAULT_NUM_ACCEPTORS, PLUGIN_PROXY_AUTH, DEFAULT_LOG_FORMAT class TestMain(unittest.TestCase): @staticmethod def mock_default_args(mock_args: mock.Mock) -> None: + """Use when trying to mock parse_args""" mock_args.version = False mock_args.cert_file = DEFAULT_CERT_FILE mock_args.key_file = DEFAULT_KEY_FILE @@ -51,129 +51,253 @@ def mock_default_args(mock_args: mock.Mock) -> None: mock_args.basic_auth = DEFAULT_BASIC_AUTH mock_args.hostname = DEFAULT_IPV6_HOSTNAME mock_args.port = DEFAULT_PORT + mock_args.num_acceptors = DEFAULT_NUM_ACCEPTORS mock_args.num_workers = DEFAULT_NUM_WORKERS mock_args.disable_http_proxy = DEFAULT_DISABLE_HTTP_PROXY - mock_args.enable_web_server = DEFAULT_ENABLE_WEB_SERVER mock_args.pac_file = DEFAULT_PAC_FILE mock_args.plugins = DEFAULT_PLUGINS + mock_args.auth_plugin = PLUGIN_PROXY_AUTH mock_args.server_recvbuf_size = DEFAULT_SERVER_RECVBUF_SIZE mock_args.client_recvbuf_size = DEFAULT_CLIENT_RECVBUF_SIZE mock_args.open_file_limit = DEFAULT_OPEN_FILE_LIMIT - mock_args.enable_static_server = DEFAULT_ENABLE_STATIC_SERVER - mock_args.enable_devtools = DEFAULT_ENABLE_DEVTOOLS mock_args.devtools_event_queue = None mock_args.devtools_ws_path = DEFAULT_DEVTOOLS_WS_PATH mock_args.timeout = DEFAULT_TIMEOUT mock_args.threadless = DEFAULT_THREADLESS + mock_args.threaded = not DEFAULT_THREADLESS + mock_args.enable_web_server = DEFAULT_ENABLE_WEB_SERVER + mock_args.enable_static_server = DEFAULT_ENABLE_STATIC_SERVER + mock_args.enable_devtools = DEFAULT_ENABLE_DEVTOOLS mock_args.enable_events = DEFAULT_ENABLE_EVENTS + mock_args.enable_dashboard = DEFAULT_ENABLE_DASHBOARD + mock_args.work_klass = DEFAULT_WORK_KLASS + mock_args.local_executor = DEFAULT_LOCAL_EXECUTOR - @mock.patch('time.sleep') - @mock.patch('proxy.proxy.Proxy.initialize') - @mock.patch('proxy.proxy.AcceptorPool') - @mock.patch('logging.basicConfig') - def test_init_with_no_arguments( - self, - mock_logging_config: mock.Mock, - mock_acceptor_pool: mock.Mock, - mock_initialize: mock.Mock, - mock_sleep: mock.Mock) -> None: - mock_sleep.side_effect = KeyboardInterrupt() - - input_args: List[str] = [] - main(input_args) - mock_acceptor_pool.assert_called_with( - flags=mock_initialize.return_value, - work_klass=HttpProtocolHandler, - ) - mock_acceptor_pool.return_value.setup.assert_called() - mock_acceptor_pool.return_value.shutdown.assert_called() - mock_sleep.assert_called() - - @mock.patch('time.sleep') @mock.patch('os.remove') @mock.patch('os.path.exists') @mock.patch('builtins.open') + @mock.patch('time.sleep') + @mock.patch('proxy.proxy.FlagParser.initialize') + @mock.patch('proxy.proxy.EventManager') @mock.patch('proxy.proxy.AcceptorPool') - @mock.patch('proxy.common.flag.FlagParser.parse_args') - def test_pid_file_is_written_and_removed( + @mock.patch('proxy.proxy.ThreadlessPool') + @mock.patch('proxy.proxy.Listener') + def test_entry_point( self, - mock_parse_args: mock.Mock, + mock_listener: mock.Mock, + mock_executor_pool: mock.Mock, mock_acceptor_pool: mock.Mock, + mock_event_manager: mock.Mock, + mock_initialize: mock.Mock, + mock_sleep: mock.Mock, mock_open: mock.Mock, mock_exists: mock.Mock, mock_remove: mock.Mock, - mock_sleep: mock.Mock) -> None: - pid_file = get_temp_file('pid') + ) -> None: + pid_file = os.path.join(tempfile.gettempdir(), 'pid') mock_sleep.side_effect = KeyboardInterrupt() - mock_args = mock_parse_args.return_value - self.mock_default_args(mock_args) - mock_args.pid_file = pid_file - main(['--pid-file', pid_file]) - mock_acceptor_pool.assert_called() - mock_acceptor_pool.return_value.setup.assert_called() + mock_initialize.return_value.local_executor = False + mock_initialize.return_value.enable_events = False + mock_initialize.return_value.pid_file = pid_file + entry_point() + mock_event_manager.assert_not_called() + mock_listener.assert_called_once_with( + flags=mock_initialize.return_value, + ) + mock_executor_pool.assert_called_once_with( + flags=mock_initialize.return_value, + event_queue=None, + ) + mock_acceptor_pool.assert_called_once_with( + flags=mock_initialize.return_value, + listener=mock_listener.return_value, + executor_queues=mock_executor_pool.return_value.work_queues, + executor_pids=mock_executor_pool.return_value.work_pids, + executor_locks=mock_executor_pool.return_value.work_locks, + event_queue=None, + ) + mock_acceptor_pool.return_value.setup.assert_called_once() + mock_acceptor_pool.return_value.shutdown.assert_called_once() + mock_listener.return_value.shutdown.assert_called_once() + mock_sleep.assert_called() + mock_open.assert_called_with(pid_file, 'wb') mock_open.return_value.__enter__.return_value.write.assert_called_with( - bytes_(os.getpid())) + bytes_(os.getpid()), + ) mock_exists.assert_called_with(pid_file) mock_remove.assert_called_with(pid_file) @mock.patch('time.sleep') + @mock.patch('proxy.proxy.FlagParser.initialize') + @mock.patch('proxy.proxy.EventManager') @mock.patch('proxy.proxy.AcceptorPool') - def test_basic_auth( + @mock.patch('proxy.proxy.ThreadlessPool') + @mock.patch('proxy.proxy.Listener') + def test_main_with_no_flags( self, + mock_listener: mock.Mock, + mock_executor_pool: mock.Mock, mock_acceptor_pool: mock.Mock, - mock_sleep: mock.Mock) -> None: + mock_event_manager: mock.Mock, + mock_initialize: mock.Mock, + mock_sleep: mock.Mock, + ) -> None: mock_sleep.side_effect = KeyboardInterrupt() + mock_initialize.return_value.local_executor = False + mock_initialize.return_value.enable_events = False + main() + mock_event_manager.assert_not_called() + mock_listener.assert_called_once_with( + flags=mock_initialize.return_value, + ) + mock_executor_pool.assert_called_once_with( + flags=mock_initialize.return_value, + event_queue=None, + ) + mock_acceptor_pool.assert_called_once_with( + flags=mock_initialize.return_value, + listener=mock_listener.return_value, + executor_queues=mock_executor_pool.return_value.work_queues, + executor_pids=mock_executor_pool.return_value.work_pids, + executor_locks=mock_executor_pool.return_value.work_locks, + event_queue=None, + ) + mock_acceptor_pool.return_value.setup.assert_called_once() + mock_acceptor_pool.return_value.shutdown.assert_called_once() + mock_listener.return_value.shutdown.assert_called_once() + mock_sleep.assert_called() - input_args = ['--basic-auth', 'user:pass'] - flgs = Proxy.initialize(input_args) + @mock.patch('time.sleep') + @mock.patch('proxy.proxy.FlagParser.initialize') + @mock.patch('proxy.proxy.EventManager') + @mock.patch('proxy.proxy.AcceptorPool') + @mock.patch('proxy.proxy.ThreadlessPool') + @mock.patch('proxy.proxy.Listener') + def test_enable_events( + self, + mock_listener: mock.Mock, + mock_executor_pool: mock.Mock, + mock_acceptor_pool: mock.Mock, + mock_event_manager: mock.Mock, + mock_initialize: mock.Mock, + mock_sleep: mock.Mock, + ) -> None: + mock_sleep.side_effect = KeyboardInterrupt() + mock_initialize.return_value.local_executor = False + mock_initialize.return_value.enable_events = True + main() + mock_event_manager.assert_called_once() + mock_event_manager.return_value.setup.assert_called_once() + mock_event_manager.return_value.shutdown.assert_called_once() + mock_listener.assert_called_once_with( + flags=mock_initialize.return_value, + ) + mock_executor_pool.assert_called_once_with( + flags=mock_initialize.return_value, + event_queue=mock_event_manager.return_value.queue, + ) + mock_acceptor_pool.assert_called_once_with( + flags=mock_initialize.return_value, + listener=mock_listener.return_value, + event_queue=mock_event_manager.return_value.queue, + executor_queues=mock_executor_pool.return_value.work_queues, + executor_pids=mock_executor_pool.return_value.work_pids, + executor_locks=mock_executor_pool.return_value.work_locks, + ) + mock_acceptor_pool.return_value.setup.assert_called_once() + mock_acceptor_pool.return_value.shutdown.assert_called_once() + mock_listener.return_value.shutdown.assert_called_once() + mock_sleep.assert_called() - main(input_args=input_args) - mock_acceptor_pool.assert_called_once() + @mock.patch('time.sleep') + @mock.patch('proxy.common.plugins.Plugins.load') + @mock.patch('proxy.common.flag.FlagParser.parse_args') + @mock.patch('proxy.proxy.EventManager') + @mock.patch('proxy.proxy.AcceptorPool') + @mock.patch('proxy.proxy.ThreadlessPool') + @mock.patch('proxy.proxy.Listener') + def test_enable_dashboard( + self, + mock_listener: mock.Mock, + mock_executor_pool: mock.Mock, + mock_acceptor_pool: mock.Mock, + mock_event_manager: mock.Mock, + mock_parse_args: mock.Mock, + mock_load_plugins: mock.Mock, + mock_sleep: mock.Mock, + ) -> None: + mock_sleep.side_effect = KeyboardInterrupt() + mock_args = mock_parse_args.return_value + self.mock_default_args(mock_args) + mock_args.enable_dashboard = True + main(enable_dashboard=True) + mock_load_plugins.assert_called() self.assertEqual( - flgs.auth_code, - b'dXNlcjpwYXNz') + mock_load_plugins.call_args_list[0][0][0], [ + bytes_(PLUGIN_WEB_SERVER), + bytes_(PLUGIN_DASHBOARD), + bytes_(PLUGIN_INSPECT_TRAFFIC), + bytes_(PLUGIN_DEVTOOLS_PROTOCOL), + bytes_(PLUGIN_HTTP_PROXY), + ], + ) + # TODO: Assert arguments passed to parse_arg + mock_parse_args.assert_called_once() + # dashboard will also enable eventing + mock_event_manager.assert_called_once() + mock_event_manager.return_value.setup.assert_called_once() + mock_event_manager.return_value.shutdown.assert_called_once() + mock_executor_pool.assert_called_once() + mock_executor_pool.return_value.setup.assert_called_once() + mock_acceptor_pool.assert_called_once() + mock_acceptor_pool.return_value.setup.assert_called_once() + mock_listener.return_value.setup.assert_called_once() @mock.patch('time.sleep') - @mock.patch('builtins.print') + @mock.patch('proxy.common.plugins.Plugins.load') + @mock.patch('proxy.common.flag.FlagParser.parse_args') + @mock.patch('proxy.proxy.EventManager') @mock.patch('proxy.proxy.AcceptorPool') - @mock.patch('proxy.proxy.Proxy.is_py3') - def test_main_py3_runs( + @mock.patch('proxy.proxy.ThreadlessPool') + @mock.patch('proxy.proxy.Listener') + def test_enable_devtools( self, - mock_is_py3: mock.Mock, + mock_listener: mock.Mock, + mock_executor_pool: mock.Mock, mock_acceptor_pool: mock.Mock, - mock_print: mock.Mock, - mock_sleep: mock.Mock) -> None: + mock_event_manager: mock.Mock, + mock_parse_args: mock.Mock, + mock_load_plugins: mock.Mock, + mock_sleep: mock.Mock, + ) -> None: mock_sleep.side_effect = KeyboardInterrupt() + mock_args = mock_parse_args.return_value + self.mock_default_args(mock_args) + mock_args.enable_devtools = True + main(enable_devtools=True) + mock_load_plugins.assert_called() + self.assertEqual( + mock_load_plugins.call_args_list[0][0][0], [ + bytes_(PLUGIN_DEVTOOLS_PROTOCOL), + bytes_(PLUGIN_WEB_SERVER), + bytes_(PLUGIN_HTTP_PROXY), + ], + ) + mock_parse_args.assert_called_once() + # Currently --enable-devtools flag alone doesn't enable eventing core + mock_event_manager.assert_not_called() + mock_executor_pool.assert_called_once() + mock_executor_pool.return_value.setup.assert_called_once() + mock_acceptor_pool.assert_called_once() + mock_acceptor_pool.return_value.setup.assert_called_once() + mock_listener.return_value.setup.assert_called_once() - input_args = ['--basic-auth', 'user:pass'] - mock_is_py3.return_value = True - - main(input_args, num_workers=1) + # def test_pac_file(self) -> None: + # pass - mock_is_py3.assert_called() - mock_print.assert_not_called() - mock_acceptor_pool.assert_called_once() - mock_acceptor_pool.return_value.setup.assert_called() + # def test_imports_plugin(self) -> None: + # pass - @mock.patch('builtins.print') - @mock.patch('proxy.proxy.Proxy.is_py3') - def test_main_py2_exit( - self, - mock_is_py3: mock.Mock, - mock_print: mock.Mock) -> None: - mock_is_py3.return_value = False - with self.assertRaises(SystemExit) as e: - main(num_workers=1) - mock_print.assert_called_with(PY2_DEPRECATION_MESSAGE) - self.assertEqual(e.exception.code, 1) - mock_is_py3.assert_called() - - @mock.patch('builtins.print') - def test_main_version( - self, - mock_print: mock.Mock) -> None: - with self.assertRaises(SystemExit) as e: - main(['--version']) - mock_print.assert_called_with(__version__) - self.assertEqual(e.exception.code, 0) + # def test_cannot_enable_https_proxy_and_tls_interception_mutually(self) -> None: + # pass diff --git a/tests/test_set_open_file_limit.py b/tests/test_set_open_file_limit.py index 3bae38cfe2..c785b5adb6 100644 --- a/tests/test_set_open_file_limit.py +++ b/tests/test_set_open_file_limit.py @@ -8,19 +8,22 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -import os import unittest from unittest import mock -from proxy.proxy import Proxy +import pytest -if os.name != 'nt': +from proxy.common._compat import IS_WINDOWS # noqa: WPS436 +from proxy.common.utils import set_open_file_limit + +if not IS_WINDOWS: import resource -@unittest.skipIf( - os.name == 'nt', - 'Open file limit tests disabled for Windows') +@pytest.mark.skipif( + IS_WINDOWS, + reason='Open file limit tests disabled for Windows', +) class TestSetOpenFileLimit(unittest.TestCase): @mock.patch('resource.getrlimit', return_value=(128, 1024)) @@ -28,8 +31,9 @@ class TestSetOpenFileLimit(unittest.TestCase): def test_set_open_file_limit( self, mock_set_rlimit: mock.Mock, - mock_get_rlimit: mock.Mock) -> None: - Proxy.set_open_file_limit(256) + mock_get_rlimit: mock.Mock, + ) -> None: + set_open_file_limit(256) mock_get_rlimit.assert_called_with(resource.RLIMIT_NOFILE) mock_set_rlimit.assert_called_with(resource.RLIMIT_NOFILE, (256, 1024)) @@ -38,8 +42,9 @@ def test_set_open_file_limit( def test_set_open_file_limit_not_called( self, mock_set_rlimit: mock.Mock, - mock_get_rlimit: mock.Mock) -> None: - Proxy.set_open_file_limit(256) + mock_get_rlimit: mock.Mock, + ) -> None: + set_open_file_limit(256) mock_get_rlimit.assert_called_with(resource.RLIMIT_NOFILE) mock_set_rlimit.assert_not_called() @@ -48,7 +53,8 @@ def test_set_open_file_limit_not_called( def test_set_open_file_limit_not_called_coz_upper_bound_check( self, mock_set_rlimit: mock.Mock, - mock_get_rlimit: mock.Mock) -> None: - Proxy.set_open_file_limit(1024) + mock_get_rlimit: mock.Mock, + ) -> None: + set_open_file_limit(1024) mock_get_rlimit.assert_called_with(resource.RLIMIT_NOFILE) mock_set_rlimit.assert_not_called() diff --git a/tests/testing/test_embed.py b/tests/testing/test_embed.py index e69c5ffe56..3bb9912e74 100644 --- a/tests/testing/test_embed.py +++ b/tests/testing/test_embed.py @@ -8,49 +8,48 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -import os -import unittest import http.client import urllib.request import urllib.error +import pytest + from proxy import TestCase +from proxy.common._compat import IS_WINDOWS # noqa: WPS436 from proxy.common.constants import DEFAULT_CLIENT_RECVBUF_SIZE, PROXY_AGENT_HEADER_VALUE -from proxy.common.utils import socket_connection, build_http_request, build_http_response -from proxy.http.codes import httpStatusCodes -from proxy.http.methods import httpMethods +from proxy.common.utils import socket_connection, build_http_request +from proxy.http import httpMethods +from proxy.http.server import HttpWebServerPlugin -@unittest.skipIf( - os.name == 'nt', 'Disabled for Windows due to weird permission issues.') +@pytest.mark.skipif( + IS_WINDOWS, + reason='Disabled for Windows due to weird permission issues.', +) class TestProxyPyEmbedded(TestCase): """This test case is a demonstration of proxy.TestCase and also serves as integration test suite for proxy.py.""" PROXY_PY_STARTUP_FLAGS = TestCase.DEFAULT_PROXY_PY_STARTUP_FLAGS + [ - '--enable-web-server', + '--enable-web-server', '--port', '0', ] def test_with_proxy(self) -> None: """Makes a HTTP request to in-build web server via proxy server.""" - with socket_connection(('localhost', self.PROXY_PORT)) as conn: + assert self.PROXY and self.PROXY.acceptors + with socket_connection(('localhost', self.PROXY.flags.port)) as conn: conn.send( build_http_request( - httpMethods.GET, b'http://localhost:%d/' % self.PROXY_PORT, + httpMethods.GET, b'http://localhost:%d/' % self.PROXY.acceptors.flags.port, headers={ - b'Host': b'localhost:%d' % self.PROXY_PORT, - }) + b'Host': b'localhost:%d' % self.PROXY.acceptors.flags.port, + }, + ), ) response = conn.recv(DEFAULT_CLIENT_RECVBUF_SIZE) self.assertEqual( response, - build_http_response( - httpStatusCodes.NOT_FOUND, reason=b'NOT FOUND', - headers={ - b'Server': PROXY_AGENT_HEADER_VALUE, - b'Connection': b'close' - } - ) + HttpWebServerPlugin.DEFAULT_404_RESPONSE.tobytes(), ) def test_proxy_vcr(self) -> None: @@ -70,14 +69,16 @@ def test_proxy_no_vcr(self) -> None: self.make_http_request_using_proxy() def make_http_request_using_proxy(self) -> None: + assert self.PROXY and self.PROXY.acceptors proxy_handler = urllib.request.ProxyHandler({ - 'http': 'http://localhost:%d' % self.PROXY_PORT, + 'http': 'http://localhost:%d' % self.PROXY.flags.port, }) opener = urllib.request.build_opener(proxy_handler) with self.assertRaises(urllib.error.HTTPError): r: http.client.HTTPResponse = opener.open( 'http://localhost:%d/' % - self.PROXY_PORT, timeout=10) + self.PROXY.flags.port, timeout=10, + ) self.assertEqual(r.status, 404) self.assertEqual(r.headers.get('server'), PROXY_AGENT_HEADER_VALUE) self.assertEqual(r.headers.get('connection'), b'close') diff --git a/tests/testing/test_test_case.py b/tests/testing/test_test_case.py index c1dafa07bb..daa28d0d63 100644 --- a/tests/testing/test_test_case.py +++ b/tests/testing/test_test_case.py @@ -16,7 +16,8 @@ class TestTestCase(unittest.TestCase): - def test_wait_for_server(self) -> None: + def test_wait_for_server_raises_timeout_error(self) -> None: with self.assertRaises(TimeoutError): proxy.TestCase.wait_for_server( - get_available_port(), wait_for_seconds=1) + get_available_port(), wait_for_seconds=0.1, + ) diff --git a/tox.ini b/tox.ini index 8fb4694b3d..daa25f9bb3 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,235 @@ [tox] -envlist = py35,py36,py37,py38 +envlist = py36,py37,py38,py39,py310 +isolated_build = true +minversion = 3.21.0 [testenv] deps = -rrequirements.txt -rrequirements-testing.txt -command = pytest + -rrequirements-tunnel.txt +# NOTE: The command is invoked by the script name and not via +# NOTE: `{envpython} -m pytest` because it'd add CWD into $PYTHONPATH +# NOTE: testing the project from the Git checkout +# NOTE: rather than one installed. +commands = pytest {posargs:} + + +[dists] +setenv = + PEP517_OUT_DIR = {env:PEP517_OUT_DIR:{toxinidir}{/}dist} + + +[testenv:build-docs] +allowlist_externals = + git +basepython = python3 +commands_pre = + # Paramiko: + {envpython} -m pip install -r{toxinidir}/requirements-tunnel.txt +commands = + # Retrieve possibly missing commits: + -git fetch --unshallow + -git fetch --tags + + # Build the html docs with Sphinx: + {envpython} -m sphinx \ + -j auto \ + -b html \ + {tty:--color} \ + -a \ + -n \ + -W --keep-going \ + -d "{temp_dir}/.doctrees" \ + . \ + "{envdir}/docs_out" + + # Print out the output docs dir and a way to serve html: + -{envpython} -c\ + 'import pathlib;\ + docs_dir = pathlib.Path(r"{envdir}") / "docs_out";\ + index_file = docs_dir / "index.html";\ + print("\n" + "=" * 120 +\ + f"\n\nDocumentation available under:\n\n\ + \tfile://\{index_file\}\n\nTo serve docs, use\n\n\ + \t$ python3 -m http.server --directory \ + \N\{QUOTATION MARK\}\{docs_dir\}\N\{QUOTATION MARK\} 0\n\n" +\ + "=" * 120)' +changedir = {toxinidir}/docs +deps = + -r{toxinidir}/docs/requirements.txt + # FIXME: re-enable the "-r" + "-c" paradigm once the pip bug is fixed. + # Ref: https://github.com/pypa/pip/issues/9243 + # -r{toxinidir}/docs/requirements.in + # -c{toxinidir}/docs/requirements.txt +description = Build The Docs +isolated_build = true +passenv = + SSH_AUTH_SOCK +skip_install = false +usedevelop = false + + +[testenv:doctest-docs] +allowlist_externals = + {[testenv:build-docs]allowlist_externals} +basepython = {[testenv:build-docs]basepython} +commands_pre = {[testenv:build-docs]commands_pre} +commands = + # Retrieve possibly missing commits: + -git fetch --unshallow + -git fetch --tags + + # Spellcheck docs site: + python -m sphinx \ + -j auto \ + -a -n -W \ + --keep-going \ + -b doctest --color \ + -d "{toxworkdir}/docs_doctree" \ + . "{toxworkdir}/docs_out" +changedir = {[testenv:build-docs]changedir} +deps = {[testenv:build-docs]deps} +description = Doctest The Docs +isolated_build = {[testenv:build-docs]isolated_build} +passenv = {[testenv:build-docs]passenv} +skip_install = {[testenv:build-docs]skip_install} +usedevelop = {[testenv:build-docs]usedevelop} + + +[testenv:linkcheck-docs] +allowlist_externals = + {[testenv:build-docs]allowlist_externals} +basepython = {[testenv:build-docs]basepython} +commands_pre = {[testenv:build-docs]commands_pre} +commands = + # Retrieve possibly missing commits: + -git fetch --unshallow + -git fetch --tags + + # Spellcheck docs site: + python -m sphinx \ + -j auto \ + -a -n -W \ + --keep-going \ + -b linkcheck --color \ + -d "{toxworkdir}/docs_doctree" \ + . "{toxworkdir}/docs_out" +changedir = {[testenv:build-docs]changedir} +deps = {[testenv:build-docs]deps} +description = Linkcheck The Docs +isolated_build = {[testenv:build-docs]isolated_build} +passenv = {[testenv:build-docs]passenv} +skip_install = {[testenv:build-docs]skip_install} +usedevelop = {[testenv:build-docs]usedevelop} + + +[testenv:spellcheck-docs] +allowlist_externals = + {[testenv:build-docs]allowlist_externals} +basepython = {[testenv:build-docs]basepython} +commands_pre = {[testenv:build-docs]commands_pre} +commands = + # Retrieve possibly missing commits: + -git fetch --unshallow + -git fetch --tags + + # Spellcheck docs site: + python -m sphinx \ + -j auto \ + -a -n -W \ + --keep-going \ + -b spelling --color \ + -d "{toxworkdir}/docs_doctree" \ + . "{toxworkdir}/docs_out" +changedir = {[testenv:build-docs]changedir} +deps = + sphinxcontrib-spelling >= 7.2.0 + -r{toxinidir}/docs/requirements.in +description = Spellcheck The Docs +isolated_build = {[testenv:build-docs]isolated_build} +passenv = {[testenv:build-docs]passenv} +skip_install = {[testenv:build-docs]skip_install} +usedevelop = {[testenv:build-docs]usedevelop} + + +[testenv:cleanup-dists] +description = + Wipe the the `{env:PEP517_OUT_DIR}{/}` folder +usedevelop = false +skip_install = true +deps = +setenv = + {[dists]setenv} +commands = + {envpython} -c \ + 'import os, shutil, sys; dists_dir = os.getenv("PEP517_OUT_DIR"); shutil.rmtree(dists_dir, ignore_errors=True); sys.exit(os.path.exists(dists_dir))' + + +[testenv:build-dists] +description = + Build non-universal dists and put them into + the `{env:PEP517_OUT_DIR}{/}` folder +depends = + cleanup-dists +isolated_build = true +# `usedevelop = true` overrides `skip_install` instruction, it's unwanted +usedevelop = false +skip_install = true +deps = + build >= 0.7.0, < 0.8.0 +passenv = + PEP517_BUILD_ARGS +setenv = + {[dists]setenv} +commands = + {envpython} -m build \ + --outdir '{env:PEP517_OUT_DIR}{/}' \ + {posargs:{env:PEP517_BUILD_ARGS:}} \ + '{toxinidir}' + + +[testenv:metadata-validation] +description = + Verify that dists under the `{env:PEP517_OUT_DIR}{/}` dir + have valid metadata +depends = + build-dists +deps = + twine +usedevelop = false +skip_install = true +setenv = + {[dists]setenv} +commands = + {envpython} -m twine check \ + --strict \ + {env:PEP517_OUT_DIR}{/}* + + +[testenv:lint] +description = + Enforce quality standards under `{basepython}` ({envpython}) +commands = + {envpython} -m \ + pre_commit run \ + --show-diff-on-failure \ + --hook-stage manual \ + {posargs:--all-files -v} + + # Print out the advice on how to install pre-commit from this env into Git: + -{envpython} -c \ + 'cmd = "{envpython} -m pre_commit install"; \ + scr_width = len(cmd) + 10; \ + sep = "=" * scr_width; \ + cmd_str = " $ \{cmd\}";' \ + 'print(f"\n\{sep\}\nTo install pre-commit hooks into the Git repo, run:\n\n\{cmd_str\}\n\n\{sep\}\n")' +deps = + pre-commit + pylint >= 2.5.3 + pylint-pytest < 1.1.0 + pytest-mock >= 3.6.1 + -r docs/requirements.in + -r requirements-tunnel.txt +isolated_build = true +skip_install = true diff --git a/version-check.py b/version-check.py deleted file mode 100644 index e783a2a36c..0000000000 --- a/version-check.py +++ /dev/null @@ -1,38 +0,0 @@ -# -*- coding: utf-8 -*- -""" - proxy.py - ~~~~~~~~ - โšกโšกโšก Fast, Lightweight, Programmable, TLS interception capable - proxy server for Application debugging, testing and development. - - :copyright: (c) 2013-present by Abhinav Singh and contributors. - :license: BSD, see LICENSE for more details. -""" -import sys -import subprocess -from proxy.common.version import __version__ as lib_version -from setup import __version__ as pkg_version - -# This script ensures our versions never run out of sync. -# -# 1. TODO: Version is hardcoded in homebrew stable package -# installer file, but it only needs to match with lib -# versions if current git branch is master - -# setup.py doesn't import proxy and hence they both use -# their own respective __version__ -if lib_version != pkg_version: - print('Version mismatch found. {0} (lib) vs {1} (pkg).'.format( - lib_version, pkg_version)) - sys.exit(1) - -# Version is also hardcoded in README.md flags section -readme_version_cmd = 'cat README.md | grep "proxy.py v" | tail -2 | head -1 | cut -d " " -f 2 | cut -c2-' -readme_version_output = subprocess.check_output( - ['bash', '-c', readme_version_cmd]) -readme_version = readme_version_output.decode().strip() - -if readme_version != lib_version: - print('Version mismatch found. {0} (readme) vs {1} (lib).'.format( - readme_version, lib_version)) - sys.exit(1) diff --git a/write-scm-version.sh b/write-scm-version.sh new file mode 100755 index 0000000000..320e8ca60f --- /dev/null +++ b/write-scm-version.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# +# write-scm-version.sh exists because `proxy.py`` +# auto-detects it's next version from git. Hence, +# for `proxy.py` to work `proxy/common/_scm_version.py` +# file must be auto-generated with necessary information. +# +# For CI/CD, this file is generated via `tox` integration. +# For local development (without editable install), you +# must run this script to pre-populate `_scm_version.py`. +# +# This file is integrated by default within `Makefile`. +# For every make target invocation, `_scm_version.py` file +# will be re-written. + +# Guessed Version 2.3.2.dev146+gad54132.d20211114 +VERSION=$(python -m setuptools_scm --version | \ + # 2.3.2.dev146+gad54132.d20211114 + awk '{print $3}') + +# Store default IFS +OLDIFS=$IFS + +IFS="+" +set -- $VERSION +SEMVER=$1 +DATE_AND_HASH=$2 + +IFS="." +set -- $SEMVER +MAJOR=$1 +MINOR=$2 +PATCH=$3 +DISTANCE=$4 + +# Reset IFS +IFS=$OLDIFS + +echo "# coding: utf-8 +# file generated by setuptools_scm +# don't change, don't track in version control +version = '${VERSION}' +version_tuple = (${MAJOR}, ${MINOR}, ${PATCH}, '${DISTANCE}', '${DATE_AND_HASH}')" > \ + proxy/common/_scm_version.py + +echo $MAJOR.$MINOR.$PATCH.$DISTANCE