From e5e900f477f3485ff46ae8f17ee4941543d1afe6 Mon Sep 17 00:00:00 2001 From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com> Date: Thu, 25 Nov 2021 20:28:26 +0530 Subject: [PATCH] Build docker container for all linux target architectures`386`, `amd64`, `arm/v6`, `arm/v7`, `arm64/v8`, `ppc64le`, `s390x` (#797) * Fixes #768 * buildx build * We need `setuptools_scm` to run `Makefile` * Build `docker` container using `wheel` * Download package distribution * Adjust supported platforms * Alpine only supports v6 * . * May-be fix `lsb_release` * Enable `multiarch` env for docker buildx * Deprecate `DOCKER_IMAGE_TAG` which removes `write-scm-version.py` dependency. Also pass targetplatform as an argument to `container-buildx` make target * Remove `setuptools_scm` dep for docker workflow step * woof * Match all target platforms to match `python-alpine` docker * yamllint * Use `yamllint disable rule:line-length` for `Dockerfile` * Tag the container using `dist-version` * Replace + with . for dev version tag * Add `PROXYPY_CONTAINER_VERSION` step --- .dockerignore | 10 +---- .github/buildkitd.toml | 4 ++ .github/workflows/test-library.yml | 72 ++++++++++++++++++++---------- Dockerfile | 42 ++++++++--------- Makefile | 39 +++++++++++----- README.md | 30 ++++++++++++- 6 files changed, 131 insertions(+), 66 deletions(-) create mode 100644 .github/buildkitd.toml diff --git a/.dockerignore b/.dockerignore index 3483deb2b5..3bd432ac85 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,13 +1,5 @@ # Ignore everything ** -# Except proxy -!.git -!proxy -!requirements.txt -!pyproject.toml -!setup.cfg +!dist/*.whl !README.md - -# Ignore __pycache__ directory -proxy/__pycache__ diff --git a/.github/buildkitd.toml b/.github/buildkitd.toml new file mode 100644 index 0000000000..59b12d17bd --- /dev/null +++ b/.github/buildkitd.toml @@ -0,0 +1,4 @@ +[worker.oci] + max-parallelism = 4 +[registry."docker.io"] + mirrors = ["mirror.gcr.io"] diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index 2d983cc55e..924f4735c6 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -1,4 +1,5 @@ --- +# yamllint disable rule:line-length name: lib on: # yamllint disable-line rule:truthy @@ -609,7 +610,7 @@ jobs: matrix: os: [ubuntu, windows, macOS] node: ['10.x', '11.x', '12.x'] - max-parallel: 4 + # max-parallel: 4 fail-fast: false steps: - uses: actions/checkout@v2 @@ -629,37 +630,62 @@ jobs: cd .. docker: - # To build our docker container, we must wait for check, + # TODO: To build our docker container, we must wait for check, # so that we can use the same distribution available. - # - # TL;DR -- Docker must be packaged with package from - # test.pypi.org or from pypi.org. - # needs: - # - publish-pypi runs-on: ${{ matrix.os }}-latest - name: 🐳 🐍${{ matrix.python }} @ ${{ matrix.os }} + needs: + - build + - pre-setup # transitive, for accessing settings + name: 🐳 🐍${{ matrix.python }} @ ${{ matrix.targetplatform }} strategy: matrix: - os: [ubuntu] - python: ['3.10'] - max-parallel: 1 + os: + - Ubuntu + python: + - '3.10' + targetplatform: + - 'linux/386' + - 'linux/amd64' + - 'linux/arm/v6' + - 'linux/arm/v7' + - 'linux/arm64/v8' + - 'linux/ppc64le' + - 'linux/s390x' + # max-parallel: 1 fail-fast: false steps: - - uses: actions/checkout@v2 - - name: Setup Python - uses: actions/setup-python@v2 + - name: Checkout + uses: actions/checkout@v2 + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 with: - python-version: ${{ matrix.python }} - - name: Install dependencies + buildkitd-flags: --debug + config: .github/buildkitd.toml + install: true + - name: Download all the dists + uses: actions/download-artifact@v2 + with: + name: python-package-distributions + path: dist/ + - name: Enable Multiarch # This slows down arm build by 4-5x + run: | + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + - name: Create builder + run: | + docker buildx create --name proxypybuilder + docker buildx use proxypybuilder + docker buildx inspect + docker buildx ls + - name: Set PROXYPY_CONTAINER_VERSION run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install -r requirements-release.txt - pip install -r requirements-testing.txt - pip install -r requirements-tunnel.txt - - name: Build + echo "PROXYPY_CONTAINER_VERSION=$(echo '${{ needs.pre-setup.outputs.dist-version }}' | tr + .)" > $GITHUB_ENV + - name: Build container run: | - make container + make container-buildx \ + -e PROXYPY_PKG_PATH='dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}' \ + -e BUILDX_TARGET_PLATFORM='${{ matrix.targetplatform }}' \ + -e PROXYPY_CONTAINER_VERSION='${{ env.PROXYPY_CONTAINER_VERSION }}' check: # This job does nothing and is only used for the branch protection needs: diff --git a/Dockerfile b/Dockerfile index 0f37fda244..0d59cd5bf5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,30 +1,26 @@ FROM python:3.10-alpine as base -RUN apk add git - -FROM base as builder - -COPY .git /app/.git -COPY requirements.txt /app/ -COPY pyproject.toml /app/ -COPY setup.cfg /app/ -COPY README.md /app/ -COPY proxy/ /app/proxy/ -WORKDIR /app -RUN pip install --upgrade pip && \ - pip install --prefix=/deps . - -FROM base - LABEL com.abhinavsingh.name="abhinavsingh/proxy.py" \ - com.abhinavsingh.description="⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on \ - Network monitoring, controls & Application development, testing, debugging." \ - com.abhinavsingh.url="https://github.com/abhinavsingh/proxy.py" \ - com.abhinavsingh.vcs-url="https://github.com/abhinavsingh/proxy.py" \ - com.abhinavsingh.docker.cmd="docker run -it --rm -p 8899:8899 abhinavsingh/proxy.py" + com.abhinavsingh.description="⚡ Fast • 🪶 Lightweight • 0️⃣ Dependency • 🔌 Pluggable • \ + 😈 TLS interception • 🔒 DNS-over-HTTPS • 🔥 Poor Man's VPN • ⏪ Reverse & ⏩ Forward • \ + 👮🏿 \"Proxy Server\" framework • 🌐 \"Web Server\" framework • ➵ ➶ ➷ ➠ \"PubSub\" framework • \ + 👷 \"Work\" acceptor & executor framework" \ + com.abhinavsingh.url="https://github.com/abhinavsingh/proxy.py" \ + com.abhinavsingh.vcs-url="https://github.com/abhinavsingh/proxy.py" \ + com.abhinavsingh.docker.cmd="docker run -it --rm -p 8899:8899 abhinavsingh/proxy.py" +ENV PYTHONUNBUFFERED 1 +ARG PROXYPY_PKG_PATH -COPY --from=builder /deps /usr/local +COPY README.md / +COPY $PROXYPY_PKG_PATH / +RUN pip install --upgrade pip && \ + pip install \ + --no-index \ + --find-links file:/// \ + proxy.py && \ + rm *.whl -# Install openssl to enable TLS interception within container +# Install openssl to enable TLS interception & HTTPS proxy options within container +# NOTE: You can comment out this line if you don't intend to use those features. RUN apk update && apk add openssl EXPOSE 8899/tcp diff --git a/Makefile b/Makefile index 418fd8b91e..d54ab9242f 100644 --- a/Makefile +++ b/Makefile @@ -2,8 +2,10 @@ SHELL := /bin/bash NS ?= abhinavsingh IMAGE_NAME ?= proxy.py -LATEST_TAG := $(NS)/$(IMAGE_NAME):latest -IMAGE_TAG := $(NS)/$(IMAGE_NAME):$(shell ./write-scm-version.sh) +# Override to target specific versions of proxy.py +PROXYPY_CONTAINER_VERSION := latest +# Used by container build and run targets +PROXYPY_CONTAINER_TAG := $(NS)/$(IMAGE_NAME):$(PROXYPY_CONTAINER_VERSION) HTTPS_KEY_FILE_PATH := https-key.pem HTTPS_CERT_FILE_PATH := https-cert.pem @@ -14,6 +16,10 @@ CA_KEY_FILE_PATH := ca-key.pem CA_CERT_FILE_PATH := ca-cert.pem CA_SIGNING_KEY_FILE_PATH := ca-signing-key.pem +# Dummy invalid hardcoded value +PROXYPY_PKG_PATH := dist/proxy.py.whl +BUILDX_TARGET_PLATFORM := linux/amd64 + OPEN=$(shell which open) UNAME := $(shell uname) ifeq ($(UNAME), Linux) @@ -24,7 +30,7 @@ endif .PHONY: lib-check lib-clean lib-test lib-package lib-coverage lib-lint lib-pytest .PHONY: lib-release-test lib-release lib-profile lib-doc .PHONY: lib-dep lib-flake8 lib-mypy -.PHONY: container container-run container-release +.PHONY: container container-run container-release container-build container-buildx .PHONY: devtools dashboard dashboard-clean all: lib-test @@ -151,12 +157,25 @@ dashboard: dashboard-clean: if [[ -d dashboard/public ]]; then rm -rf dashboard/public; fi -container: - docker build -t $(LATEST_TAG) -t $(IMAGE_TAG) . - -container-release: - docker push $(IMAGE_TAG) - docker push $(LATEST_TAG) +container: lib-package + $(MAKE) container-build -e PROXYPY_PKG_PATH=$$(ls dist/*.whl) + +# Usage: +# +# make container-buildx \ +# -e PROXYPY_PKG_PATH=$(ls dist/*.whl) \ +# -e BUILDX_TARGET_PLATFORM=linux/arm64 \ +# -e PROXYPY_CONTAINER_VERSION=latest +container-buildx: + docker buildx build \ + --platform $(BUILDX_TARGET_PLATFORM) \ + -t $(PROXYPY_CONTAINER_TAG) \ + --build-arg PROXYPY_PKG_PATH=$(PROXYPY_PKG_PATH) . + +container-build: + docker build \ + -t $(PROXYPY_CONTAINER_TAG) \ + --build-arg PROXYPY_PKG_PATH=$(PROXYPY_PKG_PATH) . container-run: - docker run -it -p 8899:8899 --rm $(LATEST_TAG) + docker run -it -p 8899:8899 --rm $(PROXYPY_CONTAINER_TAG) diff --git a/README.md b/README.md index 4d611de448..e9ba838e9f 100644 --- a/README.md +++ b/README.md @@ -257,12 +257,30 @@ or from GitHub `master` branch ## Using Docker +Stable version container releases are available for following platforms: + +- `linux/386` +- `linux/amd64` +- `linux/arm/v6` +- `linux/arm/v7` +- `linux/arm64/v8` +- `linux/ppc64le` +- `linux/s390x` + ### Stable Version from Docker Hub +Run `proxy.py` latest container: + ```console ❯ docker run -it -p 8899:8899 --rm abhinavsingh/proxy.py:latest ``` +To run specific target platform container on multi-platform supported servers: + +```console +❯ docker run -it -p 8899:8899 --rm --platform linux/arm64/v8 abhinavsingh/proxy.py:latest +``` + ### Build Development Version Locally ```console @@ -394,6 +412,16 @@ To start `proxy.py` from source code follow these instructions: ❯ make lib-dep ``` +- Generate `proxy/common/_scm_version.py` + + NOTE: *Following step is not necessary for editable installs.* + + This file writes SCM detected version to `proxy/common/_scm_version.py` file. + + ```console + ❯ ./write-scm-version.sh + ``` + - Optionally, run tests ```console @@ -2038,7 +2066,7 @@ usage: -m [-h] [--enable-events] [--enable-conn-pool] [--threadless] [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG] [--cloudflare-dns-mode CLOUDFLARE_DNS_MODE] -proxy.py v2.3.2.dev183+g808caa1.d20211124 +proxy.py v2.3.2.dev190+ge60d80d.d20211124 options: -h, --help show this help message and exit