Build docker container for all linux target architectures`386`, `amd64`, `arm/v6`, `arm/v7`, `arm64/v8`, `ppc64le`, `s390x` (#797)

* Fixes #768 * buildx build * We need `setuptools_scm` to run `Makefile` * Build `docker` container using `wheel` * Download package distribution * Adjust supported platforms * Alpine only supports v6 * . * May-be fix `lsb_release` * Enable `multiarch` env for docker buildx * Deprecate `DOCKER_IMAGE_TAG` which removes `write-scm-version.py` dependency. Also pass targetplatform as an argument to `container-buildx` make target * Remove `setuptools_scm` dep for docker workflow step * woof * Match all target platforms to match `python-alpine` docker * yamllint * Use `yamllint disable rule:line-length` for `Dockerfile` * Tag the container using `dist-version` * Replace + with . for dev version tag * Add `PROXYPY_CONTAINER_VERSION` step
2021-11-25 20:28:26 +05:30 · 2021-11-25 20:28:26 +05:30 · e5e900f477
parent 3ff7f211e7
commit e5e900f477
6 changed files with 130 additions and 65 deletions
--- a/.dockerignore
+++ b/.dockerignore
@ -1,13 +1,5 @@
 # Ignore everything
 **

-# Except proxy
-!.git
-!proxy
-!requirements.txt
-!pyproject.toml
-!setup.cfg
+!dist/*.whl
 !README.md
-
-# Ignore __pycache__ directory
-proxy/__pycache__
--- a/.github/buildkitd.toml
+++ b/.github/buildkitd.toml
@ -0,0 +1,4 @@
+[worker.oci]
+  max-parallelism = 4
+[registry."docker.io"]
+  mirrors = ["mirror.gcr.io"]
--- a/.github/workflows/test-library.yml
+++ b/.github/workflows/test-library.yml
@ -1,4 +1,5 @@
 ---
+# yamllint disable rule:line-length
 name: lib

 on:   # yamllint disable-line rule:truthy
@ -609,7 +610,7 @@ jobs:
      matrix:
        os: [ubuntu, windows, macOS]
        node: ['10.x', '11.x', '12.x']
-      max-parallel: 4
+      # max-parallel: 4
      fail-fast: false
    steps:
    - uses: actions/checkout@v2
@ -629,37 +630,62 @@ jobs:
        cd ..

  docker:
-    # To build our docker container, we must wait for check,
+    # TODO: To build our docker container, we must wait for check,
    # so that we can use the same distribution available.
-    #
-    # TL;DR -- Docker must be packaged with package from
-    # test.pypi.org or from pypi.org.
-    # needs:
-    # - publish-pypi
    runs-on: ${{ matrix.os }}-latest
-    name: 🐳 🐍${{ matrix.python }} @ ${{ matrix.os }}
+    needs:
+    - build
+    - pre-setup  # transitive, for accessing settings
+    name: 🐳 🐍${{ matrix.python }} @ ${{ matrix.targetplatform }}
    strategy:
      matrix:
-        os: [ubuntu]
-        python: ['3.10']
-      max-parallel: 1
+        os:
+        - Ubuntu
+        python:
+        - '3.10'
+        targetplatform:
+        - 'linux/386'
+        - 'linux/amd64'
+        - 'linux/arm/v6'
+        - 'linux/arm/v7'
+        - 'linux/arm64/v8'
+        - 'linux/ppc64le'
+        - 'linux/s390x'
+      # max-parallel: 1
      fail-fast: false
    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Python
-      uses: actions/setup-python@v2
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Set up Docker Buildx
+      id: buildx
+      uses: docker/setup-buildx-action@v1
      with:
-        python-version: ${{ matrix.python }}
-    - name: Install dependencies
+        buildkitd-flags: --debug
+        config: .github/buildkitd.toml
+        install: true
+    - name: Download all the dists
+      uses: actions/download-artifact@v2
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Enable Multiarch    # This slows down arm build by 4-5x
      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-        pip install -r requirements-release.txt
-        pip install -r requirements-testing.txt
-        pip install -r requirements-tunnel.txt
-    - name: Build
+        docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+    - name: Create builder
      run: |
-        make container
+        docker buildx create --name proxypybuilder
+        docker buildx use proxypybuilder
+        docker buildx inspect
+        docker buildx ls
+    - name: Set PROXYPY_CONTAINER_VERSION
+      run: |
+        echo "PROXYPY_CONTAINER_VERSION=$(echo '${{ needs.pre-setup.outputs.dist-version }}' | tr + .)" > $GITHUB_ENV
+    - name: Build container
+      run: |
+        make container-buildx \
+          -e PROXYPY_PKG_PATH='dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}' \
+          -e BUILDX_TARGET_PLATFORM='${{ matrix.targetplatform }}' \
+          -e PROXYPY_CONTAINER_VERSION='${{ env.PROXYPY_CONTAINER_VERSION }}'

  check:  # This job does nothing and is only used for the branch protection
    needs:
--- a/36
+++ b/36
@ -1,30 +1,26 @@
 FROM python:3.10-alpine as base
-RUN apk add git
-
-FROM base as builder
-
-COPY .git /app/.git
-COPY requirements.txt /app/
-COPY pyproject.toml /app/
-COPY setup.cfg /app/
-COPY README.md /app/
-COPY proxy/ /app/proxy/
-WORKDIR /app
-RUN pip install --upgrade pip && \
-    pip install --prefix=/deps .
-
-FROM base
-
 LABEL com.abhinavsingh.name="abhinavsingh/proxy.py" \
-      com.abhinavsingh.description="⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on \
-        Network monitoring, controls & Application development, testing, debugging." \
+  com.abhinavsingh.description="⚡ Fast • 🪶 Lightweight • 0️⃣ Dependency • 🔌 Pluggable • \
+  😈 TLS interception • 🔒 DNS-over-HTTPS • 🔥 Poor Man's VPN • ⏪ Reverse & ⏩ Forward • \
+  👮🏿 \"Proxy Server\" framework • 🌐 \"Web Server\" framework • ➵ ➶ ➷ ➠ \"PubSub\" framework • \
+  👷 \"Work\" acceptor & executor framework" \
  com.abhinavsingh.url="https://github.com/abhinavsingh/proxy.py" \
  com.abhinavsingh.vcs-url="https://github.com/abhinavsingh/proxy.py" \
  com.abhinavsingh.docker.cmd="docker run -it --rm -p 8899:8899 abhinavsingh/proxy.py"
+ENV PYTHONUNBUFFERED 1
+ARG PROXYPY_PKG_PATH

-COPY --from=builder /deps /usr/local
+COPY README.md /
+COPY $PROXYPY_PKG_PATH /
+RUN pip install --upgrade pip && \
+  pip install \
+  --no-index \
+  --find-links file:/// \
+  proxy.py && \
+  rm *.whl

-# Install openssl to enable TLS interception within container
+# Install openssl to enable TLS interception & HTTPS proxy options within container
+# NOTE: You can comment out this line if you don't intend to use those features.
 RUN apk update && apk add openssl

 EXPOSE 8899/tcp
--- a/37
+++ b/37
@ -2,8 +2,10 @@ SHELL := /bin/bash

 NS ?= abhinavsingh
 IMAGE_NAME ?= proxy.py
-LATEST_TAG := $(NS)/$(IMAGE_NAME):latest
-IMAGE_TAG := $(NS)/$(IMAGE_NAME):$(shell ./write-scm-version.sh)
+# Override to target specific versions of proxy.py
+PROXYPY_CONTAINER_VERSION := latest
+# Used by container build and run targets
+PROXYPY_CONTAINER_TAG := $(NS)/$(IMAGE_NAME):$(PROXYPY_CONTAINER_VERSION)

 HTTPS_KEY_FILE_PATH := https-key.pem
 HTTPS_CERT_FILE_PATH := https-cert.pem
@ -14,6 +16,10 @@ CA_KEY_FILE_PATH := ca-key.pem
 CA_CERT_FILE_PATH := ca-cert.pem
 CA_SIGNING_KEY_FILE_PATH := ca-signing-key.pem

+# Dummy invalid hardcoded value
+PROXYPY_PKG_PATH := dist/proxy.py.whl
+BUILDX_TARGET_PLATFORM := linux/amd64
+
 OPEN=$(shell which open)
 UNAME := $(shell uname)
 ifeq ($(UNAME), Linux)
@ -24,7 +30,7 @@ endif
 .PHONY: lib-check lib-clean lib-test lib-package lib-coverage lib-lint lib-pytest
 .PHONY: lib-release-test lib-release lib-profile lib-doc
 .PHONY: lib-dep lib-flake8 lib-mypy
-.PHONY: container container-run container-release
+.PHONY: container container-run container-release container-build container-buildx
 .PHONY: devtools dashboard dashboard-clean

 all: lib-test
@ -151,12 +157,25 @@ dashboard:
 dashboard-clean:
 	if [[ -d dashboard/public ]]; then rm -rf dashboard/public; fi

-container:
-	docker build -t $(LATEST_TAG) -t $(IMAGE_TAG) .
+container: lib-package
+	$(MAKE) container-build -e PROXYPY_PKG_PATH=$$(ls dist/*.whl)

-container-release:
-	docker push $(IMAGE_TAG)
-	docker push $(LATEST_TAG)
+# Usage:
+#
+# make container-buildx \
+#	-e PROXYPY_PKG_PATH=$(ls dist/*.whl) \
+#	-e BUILDX_TARGET_PLATFORM=linux/arm64 \
+#	-e PROXYPY_CONTAINER_VERSION=latest
+container-buildx:
+	docker buildx build \
+		--platform $(BUILDX_TARGET_PLATFORM) \
+		-t $(PROXYPY_CONTAINER_TAG) \
+		--build-arg PROXYPY_PKG_PATH=$(PROXYPY_PKG_PATH) .
+
+container-build:
+	docker build \
+		-t $(PROXYPY_CONTAINER_TAG) \
+		--build-arg PROXYPY_PKG_PATH=$(PROXYPY_PKG_PATH) .

 container-run:
-	docker run -it -p 8899:8899 --rm $(LATEST_TAG)
+	docker run -it -p 8899:8899 --rm $(PROXYPY_CONTAINER_TAG)
--- a/README.md
+++ b/README.md
@ -257,12 +257,30 @@ or from GitHub `master` branch

 ## Using Docker

+Stable version container releases are available for following platforms:
+
+- `linux/386`
+- `linux/amd64`
+- `linux/arm/v6`
+- `linux/arm/v7`
+- `linux/arm64/v8`
+- `linux/ppc64le`
+- `linux/s390x`
+
 ### Stable Version from Docker Hub

+Run `proxy.py` latest container:
+
 ```console
 ❯ docker run -it -p 8899:8899 --rm abhinavsingh/proxy.py:latest
 ```

+To run specific target platform container on multi-platform supported servers:
+
+```console
+❯ docker run -it -p 8899:8899 --rm --platform linux/arm64/v8 abhinavsingh/proxy.py:latest
+```
+
 ### Build Development Version Locally

 ```console
@ -394,6 +412,16 @@ To start `proxy.py` from source code follow these instructions:
  ❯ make lib-dep
  ```

+- Generate `proxy/common/_scm_version.py`
+
+  NOTE: *Following step is not necessary for editable installs.*
+
+  This file writes SCM detected version to `proxy/common/_scm_version.py` file.
+
+  ```console
+  ❯ ./write-scm-version.sh
+  ```
+
 - Optionally, run tests

  ```console
@ -2038,7 +2066,7 @@ usage: -m [-h] [--enable-events] [--enable-conn-pool] [--threadless]
          [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG]
          [--cloudflare-dns-mode CLOUDFLARE_DNS_MODE]

-proxy.py v2.3.2.dev183+g808caa1.d20211124
+proxy.py v2.3.2.dev190+ge60d80d.d20211124

 options:
  -h, --help            show this help message and exit