[GHA] Add container integration test & publish containers to GHCR (#818)

* Handle KBI in threadless

* Remove line-length disable clause

* Move `Set PROXYPY_CONTAINER_VERSION env` within pre-setup

* Use step output

* Use review suggestion

* Break line at less essential places

* Possibly fix?

* alignment necessary?

* :)

* Fix integration

* Add `v` prefix

* Load buildx image within docker for integration tests to work

* Run container in detached mode

* Enable web server for integration test to work

* Push to GHCR after successful tests

* Tag & Push to GHCR

* Add package permission

* Login to GHCR

* Login to GHCR

* Login to GHCR

* Explicitly use ghcr.io registry

* Add a ghcr manifest creation step which depends upon docker step

* Probably we need all images (tags) locally

* arm not amd

* Fix syntax

* Fix typo

* `Unable to find image ghcr.io/abhinavsingh/proxy.py:latest locally`

* GHCR doesnt support manifest?

.github/buildkitd.toml

@@ -1,4 +1,4 @@
 [worker.oci]
   max-parallelism = 4
 [registry."docker.io"]
-  mirrors = ["mirror.gcr.io"]
+  mirrors = ["ghcr.io"]

.github/workflows/test-library.yml

@@ -1,5 +1,4 @@
 ---
-# yamllint disable rule:line-length
 name: lib
 
 on:   # yamllint disable-line rule:truthy
@@ -67,6 +66,7 @@ jobs:
       git-tag: ${{ steps.git-tag.outputs.tag }}
       sdist-artifact-name: ${{ steps.artifact-name.outputs.sdist }}
       wheel-artifact-name: ${{ steps.artifact-name.outputs.wheel }}
+      container-version: v${{ steps.container.outputs.version }}
     steps:
     - name: Switch to using Python 3.9 by default
       uses: actions/setup-python@v2
@@ -195,6 +195,16 @@ jobs:
             && github.event.inputs.release-version
             || steps.scm-version.outputs.dist-version
         }}-py3-none-any.whl')
+    - name: Calculate container attributes
+      id: container
+      shell: bash
+      run: >-
+        VER=$(echo '${{
+            steps.request-check.outputs.release-requested == 'true'
+            && github.event.inputs.release-version
+            || steps.scm-version.outputs.dist-version
+        }}' | tr + .);
+        echo "::set-output name=version::$VER"
 
   build:
     name: 👷 dists ${{ needs.pre-setup.outputs.git-tag }}
@@ -547,11 +557,6 @@ jobs:
         # a pull request then we can checkout the head.
         fetch-depth: 2
 
-    # If this run was triggered by a pull request event, then checkout
-    # the head of the pull request instead of the merge commit.
-    - run: git checkout HEAD^2
-      if: ${{ github.event_name == 'pull_request' }}
-
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v1
@@ -633,7 +638,7 @@ jobs:
 
   developer:
     runs-on: ${{ matrix.os }}-latest
-    name: Developer setup ${{ matrix.node }} @ ${{ matrix.os }}
+    name: 🧑‍💻 👩‍💻 👨‍💻 Developer setup ${{ matrix.node }} @ ${{ matrix.os }}
     strategy:
       matrix:
         os: [ubuntu, macOS]
@@ -660,9 +665,9 @@ jobs:
         python3 -m proxy --version
 
   docker:
-    # TODO: To build our docker container, we must wait for check,
-    # so that we can use the same distribution available.
     runs-on: ${{ matrix.os }}-latest
+    permissions:
+      packages: write
     needs:
     - build
     - pre-setup  # transitive, for accessing settings
@@ -673,6 +678,8 @@ jobs:
         - Ubuntu
         python:
         - '3.10'
+        # NOTE: Change to target platform must also
+        # be kept consistent with ghcr-manifest tags.
         targetplatform:
         - 'linux/386'
         - 'linux/amd64'
@@ -694,8 +701,8 @@ jobs:
     - name: Set up Docker Buildx
       id: buildx
       uses: docker/setup-buildx-action@v1
+      # See https://github.com/docker/buildx/issues/850#issuecomment-996408167
       with:
-        # FIXME: See https://github.com/docker/buildx/issues/850#issuecomment-996408167
         version: v0.7.0
         buildkitd-flags: --debug
         config: .github/buildkitd.toml
@@ -709,15 +716,89 @@ jobs:
         docker buildx use proxypybuilder
         docker buildx inspect
         docker buildx ls
-    - name: Set PROXYPY_CONTAINER_VERSION
-      run: |
-        echo "PROXYPY_CONTAINER_VERSION=$(echo '${{ needs.pre-setup.outputs.dist-version }}' | tr + .)" > $GITHUB_ENV
     - name: Build container
-      run: |
+      run: >-
-        make container-buildx \
+        docker images;
-          -e PROXYPY_PKG_PATH='dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}' \
+        make container-buildx
-          -e BUILDX_TARGET_PLATFORM='${{ matrix.targetplatform }}' \
+        -e PROXYPY_PKG_PATH='dist/${{
-          -e PROXYPY_CONTAINER_VERSION='${{ env.PROXYPY_CONTAINER_VERSION }}'
+          needs.pre-setup.outputs.wheel-artifact-name
+        }}'
+        -e BUILDX_TARGET_PLATFORM='${{ matrix.targetplatform }}'
+        -e PROXYPY_CONTAINER_VERSION='${{
+          needs.pre-setup.outputs.container-version
+        }}';
+        docker images
+    - name: Run & Test container
+      run: >-
+        docker run
+        -d
+        -p 8899:8899
+        abhinavsingh/proxy.py:${{
+          needs.pre-setup.outputs.container-version
+        }}
+        --hostname 0.0.0.0
+        --enable-web-server
+        --local-executor && ./tests/integration/test_integration.sh 8899
+    - name: Login to GHCR
+      uses: docker/login-action@v1
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+    - name: Tag & Push to GHCR
+      run: >-
+        REGISTRY_URL="ghcr.io/abhinavsingh/proxy.py";
+        IMAGE_ID=$(docker images | grep ${{
+          needs.pre-setup.outputs.container-version
+        }} | awk '{ print $3 }');
+        PLATFORM=$(echo '${{
+          matrix.targetplatform
+        }}' | tr / .);
+        PLATFORM_CONTAINER_URL=$REGISTRY_URL:${{
+          needs.pre-setup.outputs.container-version
+        }}-$PLATFORM;
+        docker tag $IMAGE_ID $PLATFORM_CONTAINER_URL
+        &&
+        docker push $PLATFORM_CONTAINER_URL
+
+  # ghcr-manifest:
+  #   name: 🐳 → GHCR cross-platform manifest
+  #   runs-on: Ubuntu-latest
+  #   needs:
+  #   - docker
+  #   - pre-setup  # transitive, for accessing settings
+  #   steps:
+  #   - name: Login to GHCR
+  #     uses: docker/login-action@v1
+  #     with:
+  #       registry: ghcr.io
+  #       username: ${{ github.actor }}
+  #       password: ${{ secrets.GITHUB_TOKEN }}
+  #   - name: Create & Push manifest
+  #     run: >-
+  #       REGISTRY_URL="ghcr.io/abhinavsingh/proxy.py";
+  #       docker run --rm $REGISTRY_URL
+  #       REGISTRY_TAG=$REGISTRY_URL:${{
+  #         needs.pre-setup.outputs.container-version
+  #       }};
+  #       docker pull $REGISTRY_TAG-linux.386 &&
+  #       docker pull $REGISTRY_TAG-linux.arm.v6 &&
+  #       docker pull $REGISTRY_TAG-linux.arm.v7 &&
+  #       docker pull $REGISTRY_TAG-linux.arm64 &&
+  #       docker pull $REGISTRY_TAG-linux.arm64.v8 &&
+  #       docker pull $REGISTRY_TAG-linux.ppc64le &&
+  #       docker pull $REGISTRY_TAG-linux.s390x &&
+  #       docker manifest create
+  #       $REGISTRY_TAG
+  #       $REGISTRY_TAG-linux.386
+  #       $REGISTRY_TAG-linux.amd64
+  #       $REGISTRY_TAG-linux.arm.v6
+  #       $REGISTRY_TAG-linux.arm.v7
+  #       $REGISTRY_TAG-linux.arm64.v8
+  #       $REGISTRY_TAG-linux.ppc64le
+  #       $REGISTRY_TAG-linux.s390x
+  #       &&
+  #       docker manifest push $REGISTRY_TAG
 
   check:  # This job does nothing and is only used for the branch protection
     if: always()
@@ -762,13 +843,13 @@ jobs:
         name: python-package-distributions
         path: dist/
     - name: >-
-        Publish 🐍📦 v${{ needs.pre-setup.outputs.git-tag }} to PyPI
+        Publish 🐍📦 ${{ needs.pre-setup.outputs.git-tag }} to PyPI
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
         password: ${{ secrets.PYPI_TOKEN }}
 
   publish-testpypi:
-    name: Publish 🐍📦 to TestPyPI
+    name: Publish 🐍📦 ${{ needs.pre-setup.outputs.git-tag }} to TestPyPI
     needs:
     - check
     - pre-setup  # transitive, for accessing settings
@@ -791,12 +872,41 @@ jobs:
         name: python-package-distributions
         path: dist/
     - name: >-
-        Publish 🐍📦  v${{ needs.pre-setup.outputs.git-tag }} to TestPyPI
+        Publish 🐍📦 ${{ needs.pre-setup.outputs.git-tag }} to TestPyPI
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
         password: ${{ secrets.TESTPYPI_API_TOKEN }}
         repository_url: https://test.pypi.org/legacy/
 
+  # publish-docker:
+  #   name: Publish 🐳 📦 ${{ needs.pre-setup.outputs.git-tag }} to Docker Hub
+  #   needs:
+  #   - check
+  #   - pre-setup  # transitive, for accessing settings
+  #   if: >-
+  #     fromJSON(needs.pre-setup.outputs.release-requested)
+  #   runs-on: Ubuntu-latest
+
+  #   environment:
+  #     name: release-docker
+  #     url: >-
+  #       https://test.pypi.org/project/proxy.py/${{
+  #         needs.pre-setup.outputs.dist-version
+  #       }}
+
+  #   steps:
+  #   - name: Download all the dists
+  #     uses: actions/download-artifact@v2
+  #     with:
+  #       name: python-package-distributions
+  #       path: dist/
+  #   - name: >-
+  #       Publish 🐳 📦 ${{ needs.pre-setup.outputs.git-tag }} to Docker Hub
+  #     uses: pypa/gh-action-pypi-publish@release/v1
+  #     with:
+  #       password: ${{ secrets.TESTPYPI_API_TOKEN }}
+  #       repository_url: https://test.pypi.org/legacy/
+
   post-release-repo-update:
     name: >-
       Publish post-release Git tag

Makefile

@@ -190,6 +190,7 @@ container-build:
 #	-e PROXYPY_CONTAINER_VERSION=latest
 container-buildx:
 	docker buildx build \
+		--load \
 		--platform $(BUILDX_TARGET_PLATFORM) \
 		-t $(PROXYPY_CONTAINER_TAG) \
 		--build-arg PROXYPY_PKG_PATH=$(PROXYPY_PKG_PATH) .

README.md

@@ -15,7 +15,7 @@
 [![pypi version](https://img.shields.io/pypi/v/proxy.py)](https://pypi.org/project/proxy.py/)
 [![Python 3.x](https://img.shields.io/static/v1?label=Python&message=3.6%20%7C%203.7%20%7C%203.8%20%7C%203.9%20%7C%203.10&color=blue)](https://www.python.org/)
 [![Checked with mypy](https://img.shields.io/static/v1?label=MyPy&message=checked&color=blue)](http://mypy-lang.org/)
-[![lib](https://github.com/abhinavsingh/proxy.py/actions/workflows/test-library.yml/badge.svg)](https://github.com/abhinavsingh/proxy.py/actions/workflows/test-library.yml)
+[![lib](https://github.com/abhinavsingh/proxy.py/actions/workflows/test-library.yml/badge.svg?branch=develop&event=push)](https://github.com/abhinavsingh/proxy.py/actions/workflows/test-library.yml)
 [![codecov](https://codecov.io/gh/abhinavsingh/proxy.py/branch/develop/graph/badge.svg?token=Zh9J7b4la2)](https://codecov.io/gh/abhinavsingh/proxy.py)
 
 [![Contributions Welcome](https://img.shields.io/static/v1?label=Contributions&message=Welcome%20%F0%9F%91%8D&color=darkgreen)](https://github.com/abhinavsingh/proxy.py/issues)
@@ -2073,7 +2073,7 @@ usage: -m [-h] [--enable-events] [--enable-conn-pool] [--threadless]
           [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG]
           [--cloudflare-dns-mode CLOUDFLARE_DNS_MODE]
 
-proxy.py v2.4.0rc2.dev21+g20b3eb1.d20211203
+proxy.py v2.4.0rc3.dev33+gc341594.d20211214
 
 options:
   -h, --help            show this help message and exit