[GHA] Add container integration test & publish containers to GHCR (#818)

* Handle KBI in threadless * Remove line-length disable clause * Move `Set PROXYPY_CONTAINER_VERSION env` within pre-setup * Use step output * Use review suggestion * Break line at less essential places * Possibly fix? * alignment necessary? * :) * Fix integration * Add `v` prefix * Load buildx image within docker for integration tests to work * Run container in detached mode * Enable web server for integration test to work * Push to GHCR after successful tests * Tag & Push to GHCR * Add package permission * Login to GHCR * Login to GHCR * Login to GHCR * Explicitly use ghcr.io registry * Add a ghcr manifest creation step which depends upon docker step * Probably we need all images (tags) locally * arm not amd * Fix syntax * Fix typo * `Unable to find image ghcr.io/abhinavsingh/proxy.py:latest locally` * GHCR doesnt support manifest?
2021-12-18 08:06:24 +05:30 · 2021-12-18 08:06:24 +05:30 · b2612a1bb8
parent c56e0dac28
commit b2612a1bb8
4 changed files with 135 additions and 24 deletions
--- a/.github/buildkitd.toml
+++ b/.github/buildkitd.toml
@ -1,4 +1,4 @@
 [worker.oci]
  max-parallelism = 4
 [registry."docker.io"]
-  mirrors = ["mirror.gcr.io"]
+  mirrors = ["ghcr.io"]
--- a/.github/workflows/test-library.yml
+++ b/.github/workflows/test-library.yml
@ -1,5 +1,4 @@
 ---
-# yamllint disable rule:line-length
 name: lib

 on:   # yamllint disable-line rule:truthy
@ -67,6 +66,7 @@ jobs:
      git-tag: ${{ steps.git-tag.outputs.tag }}
      sdist-artifact-name: ${{ steps.artifact-name.outputs.sdist }}
      wheel-artifact-name: ${{ steps.artifact-name.outputs.wheel }}
+      container-version: v${{ steps.container.outputs.version }}
    steps:
    - name: Switch to using Python 3.9 by default
      uses: actions/setup-python@v2
@ -195,6 +195,16 @@ jobs:
            && github.event.inputs.release-version
            || steps.scm-version.outputs.dist-version
        }}-py3-none-any.whl')
+    - name: Calculate container attributes
+      id: container
+      shell: bash
+      run: >-
+        VER=$(echo '${{
+            steps.request-check.outputs.release-requested == 'true'
+            && github.event.inputs.release-version
+            || steps.scm-version.outputs.dist-version
+        }}' | tr + .);
+        echo "::set-output name=version::$VER"

  build:
    name: 👷 dists ${{ needs.pre-setup.outputs.git-tag }}
@ -547,11 +557,6 @@ jobs:
        # a pull request then we can checkout the head.
        fetch-depth: 2

-    # If this run was triggered by a pull request event, then checkout
-    # the head of the pull request instead of the merge commit.
-    - run: git checkout HEAD^2
-      if: ${{ github.event_name == 'pull_request' }}
-
    # Initializes the CodeQL tools for scanning.
    - name: Initialize CodeQL
      uses: github/codeql-action/init@v1
@ -633,7 +638,7 @@ jobs:

  developer:
    runs-on: ${{ matrix.os }}-latest
-    name: Developer setup ${{ matrix.node }} @ ${{ matrix.os }}
+    name: 🧑‍💻 👩‍💻 👨‍💻 Developer setup ${{ matrix.node }} @ ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu, macOS]
@ -660,9 +665,9 @@ jobs:
        python3 -m proxy --version

  docker:
-    # TODO: To build our docker container, we must wait for check,
-    # so that we can use the same distribution available.
    runs-on: ${{ matrix.os }}-latest
+    permissions:
+      packages: write
    needs:
    - build
    - pre-setup  # transitive, for accessing settings
@ -673,6 +678,8 @@ jobs:
        - Ubuntu
        python:
        - '3.10'
+        # NOTE: Change to target platform must also
+        # be kept consistent with ghcr-manifest tags.
        targetplatform:
        - 'linux/386'
        - 'linux/amd64'
@ -694,8 +701,8 @@ jobs:
    - name: Set up Docker Buildx
      id: buildx
      uses: docker/setup-buildx-action@v1
+      # See https://github.com/docker/buildx/issues/850#issuecomment-996408167
      with:
-        # FIXME: See https://github.com/docker/buildx/issues/850#issuecomment-996408167
        version: v0.7.0
        buildkitd-flags: --debug
        config: .github/buildkitd.toml
@ -709,15 +716,89 @@ jobs:
        docker buildx use proxypybuilder
        docker buildx inspect
        docker buildx ls
-    - name: Set PROXYPY_CONTAINER_VERSION
-      run: |
-        echo "PROXYPY_CONTAINER_VERSION=$(echo '${{ needs.pre-setup.outputs.dist-version }}' | tr + .)" > $GITHUB_ENV
    - name: Build container
-      run: |
-        make container-buildx \
-          -e PROXYPY_PKG_PATH='dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}' \
-          -e BUILDX_TARGET_PLATFORM='${{ matrix.targetplatform }}' \
-          -e PROXYPY_CONTAINER_VERSION='${{ env.PROXYPY_CONTAINER_VERSION }}'
+      run: >-
+        docker images;
+        make container-buildx
+        -e PROXYPY_PKG_PATH='dist/${{
+          needs.pre-setup.outputs.wheel-artifact-name
+        }}'
+        -e BUILDX_TARGET_PLATFORM='${{ matrix.targetplatform }}'
+        -e PROXYPY_CONTAINER_VERSION='${{
+          needs.pre-setup.outputs.container-version
+        }}';
+        docker images
+    - name: Run & Test container
+      run: >-
+        docker run
+        -d
+        -p 8899:8899
+        abhinavsingh/proxy.py:${{
+          needs.pre-setup.outputs.container-version
+        }}
+        --hostname 0.0.0.0
+        --enable-web-server
+        --local-executor && ./tests/integration/test_integration.sh 8899
+    - name: Login to GHCR
+      uses: docker/login-action@v1
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+    - name: Tag & Push to GHCR
+      run: >-
+        REGISTRY_URL="ghcr.io/abhinavsingh/proxy.py";
+        IMAGE_ID=$(docker images | grep ${{
+          needs.pre-setup.outputs.container-version
+        }} | awk '{ print $3 }');
+        PLATFORM=$(echo '${{
+          matrix.targetplatform
+        }}' | tr / .);
+        PLATFORM_CONTAINER_URL=$REGISTRY_URL:${{
+          needs.pre-setup.outputs.container-version
+        }}-$PLATFORM;
+        docker tag $IMAGE_ID $PLATFORM_CONTAINER_URL
+        &&
+        docker push $PLATFORM_CONTAINER_URL
+
+  # ghcr-manifest:
+  #   name: 🐳 → GHCR cross-platform manifest
+  #   runs-on: Ubuntu-latest
+  #   needs:
+  #   - docker
+  #   - pre-setup  # transitive, for accessing settings
+  #   steps:
+  #   - name: Login to GHCR
+  #     uses: docker/login-action@v1
+  #     with:
+  #       registry: ghcr.io
+  #       username: ${{ github.actor }}
+  #       password: ${{ secrets.GITHUB_TOKEN }}
+  #   - name: Create & Push manifest
+  #     run: >-
+  #       REGISTRY_URL="ghcr.io/abhinavsingh/proxy.py";
+  #       docker run --rm $REGISTRY_URL
+  #       REGISTRY_TAG=$REGISTRY_URL:${{
+  #         needs.pre-setup.outputs.container-version
+  #       }};
+  #       docker pull $REGISTRY_TAG-linux.386 &&
+  #       docker pull $REGISTRY_TAG-linux.arm.v6 &&
+  #       docker pull $REGISTRY_TAG-linux.arm.v7 &&
+  #       docker pull $REGISTRY_TAG-linux.arm64 &&
+  #       docker pull $REGISTRY_TAG-linux.arm64.v8 &&
+  #       docker pull $REGISTRY_TAG-linux.ppc64le &&
+  #       docker pull $REGISTRY_TAG-linux.s390x &&
+  #       docker manifest create
+  #       $REGISTRY_TAG
+  #       $REGISTRY_TAG-linux.386
+  #       $REGISTRY_TAG-linux.amd64
+  #       $REGISTRY_TAG-linux.arm.v6
+  #       $REGISTRY_TAG-linux.arm.v7
+  #       $REGISTRY_TAG-linux.arm64.v8
+  #       $REGISTRY_TAG-linux.ppc64le
+  #       $REGISTRY_TAG-linux.s390x
+  #       &&
+  #       docker manifest push $REGISTRY_TAG

  check:  # This job does nothing and is only used for the branch protection
    if: always()
@ -762,13 +843,13 @@ jobs:
        name: python-package-distributions
        path: dist/
    - name: >-
-        Publish 🐍📦 v${{ needs.pre-setup.outputs.git-tag }} to PyPI
+        Publish 🐍📦 ${{ needs.pre-setup.outputs.git-tag }} to PyPI
      uses: pypa/gh-action-pypi-publish@release/v1
      with:
        password: ${{ secrets.PYPI_TOKEN }}

  publish-testpypi:
-    name: Publish 🐍📦 to TestPyPI
+    name: Publish 🐍📦 ${{ needs.pre-setup.outputs.git-tag }} to TestPyPI
    needs:
    - check
    - pre-setup  # transitive, for accessing settings
@ -791,12 +872,41 @@ jobs:
        name: python-package-distributions
        path: dist/
    - name: >-
-        Publish 🐍📦  v${{ needs.pre-setup.outputs.git-tag }} to TestPyPI
+        Publish 🐍📦 ${{ needs.pre-setup.outputs.git-tag }} to TestPyPI
      uses: pypa/gh-action-pypi-publish@release/v1
      with:
        password: ${{ secrets.TESTPYPI_API_TOKEN }}
        repository_url: https://test.pypi.org/legacy/

+  # publish-docker:
+  #   name: Publish 🐳 📦 ${{ needs.pre-setup.outputs.git-tag }} to Docker Hub
+  #   needs:
+  #   - check
+  #   - pre-setup  # transitive, for accessing settings
+  #   if: >-
+  #     fromJSON(needs.pre-setup.outputs.release-requested)
+  #   runs-on: Ubuntu-latest
+
+  #   environment:
+  #     name: release-docker
+  #     url: >-
+  #       https://test.pypi.org/project/proxy.py/${{
+  #         needs.pre-setup.outputs.dist-version
+  #       }}
+
+  #   steps:
+  #   - name: Download all the dists
+  #     uses: actions/download-artifact@v2
+  #     with:
+  #       name: python-package-distributions
+  #       path: dist/
+  #   - name: >-
+  #       Publish 🐳 📦 ${{ needs.pre-setup.outputs.git-tag }} to Docker Hub
+  #     uses: pypa/gh-action-pypi-publish@release/v1
+  #     with:
+  #       password: ${{ secrets.TESTPYPI_API_TOKEN }}
+  #       repository_url: https://test.pypi.org/legacy/
+
  post-release-repo-update:
    name: >-
      Publish post-release Git tag
--- a/1
+++ b/1
@ -190,6 +190,7 @@ container-build:
 #	-e PROXYPY_CONTAINER_VERSION=latest
 container-buildx:
 	docker buildx build \
+		--load \
 		--platform $(BUILDX_TARGET_PLATFORM) \
 		-t $(PROXYPY_CONTAINER_TAG) \
 		--build-arg PROXYPY_PKG_PATH=$(PROXYPY_PKG_PATH) .
--- a/README.md
+++ b/README.md
@ -15,7 +15,7 @@
 [![pypi version](https://img.shields.io/pypi/v/proxy.py)](https://pypi.org/project/proxy.py/)
 [![Python 3.x](https://img.shields.io/static/v1?label=Python&message=3.6%20%7C%203.7%20%7C%203.8%20%7C%203.9%20%7C%203.10&color=blue)](https://www.python.org/)
 [![Checked with mypy](https://img.shields.io/static/v1?label=MyPy&message=checked&color=blue)](http://mypy-lang.org/)
-[![lib](https://github.com/abhinavsingh/proxy.py/actions/workflows/test-library.yml/badge.svg)](https://github.com/abhinavsingh/proxy.py/actions/workflows/test-library.yml)
+[![lib](https://github.com/abhinavsingh/proxy.py/actions/workflows/test-library.yml/badge.svg?branch=develop&event=push)](https://github.com/abhinavsingh/proxy.py/actions/workflows/test-library.yml)
 [![codecov](https://codecov.io/gh/abhinavsingh/proxy.py/branch/develop/graph/badge.svg?token=Zh9J7b4la2)](https://codecov.io/gh/abhinavsingh/proxy.py)

 [![Contributions Welcome](https://img.shields.io/static/v1?label=Contributions&message=Welcome%20%F0%9F%91%8D&color=darkgreen)](https://github.com/abhinavsingh/proxy.py/issues)
@ -2073,7 +2073,7 @@ usage: -m [-h] [--enable-events] [--enable-conn-pool] [--threadless]
          [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG]
          [--cloudflare-dns-mode CLOUDFLARE_DNS_MODE]

-proxy.py v2.4.0rc2.dev21+g20b3eb1.d20211203
+proxy.py v2.4.0rc3.dev33+gc341594.d20211214

 options:
  -h, --help            show this help message and exit