Rooba
/
kombu
mirror of https://github.com/celery/kombu.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactored CI (#2122) 

* Refactored CI

* Added test-integration.txt for integration tests

* Removed .tox from "make clean"

* Fixed pypy

* Cleanup

* Added "--reruns 2" to integration tests
This commit is contained in:
Tomer Nosrati 2024-09-25 21:10:28 +03:00 committed by GitHub
parent 424de01b88
commit 3448887f78
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 215 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
.github/workflows/ci.yaml vendored
View File

@ -1,86 +0,0 @@
name: CI
on: [pull_request, push]
jobs:
#################### Unittests ####################
unittest:
runs-on: blacksmith-4vcpu-ubuntu-2204
strategy:
matrix:
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
steps:
- name: Install system packages
run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev
- name: Check out code from GitHub
uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
id: python
uses: useblacksmith/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: pip install --upgrade pip wheel tox tox-docker
- name: Run unittest
run: tox -v -e ${{ matrix.python-version }}-linux-unit -- -v
#################### Integration tests ####################
integration:
needs: [unittest]
runs-on: blacksmith-4vcpu-ubuntu-2204
continue-on-error: ${{ matrix.experimental }}
strategy:
fail-fast: false
matrix:
python-version: [3.8,3.9,"3.10","3.11","3.12"]
experimental: [false]
include:
- python-version: pypy3.9
experimental: true
steps:
- name: Install system packages
run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev
- name: Check out code from GitHub
uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
id: python
uses: useblacksmith/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: pip install --upgrade pip wheel tox tox-docker
# Tox fails if a Python versions contains a hyphen, this changes "pypy-3.9" to "pypy3.9".
- name: Run AMQP integration tests
run: tox -v -e ${{ matrix.python-version }}-linux-integration-py-amqp -- -v
- name: Run redis integration tests
run: tox -v -e ${{ matrix.python-version }}-linux-integration-py-redis -- -v
- name: Run MongoDB integration tests
run: tox -v -e ${{ matrix.python-version }}-linux-integration-py-mongodb -- -v
- name: Run kafka integration tests
if: ${{ matrix.python-version != 'pypy3.9'}}
run: tox -v -e ${{ matrix.python-version }}-linux-integration-py-kafka -- -v
#################### Linters and checkers ####################
lint:
needs: [unittest, integration]
runs-on: blacksmith-4vcpu-ubuntu-2204
strategy:
matrix:
python-version: ["3.10"]
steps:
- name: Install system packages
run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev
- name: Check out code from GitHub
uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
id: python
uses: useblacksmith/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: pip install --upgrade pip wheel tox tox-docker
- name: Run flake8
run: tox -v -e flake8 -- -v
- name: Run pydocstyle
run: tox -v -e pydocstyle -- -v
- name: Run apicheck
run: tox -v -e apicheck -- -v
- name: Run mypy
run: tox -v -e mypy -- -v

40
.github/workflows/linter.yml vendored Normal file
View File

@ -0,0 +1,40 @@
name: Linter
on: [push, pull_request, workflow_dispatch]
jobs:
pre-commit:
runs-on: blacksmith-4vcpu-ubuntu-2204
steps:
- name: Checkout branch
uses: actions/checkout@v4
- name: Run pre-commit
uses: pre-commit/action@v3.0.1
lint:
runs-on: blacksmith-4vcpu-ubuntu-2204
strategy:
matrix:
python-version: ["3.12"]
steps:
- name: Install system packages
run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev
- name: Check out code from GitHub
uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
id: python
uses: useblacksmith/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: pip install --upgrade pip wheel tox tox-docker
- name: Run flake8
run: tox -v -e flake8 -- -v
- name: Run pydocstyle
run: tox -v -e pydocstyle -- -v
- name: Run apicheck
run: tox -v -e apicheck -- -v
- name: Run mypy
run: tox -v -e mypy -- -v

120
.github/workflows/python-package.yml vendored Normal file
View File

@ -0,0 +1,120 @@
# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
name: Kombu
on:
push:
branches: [ 'main']
paths:
- '**.py'
- '**.txt'
- '.github/workflows/python-package.yml'
- '**.toml'
- "tox.ini"
pull_request:
branches: [ 'main' ]
paths:
- '**.py'
- '**.txt'
- '**.toml'
- '.github/workflows/python-package.yml'
- "tox.ini"
workflow_dispatch:
permissions:
contents: read # to fetch code (actions/checkout)
jobs:
Unit:
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
os: ["blacksmith-4vcpu-ubuntu-2204"]
steps:
- name: Install apt packages
if: startsWith(matrix.os, 'blacksmith-4vcpu-ubuntu')
run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: useblacksmith/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
allow-prereleases: true
cache: 'pip'
cache-dependency-path: '**/setup.py'
- name: Install tox
run: python -m pip install --upgrade pip wheel tox tox-docker
- name: >
Run tox for
"${{ matrix.python-version }}-unit"
timeout-minutes: 5
run: >
tox --verbose --verbose -e
"${{ matrix.python-version }}-unit" -vv
- uses: codecov/codecov-action@v4
with:
flags: unittests # optional
fail_ci_if_error: false # optional (default = false)
token: ${{ secrets.CODECOV_TOKEN }}
verbose: true # optional (default = false)
Integration:
needs:
- Unit
if: needs.Unit.result == 'success'
runs-on: blacksmith-4vcpu-ubuntu-2204
strategy:
fail-fast: false
matrix:
python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
toxenv: [
'py-amqp',
'py-redis',
'py-mongodb',
'py-kafka'
]
experimental: [false]
include:
- python-version: pypy3.10
toxenv: 'py-amqp'
experimental: true
- python-version: pypy3.10
toxenv: 'py-redis'
experimental: true
- python-version: pypy3.10
toxenv: 'py-mongodb'
experimental: true
# - python-version: pypy3.10
# toxenv: 'py-kafka'
# experimental: true
steps:
- name: Install apt packages
run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: useblacksmith/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
allow-prereleases: true
cache: 'pip'
cache-dependency-path: '**/setup.py'
- name: Install tox
run: python -m pip install --upgrade pip wheel tox tox-docker
- name: >
Run tox for
"${{ matrix.python-version }}-linux-integration-${{ matrix.toxenv }}"
timeout-minutes: 30
run: >
tox --verbose --verbose -e
"${{ matrix.python-version }}-linux-integration-${{ matrix.toxenv }}" -vv

3
.gitignore vendored
View File

@ -36,3 +36,6 @@ venv/
env
.eggs
.python-version
.coverage.*
control/
.env

2
Makefile
View File

@ -128,7 +128,7 @@ clean-pyc:
removepyc: clean-pyc
clean-build:
rm -rf build/ dist/ .eggs/ *.egg-info/ .tox/ .coverage cover/
rm -rf build/ dist/ .eggs/ *.egg-info/ .coverage cover/
clean-git:
$(GIT) clean -xdn

2
requirements/test-integration.txt Normal file
View File

@ -0,0 +1,2 @@
pytest-xdist==3.6.1
pytest-rerunfailures>=11.1.2

2
requirements/test.txt
View File

@ -3,3 +3,5 @@ Pyro4==4.82
pytest-freezer==0.4.8
pytest-sugar==1.0.0
pytest==8.3.3
pre-commit>=3.5.0,<3.6.0; python_version < '3.9'
pre-commit>=3.8.0; python_version >= '3.9'

65
tox.ini
View File

@ -1,13 +1,14 @@
[tox]
envlist =
{pypy3.9,3.8,3.9,3.10,3.11,3.12}-unit
{pypy3.9,3.8,3.9,3.10,3.11,3.12}-linux-integration-py-amqp
{pypy3.9,3.8,3.9,3.10,3.11}-linux-integration-redis
{pypy3.9,3.8,3.9,3.10,3.11}-linux-integration-mongodb
{3.8,3.9,3.10,3.11,3.12}-linux-integration-kafka
{pypy3.10,3.8,3.9,3.10,3.11,3.12}-unit
{pypy3.10,3.8,3.9,3.10,3.11,3.12}-linux-integration-py-amqp
{pypy3.10,3.8,3.9,3.10,3.11,3.12}-linux-integration-redis
{pypy3.10,3.8,3.9,3.10,3.11,3.12}-linux-integration-mongodb
{pypy3.10,3.8,3.9,3.10,3.11,3.12}-linux-integration-kafka
flake8
apicheck
pydocstyle
mypy
requires =
tox-docker<=4.1
@ -17,9 +18,10 @@ requires =
python =
3.8: py38
3.9: py39
3.10: py310, mypy
3.10: py310
3.11: py311
3.12: py312
pypy3: pypy3
[testenv]
sitepackages = False
@ -28,28 +30,29 @@ passenv =
DISTUTILS_USE_SDK
deps=
-r{toxinidir}/requirements/dev.txt
apicheck,pypy3.9,3.8,3.9,3.10,3.11,3.12: -r{toxinidir}/requirements/default.txt
apicheck,pypy3.9,3.8,3.9,3.10,3.11,3.12: -r{toxinidir}/requirements/test.txt
apicheck,pypy3.9,3.8,3.9,3.10,3.11,3.12: -r{toxinidir}/requirements/test-ci.txt
apicheck,pypy3.10,3.8,3.9,3.10,3.11,3.12: -r{toxinidir}/requirements/default.txt
apicheck,pypy3.10,3.8,3.9,3.10,3.11,3.12: -r{toxinidir}/requirements/test.txt
apicheck,pypy3.10,3.8,3.9,3.10,3.11,3.12: -r{toxinidir}/requirements/test-ci.txt
apicheck,3.8-linux,3.9-linux,3.10-linux,3.11-linux,3.12-linux: -r{toxinidir}/requirements/extras/confluentkafka.txt
apicheck,linkcheck: -r{toxinidir}/requirements/docs.txt
flake8,pydocstyle,mypy: -r{toxinidir}/requirements/pkgutils.txt
integration: -r{toxinidir}/requirements/test-integration.txt
commands =
unit: python -bb -m pytest -rxs -xv --cov=kombu --cov-report=xml --no-cov-on-fail {posargs}
integration-py-amqp: pytest -xv -E py-amqp t/integration {posargs:-n2}
integration-redis: pytest -xv -E redis t/integration {posargs:-n2}
integration-mongodb: pytest -xv -E mongodb t/integration {posargs:-n2}
integration-kafka: pytest -xv -E kafka t/integration {posargs:-n2}
unit: python -bb -m pytest -rxs -xv --cov=kombu --cov-report=xml --no-cov-on-fail --cov-report term {posargs}
integration-py-amqp: pytest -xv -E py-amqp t/integration -n auto --reruns 2 --reruns-delay 1 {posargs}
integration-redis: pytest -xv -E redis t/integration -n auto --reruns 2 --reruns-delay 1 {posargs}
integration-mongodb: pytest -xv -E mongodb t/integration -n auto --reruns 2 --reruns-delay 1 {posargs}
integration-kafka: pytest -xv -E kafka t/integration -n auto --reruns 2 --reruns-delay 1 {posargs}
basepython =
pypy3.9: pypy3.9
pypy3.8: pypy3.8
pypy3: pypy3
3.8: python3.8
3.9: python3.9
3.10,apicheck,pydocstyle,flake8,linkcheck,cov,mypy: python3.10
3.10: python3.10
3.11: python3.11
3.12: python3.12
apicheck,pydocstyle,flake8,linkcheck,cov,mypy: python3.12
install_command = python -m pip --disable-pip-version-check install {opts} {packages}
@ -135,4 +138,30 @@ commands =
pydocstyle {toxinidir}/kombu
[testenv:mypy]
commands = python -m mypy --config-file setup.cfg
commands = python -m mypy --config-file setup.cfg
[testenv:lint]
allowlist_externals = pre-commit
commands =
pre-commit {posargs:run --all-files --show-diff-on-failure}
[testenv:clean]
deps = cleanpy
allowlist_externals = make
commands =
python -m cleanpy .
make clean
[testenv:parallel]
description = Run all integration tests in parallel
allowlist_externals = tox
setenv =
PYTHONUNBUFFERED = 1
PYTHONDONTWRITEBYTECODE = 1
commands =
tox -e \
3.12-linux-integration-py-amqp,\
3.12-linux-integration-redis,\
3.12-linux-integration-mongodb,\
3.12-linux-integration-kafka \
-p -o -- --exitfirst -n auto --dist=loadscope {posargs}