Rooba
/
lightning
mirror of https://github.com/Lightning-AI/lightning.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

tests: drop CircleCI (#2412) 

* drop CircleCI

* add PT testing

* fix

* cpu

* conda

* conda

* req

* base

* conda

* conda

* conda

* conda

* conda

* conda

* conda

* name

* req

* info

* tests

* pt 1.6

* drop 1.6

* info
This commit is contained in:
Jirka Borovec 2020-06-30 16:56:05 +02:00 committed by GitHub
parent a42a0e16dd
commit d4a02e3bd8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 127 additions and 214 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

187
.circleci/config.yml
View File

@ -1,187 +0,0 @@
# Python CircleCI 2.0 configuration file
#
# Check https://circleci.com/docs/2.0/language-python/ for more details
#
version: 2.0
references:
install_deps: &install_deps
run:
name: Install Dependences
command: |
sudo apt-get update && sudo apt-get install -y cmake
pip install "$TORCH_VERSION"
pip install -r requirements/base.txt -q
sudo pip install pytest -q
pip install -r requirements/devel.txt -q
tests: &tests
run:
name: Testing
command: |
python --version ; pip --version ; pip list
py.test pytorch_lightning tests -v --junitxml=test-reports/pytest_junit.xml
no_output_timeout: 15m
examples: &examples
run:
name: PL Examples
command: |
pip install -r ./requirements/examples.txt --user
python --version ; pip --version ; pip list
py.test pl_examples -v --junitxml=test-reports/pytest_junit.xml
no_output_timeout: 20m
install_pkg: &install_pkg
run:
name: Install package
command: |
virtualenv vEnv ; source vEnv/bin/activate
pip install --editable . ; cd .. & python -c "import pytorch_lightning ; print(pytorch_lightning.__version__)"
deactivate ; rm -rf vEnv
create_pkg: &create_pkg
run:
name: Create package
command: |
sudo pip install twine==1.13.0
python setup.py sdist
twine check dist/*
python setup.py clean
format: &format
run:
name: Formatting
command: |
python --version ; pip --version
sudo pip install flake8 -q
pip list
flake8 .
make_docs: &make_docs
run:
name: Make Documentation
command: |
# First run the same pipeline as Read-The-Docs
# apt-get update && apt-get install -y cmake
# using: https://hub.docker.com/r/readthedocs/build
# we need to use py3.7 ot higher becase of an issue with metaclass inheritence
pyenv global 3.7.3
python --version
pip install -r requirements/docs.txt
cd docs; make clean; make html --debug --jobs 2 SPHINXOPTS="-W"
test_docs: &test_docs
run:
name: Testing Documentation
command: |
# Second run examples in docs
bash tests/install_AMP.sh
sudo apt-get update && sudo apt-get install -y cmake
sudo pip install -r requirements/docs.txt
cd docs; make doctest; make coverage
jobs:
Build-Docs:
docker:
- image: readthedocs/build:latest
steps:
- checkout
- *make_docs
- store_artifacts:
# allows us to preview the generated html pages
path: docs/build/html/
destination: html
Formatting:
docker:
- image: circleci/python:3.7
environment:
- TORCH_VERSION: "torch"
steps:
- checkout
- *format
PyTorch:
docker:
- image: circleci/python:3.6
environment:
- TORCH_VERSION: "torch"
steps: &steps
- checkout
#- restore_cache:
# keys:
# # when lock file changes, use increasingly general patterns to restore cache
# - pip-packages--{{ .Environment.CIRCLE_JOB }}
# - pip-packages--
- *install_deps
#- save_cache:
# key: pip-packages--{{ .Environment.CIRCLE_JOB }}
# paths:
# # this path depends on where pipenv creates a virtualenv
# - "~/.cache/pip"
# - "/usr/local/lib/python3.6/site-packages"
# - "/usr/local/lib/site-python"
- *tests
- store_test_results:
path: test-reports
- store_artifacts:
path: test-reports
PyTorch-v1_3:
docker:
- image: circleci/python:3.6
environment:
- TORCH_VERSION: "torch>=1.3, <1.4"
steps: *steps
PyTorch-v1_4:
docker:
- image: circleci/python:3.6
environment:
- TORCH_VERSION: "torch>=1.4, <1.5"
steps: *steps
PyTorch-v1_5:
docker:
- image: circleci/python:3.6
environment:
- TORCH_VERSION: "torch>=1.5, <1.6"
steps: *steps
Examples:
docker:
- image: circleci/python:3.7
environment:
- TORCH_VERSION: "torch"
- SPHINX_MOCK_REQUIREMENTS: 0
steps:
- checkout
- *install_deps
- *test_docs
- *examples
Install-pkg:
docker:
- image: circleci/python:3.7
steps:
- checkout
- *create_pkg
- *install_pkg
#orbs:
# python: circleci/python@0.2.1
workflows:
version: 2
build:
jobs:
#- Formatting
#- Build-Docs
- PyTorch-v1_3
- PyTorch-v1_4
- PyTorch-v1_5
#- Install-pkg
- Examples

24
.github/workflows/ci-testing.yml vendored
View File

@ -19,7 +19,6 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
# max-parallel: 6
matrix:
os: [ubuntu-18.04, windows-2019, macOS-10.15]
python-version: [3.6, 3.7, 3.8]
@ -33,7 +32,7 @@ jobs:
os: macOS-10.15
# Timeout: https://stackoverflow.com/a/59076067/4521646
timeout-minutes: 50
timeout-minutes: 25
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
@ -51,26 +50,26 @@ jobs:
- name: Setup Windows
if: runner.os == 'windows'
run: |
python -c "lines = [line for line in open('requirements/extra.txt').readlines() if not line.startswith('horovod')] ; open('requirements/extra.txt', 'w').writelines(lines)"
python -c "fname = 'requirements/extra.txt' ; lines = [line for line in open(fname).readlines() if not line.startswith('horovod')] ; open(fname, 'w').writelines(lines)"
# TODO: remove after https://github.com/pytorch/pytorch/issues/32186 is resolved
- name: Setup Windows on Latest
if: runner.os == 'windows' && matrix.requires == 'latest'
run: |
python -c "req = open('requirements/base.txt').read().replace('torch>=1.3', 'torch<1.5') ; open('requirements/base.txt', 'w').write(req)"
python -c "fname = 'requirements/base.txt' ; req = open(fname).read().replace('torch>=1.3', 'torch<1.5') ; open(fname, 'w').write(req)"
# versions <= 1.3 may have issues on mac with some BLAS ops due to missing mkl (https://github.com/pytorch/pytorch/issues/18996)
- name: Setup MacOS Minimal
if: runner.os == 'macOS' && matrix.requires == 'minimal'
run : |
python -c "req = open('requirements/base.txt').read().replace('torch>=1.3', 'torch>=1.4') ; open('requirements/base.txt', 'w').write(req)"
python -c "fname = 'requirements/base.txt' ; req = open(fname).read().replace('torch>=1.3', 'torch>=1.4') ; open(fname, 'w').write(req)"
- name: Set min. dependencies
if: matrix.requires == 'minimal'
run: |
python -c "req = open('requirements/base.txt').read().replace('>=', '==') ; open('requirements/base.txt', 'w').write(req)"
python -c "req = open('requirements/extra.txt').read().replace('>=', '==') ; open('requirements/extra.txt', 'w').write(req)"
python -c "req = open('requirements/devel.txt').read().replace('>=', '==') ; open('requirements/devel.txt', 'w').write(req)"
python -c "fname = 'requirements/base.txt' ; req = open(fname).read().replace('>=', '==') ; open(fname, 'w').write(req)"
python -c "fname = 'requirements/extra.txt' ; req = open(fname).read().replace('>=', '==') ; open(fname, 'w').write(req)"
python -c "fname = 'requirements/devel.txt' ; req = open(fname).read().replace('>=', '==') ; open(fname, 'w').write(req)"
# Note: This uses an internal pip API and may not always work
# https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow
@ -90,9 +89,8 @@ jobs:
- name: Install dependencies
run: |
# python -m pip install --upgrade --user pip
pip install -r requirements/base.txt -U -f https://download.pytorch.org/whl/torch_stable.html -q
HOROVOD_BUILD_ARCH_FLAGS="-mfma" pip install -r ./requirements/devel.txt -q
# pip install tox coverage
pip install -r requirements/base.txt -U -f https://download.pytorch.org/whl/torch_stable.html --quiet
HOROVOD_BUILD_ARCH_FLAGS="-mfma" pip install -r ./requirements/devel.txt --quiet
python --version
pip --version
pip list
@ -112,9 +110,9 @@ jobs:
- name: Cache datasets
uses: actions/cache@v1
with:
path: tests/Datasets # This path is specific to Ubuntu
path: Datasets # This path is specific to Ubuntu
# Look to see if there is a cache hit for the corresponding requirements file
key: mnist-dataset
key: pl-dataset
- name: Tests
# env:

4
.github/workflows/code-formatting.yml vendored
View File

@ -52,8 +52,8 @@ jobs:
- name: Install dependencies
run: |
# python -m pip install --upgrade --user pip
pip install -r requirements/base.txt -U -f https://download.pytorch.org/whl/torch_stable.html -q
HOROVOD_BUILD_ARCH_FLAGS="-mfma" pip install -r ./requirements/devel.txt -q
pip install -r requirements/base.txt -U -f https://download.pytorch.org/whl/torch_stable.html --quiet
HOROVOD_BUILD_ARCH_FLAGS="-mfma" pip install -r ./requirements/devel.txt --quiet
# pip install tox coverage
python --version ; pip --version ; pip list
shell: bash

4
.github/workflows/docs-checks.yml vendored
View File

@ -39,7 +39,7 @@ jobs:
- name: Install dependencies
run: |
# python -m pip install --upgrade --user pip
pip install -r requirements/base.txt -U -f https://download.pytorch.org/whl/torch_stable.html -q
pip install -r requirements/base.txt -U -f https://download.pytorch.org/whl/torch_stable.html --quiet
pip install -r requirements/extra.txt
pip install -r requirements/docs.txt
python --version ; pip --version ; pip list
@ -76,7 +76,7 @@ jobs:
- name: Install dependencies
run: |
pip install -r requirements/base.txt -U -f https://download.pytorch.org/whl/torch_stable.html -q
pip install -r requirements/base.txt -U -f https://download.pytorch.org/whl/torch_stable.html --quiet
pip install -r requirements/docs.txt
# install Texlive, see https://linuxconfig.org/how-to-install-latex-on-ubuntu-20-04-focal-fossa-linux
sudo apt-get update && sudo apt-get install -y texlive-latex-extra dvipng texlive-pictures

94
.github/workflows/pt-conda.yml vendored Normal file
View File

@ -0,0 +1,94 @@
name: PyTorch & Conda
# see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
on:
# Trigger the workflow on push or pull request,
# but only for the master branch
push:
branches:
- master
pull_request:
branches:
- master
# Trigger the workflow on push or pull request
# on: [push, pull_request]
jobs:
conda:
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-20.04]
python-version: [3.7]
# todo: add nightly versions
pytorch-version: [1.3, 1.4, 1.5] # , 1.6, 1.7
# Timeout: https://stackoverflow.com/a/59076067/4521646
timeout-minutes: 20
steps:
- uses: actions/checkout@v2
- name: Setup pyTorch
run: |
python -c "fname = 'environment.yml' ; req = open(fname).read().replace('torch>=1.3', 'torch=${{ matrix.pytorch-version }}') ; open(fname, 'w').write(req)"
cat environment.yml
# TODO: set source for nightly
- name: Cache conda
uses: actions/cache@v1
env: # Increase this value to reset cache if etc/example-environment.yml has not changed
CACHE_NUMBER: 0
with:
path: ~/conda_pkgs_dir
key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('environment.yml') }}
# https://docs.conda.io/projects/conda/en/4.6.0/_downloads/52a95608c49671267e40c689e0bc00ca/conda-cheatsheet.pdf
# https://gist.github.com/mwouts/9842452d020c08faf9e84a3bba38a66f
- name: Setup Miniconda
uses: goanpeca/setup-miniconda@v1.6.0
with:
# auto-update-conda: true
auto-activate-base: false
miniconda-version: 'latest'
python-version: ${{ matrix.python-version }}
environment-file: environment.yml
activate-environment: pl-env
use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
- name: Reinstall Horovod if necessary
if: runner.os != 'windows' && matrix.python-version != '3.8'
run: |
HOROVOD_BUILT=$(python -c "import horovod.torch; horovod.torch.nccl_built(); print('SUCCESS')")
if [[ $HOROVOD_BUILT != "SUCCESS" ]]; then
pip uninstall -y horovod
HOROVOD_BUILD_ARCH_FLAGS="-mfma" pip install --no-cache-dir $(grep "horovod" requirements/extra.txt)
fi
horovodrun --check-build
shell: bash -l {0}
- name: Cache datasets
uses: actions/cache@v1
with:
path: Datasets # This path is specific to Ubuntu
# Look to see if there is a cache hit for the corresponding requirements file
key: pl-dataset
- name: Tests
run: |
conda info
conda list
# NOTE: run coverage on tests does not propagare faler status for Win, https://github.com/nedbat/coveragepy/issues/1003
python -m pytest pytorch_lightning tests -v --junitxml=junit/test-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}.xml
# coverage report
shell: bash -l {0}
- name: Upload pytest test results
uses: actions/upload-artifact@master
with:
name: pytest-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}
path: junit/test-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}.xml
# Use always() to always run this step to publish test results when there are test failures
if: always()

2
README.md
View File

@ -40,7 +40,7 @@
| System / PyTorch ver. | 1.3 (min. req.) | 1.4 | 1.5 (latest) |
| :---: | :---: | :---: | :---: |
| Linux py3.6 [CPU] | [![CircleCI](https://circleci.com/gh/PyTorchLightning/pytorch-lightning.svg?style=svg)](https://circleci.com/gh/PyTorchLightning/pytorch-lightning) | [![CircleCI](https://circleci.com/gh/PyTorchLightning/pytorch-lightning.svg?style=svg)](https://circleci.com/gh/PyTorchLightning/pytorch-lightning) | [![CircleCI](https://circleci.com/gh/PyTorchLightning/pytorch-lightning.svg?style=svg)](https://circleci.com/gh/PyTorchLightning/pytorch-lightning) |
| Conda py3.7 [linux] | ![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda%20compatibility/badge.svg) | ![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda%20compatibility/badge.svg) | ![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda%20compatibility/badge.svg) |
| Linux py3.7 [GPU] | - | - | [![Build Status](http://35.192.60.23/api/badges/PyTorchLightning/pytorch-lightning/status.svg)](http://35.192.60.23/PyTorchLightning/pytorch-lightning) |
| Linux py3.6 / py3.7 / py3.8 | [![CI testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20testing/badge.svg?event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | [![CI testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20testing/badge.svg?event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) |
| OSX py3.6 / py3.7 | - | [![CI testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20testing/badge.svg?event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | [![CI testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20testing/badge.svg?event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) |

19
environment.yml
View File

@ -1,22 +1,25 @@
# This is Conda environment file
# Usage: `conda env update -f environment.yml`
name:
pl-env
channels:
- conda-forge
- pytorch
dependencies:
- python==3.7.6
- pip==20.0.2
- tqdm>=4.35.0
#- python=3.7.6
- pip>=20.0.2
- numpy>=1.16.4
- pytorch>=1.3
- tensorboard>=1.14
- future>=0.17.1
- pyyaml>=3.13
- tqdm>=4.35.0
# For dev and testing
- tox
- check-manifest
- coverage
- codecov
- pytest>=3.0.5
@ -24,12 +27,15 @@ dependencies:
- pytest-flake8
- flake8
- autopep8
- check-manifest
- twine==1.13.0
- pillow<7.0.0
# Optional
- scipy>=0.13.3
- scikit-learn>=0.20.0
- matplotlib>=3.1.1
- omegaconf>=2.0.0
- torchtext>=0.3.1
- pip:
- test-tube>=0.7.5
@ -37,3 +43,4 @@ dependencies:
- comet_ml>=1.0.56
- wandb>=0.8.21
- neptune-client>=0.4.4
- horovod>=0.19.1

2
requirements/devel.txt
View File

@ -6,5 +6,3 @@
# extended list of dependencies dor development and run lint and tests
-r ./test.txt
cloudpickle>=1.2

3
requirements/test.txt
View File

@ -7,5 +7,8 @@ flake8
flake8-black
check-manifest
twine==1.13.0
black==19.10b0
pre-commit>=1.0
cloudpickle>=1.2