From 9d35c61f5f103b04b7fd1f504b8b937450482d07 Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Sun, 28 Jan 2024 11:22:07 +0100 Subject: [PATCH] ci: adding missing requirements for generating legacy ckpt (#19353) --- .github/workflows/_legacy-checkpoints.yml | 12 +++++----- .github/workflows/ci-checkpoints.yml | 6 +++-- tests/legacy/generate_checkpoints.sh | 29 ++++++++++++++--------- tests/legacy/requirements.txt | 2 -- 4 files changed, 28 insertions(+), 21 deletions(-) diff --git a/.github/workflows/_legacy-checkpoints.yml b/.github/workflows/_legacy-checkpoints.yml index 3ca59cfd8b..ffcb9306a8 100644 --- a/.github/workflows/_legacy-checkpoints.yml +++ b/.github/workflows/_legacy-checkpoints.yml @@ -42,7 +42,7 @@ on: type: boolean env: - legacy_dir: "tests/legacy" + LEGACY_FOLDER: "tests/legacy" TORCH_URL: "https://download.pytorch.org/whl/cpu/torch_stable.html" defaults: @@ -91,11 +91,11 @@ jobs: run: python -c "import pytorch_lightning as pl; print(f'pl-version={pl.__version__}')" >> $GITHUB_OUTPUT || echo pl-version='' >> $GITHUB_OUTPUT - name: Generate checkpoints - working-directory: ${{ env.legacy_dir }} + working-directory: ${{ env.LEGACY_FOLDER }} run: bash generate_checkpoints.sh ${{ inputs.pl_version }} - name: Rename local to actual version if: inputs.upload_local - working-directory: ${{ env.legacy_dir }} + working-directory: ${{ env.LEGACY_FOLDER }} run: mv checkpoints/local checkpoints/${{ steps.decide-version.outputs.pl-version }} - name: "Determine: Keep artifact & DryRun" @@ -107,12 +107,12 @@ jobs: uses: actions/upload-artifact@v3 with: name: checkpoints-${{ github.sha }} - path: ${{ env.legacy_dir }}/checkpoints/ + path: ${{ env.LEGACY_FOLDER }}/checkpoints/ retention-days: ${{ env.KEEP_DAYS }} - run: pip install -r requirements/ci.txt - name: Upload checkpoints to S3 - working-directory: ${{ env.legacy_dir }} + working-directory: ${{ env.LEGACY_FOLDER }} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PUB_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PUB_SECRET_KEY }} @@ -134,7 +134,7 @@ jobs: ref: master - name: Append a new version to legacy checkpoint list - working-directory: ${{ env.legacy_dir }} + working-directory: ${{ env.LEGACY_FOLDER }} run: echo ${PL_VERSION} >> back-compatible-versions.txt - name: Create Pull Request diff --git a/.github/workflows/ci-checkpoints.yml b/.github/workflows/ci-checkpoints.yml index 020d02dba9..f7b172b1a5 100644 --- a/.github/workflows/ci-checkpoints.yml +++ b/.github/workflows/ci-checkpoints.yml @@ -1,4 +1,4 @@ -name: Validate legacy ckpts +name: Generate legacy ckpts on: push: @@ -8,8 +8,10 @@ on: types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped paths: - ".github/workflows/ci-checkpoints.yml" - - ".github/workflows/_legacy-checkpoints.yml.yml" + - ".github/workflows/_legacy-checkpoints.yml" - "requirements/ci.txt" + - "requirements/pytorch/*.txt" + - "tests/legacy/*" jobs: try-legacy-checkpoints: diff --git a/tests/legacy/generate_checkpoints.sh b/tests/legacy/generate_checkpoints.sh index fcbfcf8a50..1d083a2a8e 100644 --- a/tests/legacy/generate_checkpoints.sh +++ b/tests/legacy/generate_checkpoints.sh @@ -6,11 +6,12 @@ # bash generate_checkpoints.sh set -e -LEGACY_PATH=$(cd $(dirname $0); pwd -P) -ENV_PATH=$LEGACY_PATH/vEnv -export PYTHONPATH=$(dirname $LEGACY_PATH) # for `import tests_pytorch` -printf "LEGACY_PATH: $LEGACY_PATH" +LEGACY_FOLDER=$(cd $(dirname $0); pwd -P) +printf "LEGACY_FOLDER: $LEGACY_FOLDER" +TESTS_FOLDER=$(dirname $LEGACY_FOLDER) +ENV_PATH=$LEGACY_FOLDER/vEnv printf "ENV_PATH: $ENV_PATH" +export PYTHONPATH=$TESTS_FOLDER # for `import tests_pytorch` printf "PYTHONPATH: $PYTHONPATH" rm -rf $ENV_PATH @@ -19,11 +20,11 @@ function create_and_save_checkpoint { python -m pip --version python -m pip list - python $LEGACY_PATH/simple_classif_training.py $pl_ver + python $LEGACY_FOLDER/simple_classif_training.py $pl_ver - cp $LEGACY_PATH/simple_classif_training.py $LEGACY_PATH/checkpoints/$pl_ver - mv $LEGACY_PATH/checkpoints/$pl_ver/lightning_logs/version_0/checkpoints/*.ckpt $LEGACY_PATH/checkpoints/$pl_ver/ - rm -rf $LEGACY_PATH/checkpoints/$pl_ver/lightning_logs + cp $LEGACY_FOLDER/simple_classif_training.py $LEGACY_FOLDER/checkpoints/$pl_ver + mv $LEGACY_FOLDER/checkpoints/$pl_ver/lightning_logs/version_0/checkpoints/*.ckpt $LEGACY_FOLDER/checkpoints/$pl_ver/ + rm -rf $LEGACY_FOLDER/checkpoints/$pl_ver/lightning_logs } # iterate over all arguments assuming that each argument is version @@ -35,9 +36,12 @@ do python -m venv $ENV_PATH source $ENV_PATH/bin/activate - python -m pip install "pytorch_lightning==$pl_ver" -r $LEGACY_PATH/requirements.txt -f https://download.pytorch.org/whl/cpu/torch_stable.html + python -m pip install "pytorch_lightning==$pl_ver" \ + -r $LEGACY_FOLDER/requirements.txt \ + -r "$(dirname $TESTS_FOLDER)/requirements/pytorch/test.txt" \ + -f https://download.pytorch.org/whl/cpu/torch_stable.html - rm -rf $LEGACY_PATH/checkpoints/$pl_ver + rm -rf $LEGACY_FOLDER/checkpoints/$pl_ver create_and_save_checkpoint deactivate @@ -48,7 +52,10 @@ done if [[ -z "$@" ]]; then printf "\n\n processing local version\n" - python -m pip install -r $LEGACY_PATH/requirements.txt + python -m pip install \ + -r $LEGACY_FOLDER/requirements.txt \ + -r "$(dirname $TESTS_FOLDER)/requirements/pytorch/test.txt" \ + -f https://download.pytorch.org/whl/cpu/torch_stable.html pl_ver="local" create_and_save_checkpoint fi diff --git a/tests/legacy/requirements.txt b/tests/legacy/requirements.txt index 43cd22c429..e69de29bb2 100644 --- a/tests/legacy/requirements.txt +++ b/tests/legacy/requirements.txt @@ -1,2 +0,0 @@ -torchmetrics # necessary because old PL versions don't have it as dependency -scikit-learn