replace local adjustment script with external (#17582)
This commit is contained in:
Jirka Borovec
GitHub
parent
bd53b0350b
commit
51b0e81105
|
|
@ -91,8 +91,10 @@ jobs:
|
|||
|
||||
- bash: |
|
||||
PYTORCH_VERSION=$(python -c "import torch; print(torch.__version__.split('+')[0])")
|
||||
pip install -q wget packaging
|
||||
python -m wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py
|
||||
for fpath in `ls requirements/**/*.txt`; do \
|
||||
python ./requirements/pytorch/adjust-versions.py $fpath ${PYTORCH_VERSION}; \
|
||||
python ./adjust-torch-versions.py $fpath ${PYTORCH_VERSION}; \
|
||||
done
|
||||
displayName: 'Adjust dependencies'
|
||||
|
||||
|
|
|
|||
|
|
@ -98,8 +98,10 @@ jobs:
|
|||
|
||||
- bash: |
|
||||
PYTORCH_VERSION=$(python -c "import torch; print(torch.__version__.split('+')[0])")
|
||||
pip install -q wget packaging
|
||||
python -m wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py
|
||||
for fpath in `ls requirements/**/*.txt`; do \
|
||||
python ./requirements/pytorch/adjust-versions.py $fpath ${PYTORCH_VERSION}; \
|
||||
python ./adjust-torch-versions.py $fpath ${PYTORCH_VERSION}; \
|
||||
done
|
||||
# prune packages with installation issues
|
||||
pip install -q -r .actions/requirements.txt
|
||||
|
|
|
|||
|
|
@ -84,8 +84,10 @@ jobs:
|
|||
- name: Adjust PyTorch versions in requirements files
|
||||
if: ${{ matrix.requires != 'oldest' && matrix.release != 'pre' }}
|
||||
run: |
|
||||
pip install -q wget packaging
|
||||
python -m wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py
|
||||
for fpath in `ls requirements/**/*.txt`; do \
|
||||
python ./requirements/pytorch/adjust-versions.py $fpath ${{ matrix.pytorch-version }}; \
|
||||
python ./adjust-torch-versions.py $fpath ${{ matrix.pytorch-version }}; \
|
||||
done
|
||||
cat requirements/fabric/base.txt
|
||||
|
||||
|
|
|
|||
|
|
@ -88,8 +88,10 @@ jobs:
|
|||
- name: Adjust PyTorch versions in requirements files
|
||||
if: ${{ matrix.requires != 'oldest' && matrix.release != 'pre' }}
|
||||
run: |
|
||||
pip install -q wget packaging
|
||||
python -m wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py
|
||||
for fpath in `ls requirements/**/*.txt`; do \
|
||||
python ./requirements/pytorch/adjust-versions.py $fpath ${{ matrix.pytorch-version }}; \
|
||||
python ./adjust-torch-versions.py $fpath ${{ matrix.pytorch-version }}; \
|
||||
done
|
||||
cat requirements/pytorch/base.txt
|
||||
|
||||
|
|
|
|||
|
|
@ -153,7 +153,6 @@ jobs:
|
|||
|
||||
- name: Upload coverage to Codecov
|
||||
uses: codecov/codecov-action@v3
|
||||
# see: https://github.com/actions/toolkit/issues/399
|
||||
continue-on-error: true
|
||||
with:
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
|
|
|
|||
|
|
@ -88,8 +88,10 @@ RUN \
|
|||
# Disable cache \
|
||||
pip config set global.cache-dir false && \
|
||||
# set particular PyTorch version \
|
||||
pip install -q wget packaging && \
|
||||
python -m wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py && \
|
||||
for fpath in `ls requirements/**/*.txt`; do \
|
||||
python ./requirements/pytorch/adjust-versions.py $fpath ${PYTORCH_VERSION}; \
|
||||
python ./adjust-torch-versions.py $fpath ${PYTORCH_VERSION}; \
|
||||
done && \
|
||||
CUDA_VERSION_MM=${CUDA_VERSION%.*} && \
|
||||
pip install \
|
||||
|
|
|
|||
|
|
@ -1,63 +0,0 @@
|
|||
import os
|
||||
import re
|
||||
import sys
|
||||
from typing import Dict, Optional
|
||||
|
||||
# IMPORTANT: this list needs to be sorted in reverse
|
||||
VERSIONS = [
|
||||
{"torch": "2.1.0", "torchvision": "0.16.0"}, # nightly
|
||||
{"torch": "2.0.0", "torchvision": "0.15.1"}, # stable
|
||||
{"torch": "1.13.1", "torchvision": "0.14.1"},
|
||||
{"torch": "1.13.0", "torchvision": "0.14.0"},
|
||||
{"torch": "1.12.1", "torchvision": "0.13.1"},
|
||||
{"torch": "1.12.0", "torchvision": "0.13.0"},
|
||||
{"torch": "1.11.0", "torchvision": "0.12.0"},
|
||||
]
|
||||
|
||||
|
||||
def find_latest(ver: str) -> Dict[str, str]:
|
||||
# drop all except semantic version
|
||||
ver = re.search(r"([\.\d]+)", ver).groups()[0]
|
||||
# in case there remaining dot at the end - e.g "1.9.0.dev20210504"
|
||||
ver = ver[:-1] if ver[-1] == "." else ver
|
||||
print(f"\n\n\nfinding ecosystem versions for: {ver}")
|
||||
|
||||
# find first match
|
||||
for option in VERSIONS:
|
||||
if option["torch"].startswith(ver):
|
||||
return option
|
||||
|
||||
raise ValueError(f"Missing {ver} in {VERSIONS}")
|
||||
|
||||
|
||||
def replace(req: str, torch_version: Optional[str] = None) -> str:
|
||||
if not torch_version:
|
||||
import torch
|
||||
|
||||
torch_version = torch.__version__
|
||||
assert torch_version, f"invalid torch: {torch_version}" # noqa: S101
|
||||
|
||||
# remove comments and strip whitespace
|
||||
req = re.sub(rf"\s*#.*{os.linesep}", os.linesep, req).strip()
|
||||
|
||||
latest = find_latest(torch_version)
|
||||
for lib, version in latest.items():
|
||||
replace = f"{lib}=={version}" if version else ""
|
||||
req = re.sub(rf"\b{lib}(?!\w).*", replace, req)
|
||||
|
||||
return req
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) == 3:
|
||||
requirements_path, torch_version = sys.argv[1:]
|
||||
else:
|
||||
requirements_path, torch_version = sys.argv[1], None
|
||||
print(f"requirements_path='{requirements_path}' with torch_version='{torch_version}'")
|
||||
|
||||
with open(requirements_path) as fp:
|
||||
requirements = fp.read()
|
||||
requirements = replace(requirements, torch_version)
|
||||
print(requirements) # on purpose - to debug
|
||||
with open(requirements_path, "w") as fp:
|
||||
fp.write(requirements)
|
||||
|
|
@ -13,8 +13,10 @@ fi
|
|||
|
||||
echo "--- Install packages ---"
|
||||
# set particular PyTorch version
|
||||
pip install -q wget packaging
|
||||
python3 -m wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py
|
||||
for fpath in `ls requirements/**/*.txt`; do
|
||||
python3 requirements/pytorch/adjust-versions.py $fpath {PYTORCH_VERSION};
|
||||
python3 adjust-torch-versions.py $fpath {PYTORCH_VERSION};
|
||||
done
|
||||
pip install .[fabric-test] pytest-timeout
|
||||
pip list
|
||||
|
|
|
|||
|
|
@ -13,8 +13,10 @@ fi
|
|||
|
||||
echo "--- Install packages ---"
|
||||
# set particular PyTorch version
|
||||
pip install -q wget packaging
|
||||
python3 -m wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py
|
||||
for fpath in `ls requirements/**/*.txt`; do
|
||||
python3 requirements/pytorch/adjust-versions.py $fpath {PYTORCH_VERSION};
|
||||
python3 adjust-torch-versions.py $fpath {PYTORCH_VERSION};
|
||||
done
|
||||
pip install .[pytorch-extra,pytorch-test] pytest-timeout
|
||||
pip list
|
||||
|
|
|
|||
Loading…
Reference in New Issue