diff --git a/.azure/gpu-tests-pytorch.yml b/.azure/gpu-tests-pytorch.yml index 156513d604..dd1a3d4abc 100644 --- a/.azure/gpu-tests-pytorch.yml +++ b/.azure/gpu-tests-pytorch.yml @@ -31,6 +31,7 @@ pr: - "src/lightning/pytorch/**" - "src/pytorch_lightning/*" - "tests/tests_pytorch/**" + - "tests/run_standalone_*.sh" - "pyproject.toml" # includes pytest config - "requirements/fabric/**" - "src/lightning/fabric/**" diff --git a/.github/checkgroup.yml b/.github/checkgroup.yml index 0c5e7e4579..c4c88b12a2 100644 --- a/.github/checkgroup.yml +++ b/.github/checkgroup.yml @@ -60,6 +60,7 @@ subprojects: - "src/lightning/pytorch/**" - "src/pytorch_lightning/*" - "tests/tests_pytorch/**" + - "tests/run_standalone_*.sh" - "pyproject.toml" # includes pytest config - "requirements/fabric/**" - "src/lightning/fabric/**" @@ -201,7 +202,6 @@ subprojects: - ".azure/gpu-tests-fabric.yml" - "examples/fabric/**" - "examples/run_fabric_examples.sh" - - "tests/run_standalone_*.sh" - "requirements/fabric/**" - "src/lightning/__init__.py" - "src/lightning/__setup__.py" @@ -209,6 +209,7 @@ subprojects: - "src/lightning/fabric/**" - "src/lightning_fabric/*" - "tests/tests_fabric/**" + - "tests/run_standalone_*.sh" - "pyproject.toml" # includes pytest config - "!requirements/*/docs.txt" - "!*.md" diff --git a/tests/run_standalone_tests.sh b/tests/run_standalone_tests.sh index 0de781b0c4..0aa0bacff1 100755 --- a/tests/run_standalone_tests.sh +++ b/tests/run_standalone_tests.sh @@ -26,18 +26,17 @@ export PL_RUN_STANDALONE_TESTS=1 defaults=" -m coverage run --source ${source} --append -m pytest --no-header -v -s --timeout 120 " echo "Using defaults: ${defaults}" -# get the testing location as the fist argument +# get the testing location as the first argument test_path=$1 printf "source path: $test_path\n" # collect all tests with parametrization based filtering with PL_RUN_STANDALONE_TESTS standalone_tests=$(python3 -m pytest $test_path -q --collect-only --pythonwarnings ignore) -printf "Collected tests: \n $standalone_tests" +printf "Collected tests: \n $standalone_tests\n" # match only lines with tests -parametrizations=$(grep -oP '\S+::test_\S+' <<< "$standalone_tests") +parametrizations=$(perl -nle 'print $& while m{\S+::test_\S+}g' <<< "$standalone_tests") # convert the list to be array parametrizations_arr=($parametrizations) - report='' rm -f standalone_test_output.txt # in case it exists, remove it @@ -47,7 +46,7 @@ function show_batched_output { if [ -f standalone_test_output.txt ]; then # if exists cat standalone_test_output.txt # heuristic: stop if there's mentions of errors. this can prevent false negatives when only some of the ranks fail - if grep -iE 'error|exception|traceback|failed' standalone_test_output.txt | grep -vE 'on_exception|xfailed' | grep -qv -f testnames.txt; then + if perl -nle 'print if /error|(?