CI Split up package test jobs (#2852)

This PR splits package test in CI so that no-numpy-dependents packages can be tested earlier. In detail, pytest will now save test results into its cache directory, and if --skip-passed option is given, it will skip previously successful tests.

.circleci/config.yml

@@ -224,6 +224,10 @@ jobs:
       test-params:
         description: The tests to run.
         type: string
+      cache-dir:
+        description: pytest-cache-dir.
+        type: string
+        default: ""
     <<: *defaults
     resource_class: medium+
     steps:
@@ -236,11 +240,18 @@ jobs:
             mkdir test-results
             pip install ./pyodide-test-runner
             npm install -g node-fetch@2
+            if [ -z "<< parameters.cache-dir >>" ]; then
+              export CACHE_DIR=".test_cache/.pytest_cache_$(echo $RANDOM | md5sum | head -c 10)"
+            else
+              export CACHE_DIR=".test_cache/<< parameters.cache-dir >>"
+            fi
+            echo "pytest cache dir: $CACHE_DIR"
             tools/pytest_wrapper.py \
               --junitxml=test-results/junit.xml \
               --verbose \
               --durations 50 \
-              << parameters.test-params >>
+              << parameters.test-params >> \
+              -o cache_dir=$CACHE_DIR
       - store_test_results:
           path: test-results
 
@@ -572,27 +583,78 @@ workflows:
               only: /.*/
 
       - test-main:
-          name: test-packages-chrome
+          name: test-packages-chrome-no-numpy-dependents
           test-params: -k chrome packages/test* packages/*/test*
+          cache-dir: .pytest_cache_chrome
           requires:
+            - build-packages-no-numpy-dependents
+          filters:
+            tags:
+              only: /.*/
+          post-steps:
+            - persist_to_workspace:
+                root: .
+                paths:
+                  - ./.test_cache
+
+      - test-main:
+          name: test-packages-chrome
+          test-params: -k chrome packages/test* packages/*/test* --skip-passed
+          cache-dir: .pytest_cache_chrome
+          requires:
+            - test-packages-chrome-no-numpy-dependents
             - build-packages
           filters:
             tags:
               only: /.*/
 
+      - test-main:
+          name: test-packages-firefox-no-numpy-dependents
+          test-params: -k firefox packages/test* packages/*/test*
+          cache-dir: .pytest_cache_firefox
+          requires:
+            - build-packages-no-numpy-dependents
+          filters:
+            tags:
+              only: /.*/
+          post-steps:
+            - persist_to_workspace:
+                root: .
+                paths:
+                  - ./.test_cache
+
       - test-main:
           name: test-packages-firefox
-          test-params: -k firefox packages/test* packages/*/test*
+          test-params: -k firefox packages/test* packages/*/test* --skip-passed
+          cache-dir: .pytest_cache_firefox
           requires:
+            - test-packages-firefox-no-numpy-dependents
             - build-packages
           filters:
             tags:
               only: /.*/
 
       - test-main:
-          name: test-packages-node
+          name: test-packages-node-no-numpy-dependents
           test-params: -k node packages/test* packages/*/test*
+          cache-dir: .pytest_cache_node
           requires:
+            - build-packages-no-numpy-dependents
+          filters:
+            tags:
+              only: /.*/
+          post-steps:
+            - persist_to_workspace:
+                root: .
+                paths:
+                  - ./.test_cache
+
+      - test-main:
+          name: test-packages-node
+          test-params: -k node packages/test* packages/*/test* --skip-passed
+          cache-dir: .pytest_cache_node
+          requires:
+            - test-packages-node-no-numpy-dependents
             - build-packages
           filters:
             tags:

conftest.py

@@ -24,6 +24,14 @@ def pytest_addoption(parser):
         action="store_true",
         help="If provided, tests marked as xfail will be run",
     )
+    group.addoption(
+        "--skip-passed",
+        action="store_true",
+        help=(
+            "If provided, tests that passed on the last run will be skipped. "
+            "CAUTION: this will skip tests even if tests are modified"
+        ),
+    )
 
 
 def pytest_configure(config):
@@ -56,10 +64,44 @@ def pytest_collection_modifyitems(config, items):
     config : pytest config
     items : list of collected items
     """
+    prev_test_result = {}
+    if config.getoption("--skip-passed"):
+        cache = config.cache
+        prev_test_result = cache.get("cache/lasttestresult", {})
+
     for item in items:
+        if prev_test_result.get(item.nodeid) in ("passed", "warnings", "skip_passed"):
+            item.add_marker(pytest.mark.skip(reason="previously passed"))
+            continue
+
         maybe_skip_test(item, config.getoption("--dist-dir"), delayed=True)
 
 
+# Save test results to a cache
+# Code adapted from: https://github.com/pytest-dev/pytest/blob/main/src/_pytest/pastebin.py
+@pytest.hookimpl(trylast=True)
+def pytest_terminal_summary(terminalreporter):
+    tr = terminalreporter
+    cache = tr.config.cache
+    assert cache
+
+    test_result = {}
+    for status in tr.stats:
+        if status in ("warnings", "deselected"):
+            continue
+
+        for test in tr.stats[status]:
+            try:
+                if test.longrepr and test.longrepr[2] in "previously passed":
+                    test_result[test.nodeid] = "skip_passed"
+                else:
+                    test_result[test.nodeid] = test.outcome
+            except Exception:
+                pass
+
+    cache.set("cache/lasttestresult", test_result)
+
+
 @pytest.hookimpl(hookwrapper=True)
 def pytest_runtest_call(item):
     """We want to run extra verification at the start and end of each test to