Rerun failed tests sequentially (#859)

.circleci/config.yml

@@ -119,7 +119,7 @@ jobs:
       - run:
           name: test
           command: |
-            pytest src packages/micropip/ -v -k firefox -n 3
+            tools/pytest_wrapper.py src packages/micropip/ -v -k firefox -n 3
 
   test-core-chrome:
     <<: *defaults
@@ -129,7 +129,7 @@ jobs:
       - run:
           name: test
           command: |
-            pytest src packages/micropip/ -v -k chrome -n 3
+            tools/pytest_wrapper.py src packages/micropip/ -v -k chrome -n 3
 
   test-packages-firefox:
     <<: *defaults
@@ -139,7 +139,7 @@ jobs:
       - run:
           name: test
           command: |
-            pytest packages/test* packages/*/test* -v -k firefox -n 2
+            tools/pytest_wrapper.py packages/test* packages/*/test* -v -k firefox -n 2
 
   test-packages-chrome:
     <<: *defaults
@@ -149,7 +149,7 @@ jobs:
       - run:
           name: test
           command: |
-            pytest packages/test* packages/*/test* -v -k chrome -n 2
+            tools/pytest_wrapper.py packages/test* packages/*/test* -v -k chrome -n 2
 
   test-python:
     <<: *defaults

src/tests/test_pytest_wrapper.py

@@ -0,0 +1,16 @@
+import sys
+from pathlib import Path
+
+sys.path.append(str(Path(__file__).parents[2] / "tools"))
+
+from pytest_wrapper import clean_args
+
+
+def test_find_imports():
+    args = ["-v", "-n", "3"]
+    clean_args(args)
+    assert args == ["-v"]
+
+    args = ["-v", "-n", "3", "-k", "firefox"]
+    clean_args(args)
+    assert args == ["-v", "-k", "firefox"]

tools/pytest_wrapper.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+
+import subprocess
+from typing import List
+import sys
+
+args = sys.argv[1:]
+
+
+def clean_args(args: List[str]) -> None:
+    """Remove -n <n> from argument list"""
+    for i in range(0, len(args)):
+        if args[i] == "-n":
+            del args[i : i + 2]
+            break
+
+
+if __name__ == "__main__":
+    try:
+        subprocess.run(["pytest"] + args, check=True)
+    except subprocess.CalledProcessError:
+        # Failed tests. Look up number of failed tests
+        with open(".pytest_cache/v/cache/lastfailed") as f:
+            num_failed = sum(1 for line in f) - 2
+
+        if num_failed < 10:
+            print("Rerunnning failed tests sequentially")
+            clean_args(args)
+            subprocess.run(["pytest", "--lf"] + args, check=True)
+        else:
+            print("More than 9 tests failed. Not rerunning")
+            raise