diff --git a/.circleci/config.yml b/.circleci/config.yml
index e87ad6780..96e6b42a1 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -63,7 +63,6 @@ jobs:
             ccache -z
             make
             ccache -s
-            make build/test.html build/test_data.txt
 
       - save_cache:
           paths:
@@ -93,7 +92,7 @@ jobs:
 
             source pyodide-env/bin/activate
             export PATH=$PWD/firefox:$PATH
-            pytest test -v --instafail -k firefox
+            pytest test -v -k firefox
 
   test-chrome:
     <<: *defaults
@@ -110,7 +109,7 @@ jobs:
 
             source pyodide-env/bin/activate
             export PATH=$PWD/firefox:$PATH
-            pytest test -v --instafail -k chrome
+            pytest test -v -k chrome
 
   deploy:
     machine:
diff --git a/Makefile b/Makefile
index 529466049..1181d69dd 100644
--- a/Makefile
+++ b/Makefile
@@ -101,12 +101,8 @@ build/renderedhtml.css: src/renderedhtml.less
 	lessc $< $@
 
 
-test: all build/test.html build/test_data.txt
-	py.test test -v -r sxX --instafail
-
-
-build/test_data.txt: test/data.txt
-	cp test/data.txt build/test_data.txt
+test: all
+	pytest test/ -v
 
 
 lint:
diff --git a/README.md b/README.md
index 94c0cedd7..6d861de7f 100644
--- a/README.md
+++ b/README.md
@@ -43,7 +43,7 @@ Install [geckodriver](https://github.com/mozilla/geckodriver/releases) and
 [chromedriver](https://sites.google.com/a/chromium.org/chromedriver/downloads) somewhere
 on your `PATH`.
 
-`make test`
+`pytest test/`
 
 # Benchmarking
 
diff --git a/setup.cfg b/setup.cfg
new file mode 100644
index 000000000..16b16363b
--- /dev/null
+++ b/setup.cfg
@@ -0,0 +1,4 @@
+[tool:pytest]
+addopts =
+    -r sxX
+    --instafail
diff --git a/test/conftest.py b/test/conftest.py
index 6a724be4a..4cbc1d9ee 100644
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -7,6 +7,7 @@ import multiprocessing
 import os
 import pathlib
 import queue
+import shutil
 import sys
 
 try:
@@ -39,6 +40,10 @@ class SeleniumWrapper:
 
         driver = self.get_driver()
         wait = WebDriverWait(driver, timeout=20)
+        if not (BUILD_PATH / 'test.html').exists():
+            # selenium does not expose HTTP response codes
+            raise ValueError(f"{(BUILD_PATH / 'test.html').resolve()} "
+                             f"does not exist!")
         driver.get(f'http://127.0.0.1:{PORT}/test.html')
         wait.until(PyodideInited())
         self.wait = wait
@@ -109,8 +114,16 @@ class ChromeWrapper(SeleniumWrapper):
 
 
 if pytest is not None:
+
+    @pytest.fixture(scope='session')
+    def setup_resources():
+        shutil.copyfile(TEST_PATH / 'data.txt',
+                        BUILD_PATH / 'test_data.txt')
+        shutil.copyfile(TEST_PATH.parent / 'src' / 'test.html',
+                        BUILD_PATH / 'test.html')
+
     @pytest.fixture(params=['firefox', 'chrome'])
-    def selenium_standalone(request):
+    def selenium_standalone(request, setup_resources):
         if request.param == 'firefox':
             cls = FirefoxWrapper
         elif request.param == 'chrome':
@@ -123,7 +136,7 @@ if pytest is not None:
             selenium.driver.quit()
 
     @pytest.fixture(params=['firefox', 'chrome'], scope='module')
-    def _selenium_cached(request):
+    def _selenium_cached(request, setup_resources):
         # Cached selenium instance. This is a copy-paste of
         # selenium_standalone to avoid fixture scope issues
         if request.param == 'firefox':