diff --git a/conftest.py b/conftest.py
index 36c9fb40e..33f200eab 100644
--- a/conftest.py
+++ b/conftest.py
@@ -2,6 +2,7 @@
 Various common utilities for testing.
 """
 import pathlib
+import re
 import sys
 
 import pytest
@@ -13,7 +14,6 @@ sys.path.append(str(ROOT_PATH / "pyodide-build"))
 sys.path.append(str(ROOT_PATH / "src" / "py"))
 
 import pytest_pyodide.runner
-from pytest_pyodide.utils import maybe_skip_test
 from pytest_pyodide.utils import package_is_built as _package_is_built
 
 # There are a bunch of global objects that occasionally enter the hiwire cache
@@ -52,6 +52,49 @@ def pytest_addoption(parser):
     )
 
 
+def maybe_skip_test(item, delayed=False):
+    """If necessary skip test at the fixture level, to avoid
+    loading the selenium_standalone fixture which takes a long time.
+    """
+    browsers = "|".join(["firefox", "chrome", "node"])
+    is_common_test = str(item.fspath).endswith("test_packages_common.py")
+
+    skip_msg = None
+    # Testing a package. Skip the test if the package is not built.
+    match = re.match(
+        r".*/packages/(?P<name>[\w\-]+)/test_[\w\-]+\.py", str(item.parent.fspath)
+    )
+    if match and not is_common_test:
+        package_name = match.group("name")
+        if not package_is_built(package_name) and re.match(
+            rf"test_[\w\-]+\[({browsers})[^\]]*\]", item.name
+        ):
+            skip_msg = f"package '{package_name}' is not built."
+
+    # Common package import test. Skip it if the package is not built.
+    if skip_msg is None and is_common_test and item.name.startswith("test_import"):
+        match = re.match(rf"test_import\[({browsers})-(?P<name>[\w-]+)\]", item.name)
+        if match:
+            package_name = match.group("name")
+            if not package_is_built(package_name):
+                # If the test is going to be skipped remove the
+                # selenium_standalone as it takes a long time to initialize
+                skip_msg = f"package '{package_name}' is not built."
+        else:
+            raise AssertionError(
+                f"Couldn't parse package name from {item.name}. This should not happen!"
+            )
+
+    # TODO: also use this hook to skip doctests we cannot run (or run them
+    # inside the selenium wrapper)
+
+    if skip_msg is not None:
+        if delayed:
+            item.add_marker(pytest.mark.skip(reason=skip_msg))
+        else:
+            pytest.skip(skip_msg)
+
+
 def pytest_configure(config):
     """Monkey patch the function cwd_relative_nodeid
 
@@ -92,7 +135,7 @@ def pytest_collection_modifyitems(config, items):
             item.add_marker(pytest.mark.skip(reason="previously passed"))
             continue
 
-        maybe_skip_test(item, config.getoption("--dist-dir"), delayed=True)
+        maybe_skip_test(item, delayed=True)
 
 
 # Save test results to a cache