diff --git a/Makefile b/Makefile
index 1709c9a8b..84820d9ea 100644
--- a/Makefile
+++ b/Makefile
@@ -58,7 +58,8 @@ all: build/pyodide.asm.js \
   build/test.data \
   build/packages.json \
   build/test.html \
-  build/webworker.js
+  build/webworker.js \
+  build/webworker_dev.js
 
 
 build/pyodide.asm.js: src/main.bc src/jsimport.bc src/jsproxy.bc src/js2python.bc \
@@ -115,6 +116,12 @@ build/renderedhtml.css: src/renderedhtml.less
 
 build/webworker.js: src/webworker.js
 	cp $< $@
+	sed -i -e 's#{{DEPLOY}}#https://iodide.io/pyodide-demo/#g' $@
+
+build/webworker_dev.js: src/webworker.js
+	cp $< $@
+	sed -i -e "s#{{DEPLOY}}##g" $@
+	sed -i -e "s#pyodide.js#pyodide_dev.js#g" $@
 
 test: all
 	pytest test/ -v
diff --git a/src/webworker.js b/src/webworker.js
index 07dfc4992..a4ae5273a 100644
--- a/src/webworker.js
+++ b/src/webworker.js
@@ -1,4 +1,4 @@
-self.languagePluginUrl = 'http://localhost:8000/'
+self.languagePluginUrl = '{{DEPLOY}}'
 importScripts('./pyodide.js')
 
 var onmessage = function(e) { // eslint-disable-line no-unused-vars
diff --git a/test/conftest.py b/test/conftest.py
index 85323d0f7..1538a3ac9 100644
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -143,6 +143,66 @@ class SeleniumWrapper:
             catch (error) {{ console.log(error.stack); throw error; }}"""
         return self.driver.execute_script(catch)
 
+    def initWebWorker(self):
+        url = f'http://{self.server_hostname}:{self.server_port}/webworker_dev.js'
+        self.run_js(
+            f"""
+            window.done = false;
+            window.pyodideWorker = new Worker( '{url}' );
+
+            window.pyodideWorker.onerror = function(e) {{
+                window.output = e;
+                window.error = true;
+                window.done = true;
+            }};
+
+            window.pyodideWorker.onmessage = function(e) {{
+              if (e.data.results) {{
+                window.output = e.data.results;
+                window.error = false;
+              }} else {{
+                window.output = e.data.error;
+                window.error = true;
+              }}
+              window.done = true;
+            }};
+            """
+        )
+
+    def run_webworker(self, code):
+        from selenium.common.exceptions import TimeoutException
+        self.initWebWorker()
+        if isinstance(code, str) and code.startswith('\n'):
+            # we have a multiline string, fix indentation
+            code = textwrap.dedent(code)
+        self.run_js(
+            """
+            var data = {{
+              python: {!r}
+            }};
+
+            window.pyodideWorker.postMessage(data);
+            """.format(code)
+        )
+        try:
+            self.wait.until(PackageLoaded())
+        except TimeoutException:
+            _display_driver_logs(self.browser, self.driver)
+            print(self.logs)
+            raise TimeoutException('run_webworker timed out')
+        return self.run_js(
+            """
+            if (window.error) {
+                if (window.output instanceof Error) {
+                    throw window.output;
+                } else {
+                    throw new Error(String(window.output))
+                }
+            }
+            return window.output;
+            """
+        )
+
     def load_package(self, packages):
         self.run_js(
             'window.done = false\n' +
diff --git a/test/test_webworker.py b/test/test_webworker.py
new file mode 100644
index 000000000..2b49d3446
--- /dev/null
+++ b/test/test_webworker.py
@@ -0,0 +1,69 @@
+def test_runwebworker(selenium_standalone):
+    output = selenium_standalone.run_webworker(
+        """
+        import numpy as np
+        x = np.zeros(5)
+        str(x)
+        """
+    )
+    assert output == '[0. 0. 0. 0. 0.]'
+
+
+def test_runwebworker_different_package_name(selenium_standalone):
+    output = selenium_standalone.run_webworker(
+        """
+        import dateutil
+        dateutil.__version__
+        """
+    )
+    assert isinstance(output, str)
+
+
+def test_runwebworker_no_imports(selenium_standalone):
+    output = selenium_standalone.run_webworker(
+        """
+        42
+        """
+    )
+    assert output == 42
+
+
+def test_runwebworker_missing_import(selenium_standalone):
+    try:
+        selenium_standalone.run_webworker(
+            """
+            import foo
+            """
+        )
+    except selenium_standalone.JavascriptException as e:
+        assert "ModuleNotFoundError" in str(e)
+    else:
+        assert False
+
+
+def test_runwebworker_exception(selenium_standalone):
+    try:
+        selenium_standalone.run_webworker(
+            """
+            42 / 0
+            """
+        )
+    except selenium_standalone.JavascriptException as e:
+        assert "ZeroDivisionError" in str(e)
+    else:
+        assert False
+
+
+def test_runwebworker_exception_after_import(selenium_standalone):
+    try:
+        selenium_standalone.run_webworker(
+            """
+            import numpy as np
+            x = np.empty(5)
+            42 / 0
+            """
+        )
+    except selenium_standalone.JavascriptException as e:
+        assert "ZeroDivisionError" in str(e)
+    else:
+        assert False