Split off webworker / node compatibility into separate file (#2129)

.circleci/config.yml

@@ -308,8 +308,7 @@ jobs:
             git clone https://github.com/pyodide/pyodide-webpack-example.git
             cd pyodide-webpack-example
             npm ci
-            cp ../src/js/load-pyodide.js node_modules/pyodide/load-pyodide.js
-            head -20 node_modules/pyodide/load-pyodide.js
+            cp ../src/js/*.js node_modules/pyodide/
             npx webpack
 
   benchmark:

src/js/api.js

@@ -1,5 +1,5 @@
 import { Module } from "./module.js";
-import { loadPackage, loadedPackages } from "./load-pyodide.js";
+import { loadPackage, loadedPackages } from "./load-package.js";
 import { isPyProxy, PyBuffer } from "./pyproxy.gen.js";
 export { loadPackage, loadedPackages, isPyProxy };
 

src/js/compat.js

@@ -0,0 +1,113 @@
+// Detect if we're in node
+export const IN_NODE =
+  typeof process !== "undefined" &&
+  process.release &&
+  process.release.name === "node" &&
+  typeof process.browser ===
+    "undefined"; /* This last condition checks if we run the browser shim of process */
+
+export let nodePathMod;
+export let nodeFetch;
+export let nodeFsPromisesMod;
+export let nodeVmMod;
+
+/**
+ * If we're in node, it's most convenient to import various node modules on
+ * initialization. Otherwise, this does nothing.
+ * @private
+ */
+export async function initNodeModules() {
+  if (!IN_NODE) {
+    return;
+  }
+  nodePathMod = (await import(/* webpackIgnore: true */ "path")).default;
+  nodeFsPromisesMod = await import(/* webpackIgnore: true */ "fs/promises");
+  nodeFetch = (await import(/* webpackIgnore: true */ "node-fetch")).default;
+  nodeVmMod = (await import(/* webpackIgnore: true */ "vm")).default;
+}
+
+/**
+ * Load a binary file, only for use in Node. If the path explicitly is a URL,
+ * then fetch from a URL, else load from the file system.
+ * @param {str} indexURL base path to resolve relative paths
+ * @param {str} path the path to load
+ * @returns An ArrayBuffer containing the binary data
+ * @private
+ */
+async function node_loadBinaryFile(indexURL, path) {
+  if (path.includes("://")) {
+    let response = await nodeFetch(path);
+    if (!response.ok) {
+      throw new Error(`Failed to load '${path}': request failed.`);
+    }
+    return await response.arrayBuffer();
+  } else {
+    const data = await nodeFsPromisesMod.readFile(`${indexURL}${path}`);
+    return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
+  }
+}
+
+/**
+ * Load a binary file, only for use in browser. Resolves relative paths against
+ * indexURL.
+ *
+ * @param {str} indexURL base path to resolve relative paths
+ * @param {str} path the path to load
+ * @returns An ArrayBuffer containing the binary data
+ * @private
+ */
+async function browser_loadBinaryFile(indexURL, path) {
+  const base = new URL(indexURL, location);
+  const url = new URL(path, base);
+  let response = await fetch(url);
+  if (!response.ok) {
+    throw new Error(`Failed to load '${url}': request failed.`);
+  }
+  return new Uint8Array(await response.arrayBuffer());
+}
+
+export let _loadBinaryFile;
+if (IN_NODE) {
+  _loadBinaryFile = node_loadBinaryFile;
+} else {
+  _loadBinaryFile = browser_loadBinaryFile;
+}
+
+/**
+ * Currently loadScript is only used once to load `pyodide.asm.js`.
+ * @param {string) url
+ * @async
+ * @private
+ */
+export let loadScript;
+
+if (globalThis.document) {
+  // browser
+  loadScript = async (url) => await import(/* webpackIgnore: true */ url);
+} else if (globalThis.importScripts) {
+  // webworker
+  loadScript = async (url) => {
+    // This is async only for consistency
+    globalThis.importScripts(url);
+  };
+} else if (IN_NODE) {
+  loadScript = nodeLoadScript;
+} else {
+  throw new Error("Cannot determine runtime environment");
+}
+
+/**
+ * Load a text file and executes it as Javascript
+ * @param {str} url The path to load. May be a url or a relative file system path.
+ * @private
+ */
+async function nodeLoadScript(url) {
+  if (url.includes("://")) {
+    // If it's a url, load it with fetch then eval it.
+    nodeVmMod.runInThisContext(await (await nodeFetch(url)).text());
+  } else {
+    // Otherwise, hopefully it is a relative path we can load from the file
+    // system.
+    await import(nodePathMod.resolve(url));
+  }
+}

src/js/load-package.js

@@ -1,36 +1,5 @@
 import { Module } from "./module.js";
-
-//
-// Initialization code and node/browser shims
-//
-
-// Detect if we're in node
-const IN_NODE =
-  typeof process !== "undefined" &&
-  process.release &&
-  process.release.name === "node" &&
-  typeof process.browser ===
-    "undefined"; /* This last condition checks if we run the browser shim of process */
-
-let nodePathMod;
-let nodeFetch;
-let nodeFsPromisesMod;
-let nodeVmMod;
-
-/**
- * If we're in node, it's most convenient to import various node modules on
- * initialization. Otherwise, this does nothing.
- * @private
- */
-export async function initNodeModules() {
-  if (!IN_NODE) {
-    return;
-  }
-  nodePathMod = (await import(/* webpackIgnore: true */ "path")).default;
-  nodeFsPromisesMod = await import(/* webpackIgnore: true */ "fs/promises");
-  nodeFetch = (await import(/* webpackIgnore: true */ "node-fetch")).default;
-  nodeVmMod = (await import(/* webpackIgnore: true */ "vm")).default;
-}
+import { IN_NODE, nodeFsPromisesMod, _loadBinaryFile } from "./compat.js";
 
 /** @typedef {import('./pyproxy.js').PyProxy} PyProxy */
 /** @private */
@@ -70,92 +39,6 @@ export async function initializePackageIndex(indexURL) {
   }
 }
 
-/**
- * Load a binary file, only for use in Node. If the path explicitly is a URL,
- * then fetch from a URL, else load from the file system.
- * @param {str} indexURL base path to resolve relative paths
- * @param {str} path the path to load
- * @returns An ArrayBuffer containing the binary data
- * @private
- */
-async function node_loadBinaryFile(indexURL, path) {
-  if (path.includes("://")) {
-    let response = await nodeFetch(path);
-    if (!response.ok) {
-      throw new Error(`Failed to load '${path}': request failed.`);
-    }
-    return await response.arrayBuffer();
-  } else {
-    const data = await nodeFsPromisesMod.readFile(`${indexURL}${path}`);
-    return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
-  }
-}
-
-/**
- * Load a binary file, only for use in browser. Resolves relative paths against
- * indexURL.
- *
- * @param {str} indexURL base path to resolve relative paths
- * @param {str} path the path to load
- * @returns An ArrayBuffer containing the binary data
- * @private
- */
-async function browser_loadBinaryFile(indexURL, path) {
-  const base = new URL(indexURL, location);
-  const url = new URL(path, base);
-  let response = await fetch(url);
-  if (!response.ok) {
-    throw new Error(`Failed to load '${url}': request failed.`);
-  }
-  return new Uint8Array(await response.arrayBuffer());
-}
-
-export let _loadBinaryFile;
-if (IN_NODE) {
-  _loadBinaryFile = node_loadBinaryFile;
-} else {
-  _loadBinaryFile = browser_loadBinaryFile;
-}
-
-/**
- * Load a text file and executes it as Javascript
- * @param {str} url The path to load. May be a url or a relative file system path.
- * @private
- */
-async function nodeLoadScript(url) {
-  if (url.includes("://")) {
-    // If it's a url, load it with fetch then eval it.
-    nodeVmMod.runInThisContext(await (await nodeFetch(url)).text());
-  } else {
-    // Otherwise, hopefully it is a relative path we can load from the file
-    // system.
-    await import(nodePathMod.resolve(url));
-  }
-}
-
-/**
- * Currently loadScript is only used once to load `pyodide.asm.js`.
- * @param {string) url
- * @async
- * @private
- */
-export let loadScript;
-
-if (globalThis.document) {
-  // browser
-  loadScript = async (url) => await import(/* webpackIgnore: true */ url);
-} else if (globalThis.importScripts) {
-  // webworker
-  loadScript = async (url) => {
-    // This is async only for consistency
-    globalThis.importScripts(url);
-  };
-} else if (IN_NODE) {
-  loadScript = nodeLoadScript;
-} else {
-  throw new Error("Cannot determine runtime environment");
-}
-
 //
 // Dependency resolution
 //

src/js/pyodide.js

@@ -2,13 +2,8 @@
  * The main bootstrap code for loading pyodide.
  */
 import { Module, setStandardStreams, setHomeDirectory } from "./module.js";
-import {
-  loadScript,
-  initializePackageIndex,
-  _loadBinaryFile,
-  loadPackage,
-  initNodeModules,
-} from "./load-pyodide.js";
+import { loadScript, _loadBinaryFile, initNodeModules } from "./compat.js";
+import { initializePackageIndex, loadPackage } from "./load-package.js";
 import { makePublicAPI, registerJsModule } from "./api.js";
 import "./pyproxy.gen.js";
 import "./error_handling.gen.js";