From 704783ba916a2928093869ed652507da465e8112 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 24 Jan 2022 09:52:50 -0800 Subject: [PATCH] Split off webworker / node compatibility into separate file (#2129) --- .circleci/config.yml | 3 +- src/js/api.js | 2 +- src/js/compat.js | 113 +++++++++++++++++++ src/js/{load-pyodide.js => load-package.js} | 119 +------------------- src/js/pyodide.js | 9 +- 5 files changed, 118 insertions(+), 128 deletions(-) create mode 100644 src/js/compat.js rename src/js/{load-pyodide.js => load-package.js} (77%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1336ac75f..fb29e0a21 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -308,8 +308,7 @@ jobs: git clone https://github.com/pyodide/pyodide-webpack-example.git cd pyodide-webpack-example npm ci - cp ../src/js/load-pyodide.js node_modules/pyodide/load-pyodide.js - head -20 node_modules/pyodide/load-pyodide.js + cp ../src/js/*.js node_modules/pyodide/ npx webpack benchmark: diff --git a/src/js/api.js b/src/js/api.js index ba7858301..a2a3a3309 100644 --- a/src/js/api.js +++ b/src/js/api.js @@ -1,5 +1,5 @@ import { Module } from "./module.js"; -import { loadPackage, loadedPackages } from "./load-pyodide.js"; +import { loadPackage, loadedPackages } from "./load-package.js"; import { isPyProxy, PyBuffer } from "./pyproxy.gen.js"; export { loadPackage, loadedPackages, isPyProxy }; diff --git a/src/js/compat.js b/src/js/compat.js new file mode 100644 index 000000000..cc22d07e1 --- /dev/null +++ b/src/js/compat.js @@ -0,0 +1,113 @@ +// Detect if we're in node +export const IN_NODE = + typeof process !== "undefined" && + process.release && + process.release.name === "node" && + typeof process.browser === + "undefined"; /* This last condition checks if we run the browser shim of process */ + +export let nodePathMod; +export let nodeFetch; +export let nodeFsPromisesMod; +export let nodeVmMod; + +/** + * If we're in node, it's most convenient to import various node modules on + * initialization. Otherwise, this does nothing. + * @private + */ +export async function initNodeModules() { + if (!IN_NODE) { + return; + } + nodePathMod = (await import(/* webpackIgnore: true */ "path")).default; + nodeFsPromisesMod = await import(/* webpackIgnore: true */ "fs/promises"); + nodeFetch = (await import(/* webpackIgnore: true */ "node-fetch")).default; + nodeVmMod = (await import(/* webpackIgnore: true */ "vm")).default; +} + +/** + * Load a binary file, only for use in Node. If the path explicitly is a URL, + * then fetch from a URL, else load from the file system. + * @param {str} indexURL base path to resolve relative paths + * @param {str} path the path to load + * @returns An ArrayBuffer containing the binary data + * @private + */ +async function node_loadBinaryFile(indexURL, path) { + if (path.includes("://")) { + let response = await nodeFetch(path); + if (!response.ok) { + throw new Error(`Failed to load '${path}': request failed.`); + } + return await response.arrayBuffer(); + } else { + const data = await nodeFsPromisesMod.readFile(`${indexURL}${path}`); + return new Uint8Array(data.buffer, data.byteOffset, data.byteLength); + } +} + +/** + * Load a binary file, only for use in browser. Resolves relative paths against + * indexURL. + * + * @param {str} indexURL base path to resolve relative paths + * @param {str} path the path to load + * @returns An ArrayBuffer containing the binary data + * @private + */ +async function browser_loadBinaryFile(indexURL, path) { + const base = new URL(indexURL, location); + const url = new URL(path, base); + let response = await fetch(url); + if (!response.ok) { + throw new Error(`Failed to load '${url}': request failed.`); + } + return new Uint8Array(await response.arrayBuffer()); +} + +export let _loadBinaryFile; +if (IN_NODE) { + _loadBinaryFile = node_loadBinaryFile; +} else { + _loadBinaryFile = browser_loadBinaryFile; +} + +/** + * Currently loadScript is only used once to load `pyodide.asm.js`. + * @param {string) url + * @async + * @private + */ +export let loadScript; + +if (globalThis.document) { + // browser + loadScript = async (url) => await import(/* webpackIgnore: true */ url); +} else if (globalThis.importScripts) { + // webworker + loadScript = async (url) => { + // This is async only for consistency + globalThis.importScripts(url); + }; +} else if (IN_NODE) { + loadScript = nodeLoadScript; +} else { + throw new Error("Cannot determine runtime environment"); +} + +/** + * Load a text file and executes it as Javascript + * @param {str} url The path to load. May be a url or a relative file system path. + * @private + */ +async function nodeLoadScript(url) { + if (url.includes("://")) { + // If it's a url, load it with fetch then eval it. + nodeVmMod.runInThisContext(await (await nodeFetch(url)).text()); + } else { + // Otherwise, hopefully it is a relative path we can load from the file + // system. + await import(nodePathMod.resolve(url)); + } +} diff --git a/src/js/load-pyodide.js b/src/js/load-package.js similarity index 77% rename from src/js/load-pyodide.js rename to src/js/load-package.js index a388eb315..3bc8395c4 100644 --- a/src/js/load-pyodide.js +++ b/src/js/load-package.js @@ -1,36 +1,5 @@ import { Module } from "./module.js"; - -// -// Initialization code and node/browser shims -// - -// Detect if we're in node -const IN_NODE = - typeof process !== "undefined" && - process.release && - process.release.name === "node" && - typeof process.browser === - "undefined"; /* This last condition checks if we run the browser shim of process */ - -let nodePathMod; -let nodeFetch; -let nodeFsPromisesMod; -let nodeVmMod; - -/** - * If we're in node, it's most convenient to import various node modules on - * initialization. Otherwise, this does nothing. - * @private - */ -export async function initNodeModules() { - if (!IN_NODE) { - return; - } - nodePathMod = (await import(/* webpackIgnore: true */ "path")).default; - nodeFsPromisesMod = await import(/* webpackIgnore: true */ "fs/promises"); - nodeFetch = (await import(/* webpackIgnore: true */ "node-fetch")).default; - nodeVmMod = (await import(/* webpackIgnore: true */ "vm")).default; -} +import { IN_NODE, nodeFsPromisesMod, _loadBinaryFile } from "./compat.js"; /** @typedef {import('./pyproxy.js').PyProxy} PyProxy */ /** @private */ @@ -70,92 +39,6 @@ export async function initializePackageIndex(indexURL) { } } -/** - * Load a binary file, only for use in Node. If the path explicitly is a URL, - * then fetch from a URL, else load from the file system. - * @param {str} indexURL base path to resolve relative paths - * @param {str} path the path to load - * @returns An ArrayBuffer containing the binary data - * @private - */ -async function node_loadBinaryFile(indexURL, path) { - if (path.includes("://")) { - let response = await nodeFetch(path); - if (!response.ok) { - throw new Error(`Failed to load '${path}': request failed.`); - } - return await response.arrayBuffer(); - } else { - const data = await nodeFsPromisesMod.readFile(`${indexURL}${path}`); - return new Uint8Array(data.buffer, data.byteOffset, data.byteLength); - } -} - -/** - * Load a binary file, only for use in browser. Resolves relative paths against - * indexURL. - * - * @param {str} indexURL base path to resolve relative paths - * @param {str} path the path to load - * @returns An ArrayBuffer containing the binary data - * @private - */ -async function browser_loadBinaryFile(indexURL, path) { - const base = new URL(indexURL, location); - const url = new URL(path, base); - let response = await fetch(url); - if (!response.ok) { - throw new Error(`Failed to load '${url}': request failed.`); - } - return new Uint8Array(await response.arrayBuffer()); -} - -export let _loadBinaryFile; -if (IN_NODE) { - _loadBinaryFile = node_loadBinaryFile; -} else { - _loadBinaryFile = browser_loadBinaryFile; -} - -/** - * Load a text file and executes it as Javascript - * @param {str} url The path to load. May be a url or a relative file system path. - * @private - */ -async function nodeLoadScript(url) { - if (url.includes("://")) { - // If it's a url, load it with fetch then eval it. - nodeVmMod.runInThisContext(await (await nodeFetch(url)).text()); - } else { - // Otherwise, hopefully it is a relative path we can load from the file - // system. - await import(nodePathMod.resolve(url)); - } -} - -/** - * Currently loadScript is only used once to load `pyodide.asm.js`. - * @param {string) url - * @async - * @private - */ -export let loadScript; - -if (globalThis.document) { - // browser - loadScript = async (url) => await import(/* webpackIgnore: true */ url); -} else if (globalThis.importScripts) { - // webworker - loadScript = async (url) => { - // This is async only for consistency - globalThis.importScripts(url); - }; -} else if (IN_NODE) { - loadScript = nodeLoadScript; -} else { - throw new Error("Cannot determine runtime environment"); -} - // // Dependency resolution // diff --git a/src/js/pyodide.js b/src/js/pyodide.js index a81f88421..4a7cd0e79 100644 --- a/src/js/pyodide.js +++ b/src/js/pyodide.js @@ -2,13 +2,8 @@ * The main bootstrap code for loading pyodide. */ import { Module, setStandardStreams, setHomeDirectory } from "./module.js"; -import { - loadScript, - initializePackageIndex, - _loadBinaryFile, - loadPackage, - initNodeModules, -} from "./load-pyodide.js"; +import { loadScript, _loadBinaryFile, initNodeModules } from "./compat.js"; +import { initializePackageIndex, loadPackage } from "./load-package.js"; import { makePublicAPI, registerJsModule } from "./api.js"; import "./pyproxy.gen.js"; import "./error_handling.gen.js";