Rooba
/
pyodide
mirror of https://github.com/pyodide/pyodide.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Allow multiple loading of Pyodide instances (#2391)

This commit is contained in:
Hood Chatham 2022-04-14 18:04:01 -07:00 committed by GitHub
parent a95a27e754
commit 993940bbb9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 122 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -2,6 +2,7 @@
*.o
*.pyc
src/js/*.gen.*
src/js/*.out.*
*.egg-info/

10
Makefile
View File

@ -40,6 +40,7 @@ dist/pyodide.asm.js: \
src/core/pyproxy.o \
src/core/python2js_buffer.o \
src/core/python2js.o \
src/js/_pyodide.out.js \
$(wildcard src/py/lib/*.py) \
$(CPYTHONLIB)/tzdata \
$(CPYTHONLIB)
@ -52,12 +53,7 @@ dist/pyodide.asm.js: \
# To show some stats on the symbols you can use the following:
# cat dist/pyodide.asm.js | grep -ohE 'var _{0,5}.' | sort | uniq -c | sort -nr | head -n 20
sed -i -E 's/var __Z[^;]*;//g' dist/pyodide.asm.js
sed -i '1i\
"use strict";\
let setImmediate = globalThis.setImmediate;\
let clearImmediate = globalThis.clearImmediate;\
let baseName, fpcGOT, dyncallGOT, fpVal, dcVal;\
' dist/pyodide.asm.js
sed -i '1i "use strict";' dist/pyodide.asm.js
# Remove last 6 lines of pyodide.asm.js, see issue #2282
# Hopefully we will remove this after emscripten fixes it, upstream issue
# emscripten-core/emscripten#16518
@ -76,7 +72,7 @@ node_modules/.installed : src/js/package.json src/js/package-lock.json
ln -sfn src/js/node_modules/ node_modules
touch node_modules/.installed
dist/pyodide.js: src/js/*.ts src/js/pyproxy.gen.ts src/js/error_handling.gen.ts node_modules/.installed
dist/pyodide.js src/js/_pyodide.out.js: src/js/*.ts src/js/pyproxy.gen.ts src/js/error_handling.gen.ts node_modules/.installed
npx rollup -c src/js/rollup.config.js
dist/pyodide.d.ts: src/js/*.ts src/js/pyproxy.gen.ts src/js/error_handling.gen.ts

3
Makefile.envs
View File

@ -110,7 +110,8 @@ export MAIN_MODULE_LDFLAGS= $(LDFLAGS_BASE) \
--exclude-file "*/test/*" \
--exclude-file "*/tests/*" \
--exclude-file "*/distutils/*" \
--pre-js src/core/pre.js
--pre-js src/core/pre.js \
--pre-js src/js/_pyodide.out.js
export SIDE_MODULE_CXXFLAGS = $(CXXFLAGS_BASE)

14
docs/project/changelog.md
View File

@ -12,6 +12,20 @@ substitutions:
# Change Log
## Unreleased
- {{ Enhancement }} We now put our built files into the `dist` directory rather
than the `build` directory. {pr}`2387`
- {{ Enhancement }} `loadPyodide` no longer uses any global state, so it can be
used more than once in the same thread. This is recommended if a network
request causes a loading failure, if there is a fatal error, if you damage the
state of the runtime so badly that it is no longer usable, or for certain
testing purposes. It is not recommended for creating multiple execution
environments, for which you should use
`pyodide.runPython(code, { globals : some_dict})`;
{pr}`2391`
## Version 0.20.0
[See the release notes for a summary.](https://blog.pyodide.org/posts/0.20-release/)

10
src/core/error_handling.ts
View File

@ -1,5 +1,8 @@
import ErrorStackParser from "error-stack-parser";
import { Module, API, Hiwire, Tests } from "./module.js";
declare var Module: any;
declare var Hiwire: any;
declare var API: any;
declare var Tests: any;
/**
* Dump the Python traceback to the browser console.
@ -192,13 +195,10 @@ Tests.convertCppException = convertCppException;
function isPyodideFrame(frame: ErrorStackParser.StackFrame): boolean {
const fileName = frame.fileName || "";
if (fileName.includes("pyodide.asm")) {
return true;
}
if (fileName.includes("wasm-function")) {
return true;
}
if (!fileName.includes("pyodide.js")) {
if (!fileName.includes("pyodide.asm.js")) {
return false;
}
let funcName = frame.functionName || "";

6
src/core/pre.js
View File

@ -1,3 +1,5 @@
const API = Module.API;
const Hiwire = Module.hiwire;
const Tests = API.tests;
const Hiwire = {};
const Tests = {};
API.tests = Tests;
Module.hiwire = Hiwire;

4
src/core/pyproxy.ts
View File

@ -14,7 +14,9 @@
* See Makefile recipe for src/js/pyproxy.gen.ts
*/
import { Module, API, Hiwire } from "./module.js";
declare var Module: any;
declare var Hiwire: any;
declare var API: any;
// pyodide-skip

21
src/js/api.ts
View File

@ -1,8 +1,13 @@
import { Module, API, Hiwire } from "./module";
declare var Module: any;
declare var Hiwire: any;
declare var API: any;
import "./module.ts";
import { loadPackage, loadedPackages } from "./load-package";
import { isPyProxy, PyBuffer, PyProxy, TypedArray } from "./pyproxy.gen";
import { PythonError } from "./error_handling.gen";
export { loadPackage, loadedPackages, isPyProxy };
import "./error_handling.gen.js";
/**
* An alias to the Python :py:mod:`pyodide` package.
@ -31,6 +36,16 @@ export let globals: PyProxy; // actually defined in loadPyodide (see pyodide.js)
*/
export let version: string = ""; // actually defined in loadPyodide (see pyodide.js)
/**
* Just like `runPython` except uses a different globals dict and gets
* `eval_code` from `_pyodide` so that it can work before `pyodide` is imported.
* @private
*/
API.runPythonInternal = function (code: string): any {
// API.runPythonInternal_dict is initialized in finalizeBootstrap
return API._pyodide._base.eval_code(code, API.runPythonInternal_dict);
};
let runPythonPositionalGlobalsDeprecationWarned = false;
/**
* Runs a string of Python code from JavaScript, using :any:`pyodide.eval_code`
@ -463,7 +478,7 @@ export let FS: any;
/**
* @private
*/
export function makePublicAPI(): PyodideInterface {
API.makePublicAPI = function (): PyodideInterface {
FS = Module.FS;
let namespace = {
globals,
@ -492,4 +507,4 @@ export function makePublicAPI(): PyodideInterface {
API.public_api = namespace;
return namespace;
}
};

15
src/js/load-package.ts
View File

@ -1,5 +1,13 @@
import { Module, API, Tests } from "./module.js";
import { IN_NODE, nodeFsPromisesMod, _loadBinaryFile } from "./compat.js";
declare var Module: any;
declare var Tests: any;
declare var API: any;
import {
IN_NODE,
nodeFsPromisesMod,
_loadBinaryFile,
initNodeModules,
} from "./compat.js";
import { PyProxy, isPyProxy } from "./pyproxy.gen";
/** @private */
@ -15,6 +23,7 @@ export async function initializePackageIndex(indexURL: string) {
baseURL = indexURL;
let package_json;
if (IN_NODE) {
await initNodeModules();
const package_string = await nodeFsPromisesMod.readFile(
`${indexURL}packages.json`
);
@ -412,3 +421,5 @@ export async function loadPackage(
* install location for a particular ``package_name``.
*/
export let loadedPackages: { [key: string]: string } = {};
API.packageIndexReady = initializePackageIndex(API.config.indexURL);

27
src/js/module.ts
View File

@ -3,21 +3,15 @@
*
* @private
*/
export let Module: any = {};
Module.noImageDecoding = true;
Module.noAudioDecoding = true;
Module.noWasmDecoding = false; // we preload wasm using the built in plugin now
Module.preloadedWasm = {};
Module.preRun = [];
export let API: any = {};
Module.API = API;
export let Hiwire: any = {};
Module.hiwire = Hiwire;
// Put things that are exposed only for testing purposes here.
export let Tests: any = {};
API.tests = Tests;
export function createModule(): any {
let Module: any = {};
Module.noImageDecoding = true;
Module.noAudioDecoding = true;
Module.noWasmDecoding = false; // we preload wasm using the built in plugin now
Module.preloadedWasm = {};
Module.preRun = [];
return Module;
}
/**
*
@ -27,6 +21,7 @@ API.tests = Tests;
* @private
*/
export function setStandardStreams(
Module: any,
stdin?: () => string,
stdout?: (a: string) => void,
stderr?: (a: string) => void
@ -100,7 +95,7 @@ function createStdinWrapper(stdin: () => string) {
* @param path
* @private
*/
export function setHomeDirectory(path: string) {
export function setHomeDirectory(Module: any, path: string) {
Module.preRun.push(function () {
const fallbackPath = "/";
try {

60
src/js/pyodide.ts
View File

@ -2,15 +2,14 @@
* The main bootstrap code for loading pyodide.
*/
import ErrorStackParser from "error-stack-parser";
import { Module, setStandardStreams, setHomeDirectory, API } from "./module.js";
import { loadScript, _loadBinaryFile, initNodeModules } from "./compat.js";
import { initializePackageIndex, loadPackage } from "./load-package.js";
import { makePublicAPI, PyodideInterface } from "./api.js";
import "./error_handling.gen.js";
import { PyProxy, PyProxyDict } from "./pyproxy.gen";
import { createModule, setStandardStreams, setHomeDirectory } from "./module";
export {
import type { PyodideInterface } from "./api.js";
import type { PyProxy, PyProxyDict } from "./pyproxy.gen";
export type {
PyProxy,
PyProxyWithLength,
PyProxyDict,
@ -28,16 +27,6 @@ export {
export type Py2JsResult = any;
let runPythonInternal_dict: PyProxy; // Initialized in finalizeBootstrap
/**
* Just like `runPython` except uses a different globals dict and gets
* `eval_code` from `_pyodide` so that it can work before `pyodide` is imported.
* @private
*/
API.runPythonInternal = function (code: string): any {
return API._pyodide._base.eval_code(code, runPythonInternal_dict);
};
/**
* A proxy around globals that falls back to checking for a builtin if has or
* get fails to find a global with the given key. Note that this proxy is
@ -68,7 +57,7 @@ function wrapPythonGlobals(
});
}
function unpackPyodidePy(pyodide_py_tar: Uint8Array) {
function unpackPyodidePy(Module: any, pyodide_py_tar: Uint8Array) {
const fileName = "/pyodide_py.tar";
let stream = Module.FS.open(fileName, "w");
Module.FS.write(
@ -105,11 +94,11 @@ del importlib
* the core `pyodide` apis. (But package loading is not ready quite yet.)
* @private
*/
function finalizeBootstrap(config: ConfigType) {
function finalizeBootstrap(API: any, config: ConfigType) {
// First make internal dict so that we can use runPythonInternal.
// runPythonInternal uses a separate namespace, so we don't pollute the main
// environment with variables from our setup.
runPythonInternal_dict = API._pyodide._base.eval_code("{}") as PyProxy;
API.runPythonInternal_dict = API._pyodide._base.eval_code("{}") as PyProxy;
API.importlib = API.runPythonInternal("import importlib; importlib");
let import_module = API.importlib.import_module;
@ -130,7 +119,7 @@ function finalizeBootstrap(config: ConfigType) {
importhook.register_js_finder();
importhook.register_js_module("js", config.jsglobals);
let pyodide = makePublicAPI();
let pyodide = API.makePublicAPI();
importhook.register_js_module("pyodide_js", pyodide);
// import pyodide_py. We want to ensure that as much stuff as possible is
@ -183,7 +172,7 @@ function calculateIndexURL(): string {
* See documentation for loadPyodide.
* @private
*/
type ConfigType = {
export type ConfigType = {
indexURL: string;
homedir: string;
fullStdLib?: boolean;
@ -244,13 +233,9 @@ export async function loadPyodide(
jsglobals?: object;
} = {}
): Promise<PyodideInterface> {
if ((loadPyodide as any).inProgress) {
throw new Error("Pyodide is already loading.");
}
if (!options.indexURL) {
options.indexURL = calculateIndexURL();
}
(loadPyodide as any).inProgress = true;
const default_config = {
fullStdLib: true,
@ -258,21 +243,24 @@ export async function loadPyodide(
stdin: globalThis.prompt ? globalThis.prompt : undefined,
homedir: "/home/pyodide",
};
let config = Object.assign(default_config, options) as ConfigType;
const config = Object.assign(default_config, options) as ConfigType;
if (!config.indexURL.endsWith("/")) {
config.indexURL += "/";
}
await initNodeModules();
let packageIndexReady = initializePackageIndex(config.indexURL);
let pyodide_py_tar_promise = _loadBinaryFile(
const pyodide_py_tar_promise = _loadBinaryFile(
config.indexURL,
"pyodide_py.tar"
);
setStandardStreams(config.stdin, config.stdout, config.stderr);
setHomeDirectory(config.homedir);
const Module = createModule();
const API: any = { config };
Module.API = API;
let moduleLoaded = new Promise((r) => (Module.postRun = r));
setStandardStreams(Module, config.stdin, config.stdout, config.stderr);
setHomeDirectory(Module, config.homedir);
const moduleLoaded = new Promise((r) => (Module.postRun = r));
// locateFile tells Emscripten where to find the data files that initialize
// the file system.
@ -294,15 +282,15 @@ export async function loadPyodide(
};
const pyodide_py_tar = await pyodide_py_tar_promise;
unpackPyodidePy(pyodide_py_tar);
unpackPyodidePy(Module, pyodide_py_tar);
Module._pyodide_init();
let pyodide = finalizeBootstrap(config);
// Module.runPython works starting here.
const pyodide = finalizeBootstrap(API, config);
// API.runPython works starting here.
await packageIndexReady;
await API.packageIndexReady;
if (config.fullStdLib) {
await loadPackage(["distutils"]);
await pyodide.loadPackage(["distutils"]);
}
pyodide.runPython("print('Python initialization complete')");
return pyodide;

32
src/js/rollup.config.js
View File

@ -3,15 +3,12 @@ import { nodeResolve } from "@rollup/plugin-node-resolve";
import { terser } from "rollup-plugin-terser";
import ts from "rollup-plugin-ts";
function config({ input, format, minify, ext }) {
const dir = `dist/`;
// const minifierSuffix = minify ? ".min" : "";
const minifierSuffix = "";
function config({ input, output, name, format, minify }) {
return {
input: `./src/js/${input}.ts`,
output: {
name: "loadPyodide",
file: `${dir}/pyodide${minifierSuffix}.${ext}`,
file: output,
name,
format,
sourcemap: true,
},
@ -36,8 +33,23 @@ function config({ input, format, minify, ext }) {
}
export default [
// { input: "pyodide", format: "esm", minify: false, ext: "mjs" },
{ input: "pyodide", format: "esm", minify: true, ext: "mjs" },
// { input: "pyodide", format: "umd", minify: false },
{ input: "pyodide.umd", format: "umd", minify: true, ext: "js" },
{
input: "pyodide",
output: "dist/pyodide.mjs",
format: "esm",
minify: true,
},
{
input: "pyodide.umd",
output: "dist/pyodide.js",
format: "umd",
name: "loadPyodide",
minify: true,
},
{
input: "api",
output: "src/js/_pyodide.out.js",
format: "iife",
minify: true,
},
].map(config);

4
src/tests/test_pyodide.py
View File

@ -892,14 +892,14 @@ def test_js_stackframes(selenium):
["test.html", "d3"],
["test.html", "d2"],
["test.html", "d1"],
["pyodide.js", "runPython"],
["pyodide.asm.js", "runPython"],
["_pyodide/_base.py", "eval_code"],
["_pyodide/_base.py", "run"],
["<exec>", "<module>"],
["<exec>", "c2"],
["<exec>", "c1"],
["test.html", "b"],
["pyodide.js", "pyimport"],
["pyodide.asm.js", "pyimport"],
["importlib/__init__.py", "import_module"],
]
assert normalize_tb(res[: len(frames)]) == frames