diff --git a/conftest.py b/conftest.py index be478c0f5..6e3bf3c00 100644 --- a/conftest.py +++ b/conftest.py @@ -19,6 +19,13 @@ import pytest_pyodide.runner from pytest_pyodide.utils import package_is_built as _package_is_built os.environ["IN_PYTEST"] = "1" +pytest_pyodide.runner.CHROME_FLAGS.extend( + [ + "--enable-features=WebAssemblyExperimentalJSPI", + "--enable-experimental-webassembly-features", + ] +) +pytest_pyodide.runner.NODE_FLAGS.extend(["--experimental-wasm-stack-switching"]) # There are a bunch of global objects that occasionally enter the hiwire cache # but never leave. The refcount checks get angry about them if they aren't preloaded. diff --git a/docs/project/changelog.md b/docs/project/changelog.md index d69fa3292..83d2c6d69 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -16,6 +16,9 @@ myst: ## Unreleased +- {{ Enhancement }} Added experimental support for stack switching. + {pr}`3957`, {pr}`3964`, {pr}`3987`, {pr}`3990`, {pr}`3210` + ### Packages - Added `river` version 0.19.0 {pr}`4197` diff --git a/packages/fpcast-test/fpcast-test/fpcast-test.c b/packages/fpcast-test/fpcast-test/fpcast-test.c index 807042b69..b5ffbdd06 100644 --- a/packages/fpcast-test/fpcast-test/fpcast-test.c +++ b/packages/fpcast-test/fpcast-test/fpcast-test.c @@ -31,6 +31,36 @@ set_two(PyObject* self, PyObject* value) return 0; } +/** + * This get-set pair test that the getter/setter call trampolines interact + * correctly with stack switching. They are used in `src/tests/test_syncify.py`. + * We see that the descriptor trampoline is used because the signatures don't + * take a closure argument. We assign `getset_func` to be a function that calls + * `syncify()` so that if the trampoline uses JS frames then the stack switch + * would fail. + */ +PyObject* getset_func = NULL; + +static PyObject* +get_one_call(PyObject* self) +{ + return PyObject_CallNoArgs(getset_func); +} + +static int +set_two_call(PyObject* self, PyObject* value) +{ + Py_CLEAR(getset_func); + Py_INCREF(value); + getset_func = value; + if (value == Py_None) { + return 0; + } + PyObject* result = PyObject_CallNoArgs(value); + Py_XDECREF(result); + return result ? 0 : -1; +} + // These two structs are the same but it's important that they have to be // duplicated here or else we miss test coverage. static PyMethodDef Test_Functions[] = { @@ -72,6 +102,9 @@ static PyMethodDef Test_Methods[] = { static PyGetSetDef Test_GetSet[] = { { "getset0", .get = (getter)zero }, { "getset1", .get = (getter)one, .set = (setter)set_two }, + { "getset_jspi_test", + .get = (getter)get_one_call, + .set = (setter)set_two_call }, { NULL } }; diff --git a/src/core/hiwire.c b/src/core/hiwire.c index 807866628..32e42521f 100644 --- a/src/core/hiwire.c +++ b/src/core/hiwire.c @@ -294,6 +294,32 @@ EM_JS_REF(JsRef, }); // clang-format on +// Either syncifyHandler will get filled in by stack_switching/suspenders.mjs or +// stack switching is not available so syncify will always return an error in +// JsProxy.c and syncifyHandler will never be called. +JsRef (*syncifyHandler)(JsRef idpromise) = NULL; + +EM_JS(void, hiwire_syncify_handle_error, (void), { + if (!Module.syncify_error) { + // In this case we tried to syncify in a context where there is no + // suspender. JsProxy.c checks for this case and sets the error flag + // appropriately. + return; + } + Module.handle_js_error(Module.syncify_error); + delete Module.syncify_error; +}) + +JsRef +hiwire_syncify(JsRef idpromise) +{ + JsRef result = syncifyHandler(idpromise); + if (result == 0) { + hiwire_syncify_handle_error(); + } + return result; +} + EM_JS_BOOL(bool, hiwire_HasMethod, (JsRef obj_id, JsRef name), { // clang-format off let obj = Hiwire.get_value(obj_id); diff --git a/src/core/hiwire.h b/src/core/hiwire.h index c21ea0f78..8cbeba1a3 100644 --- a/src/core/hiwire.h +++ b/src/core/hiwire.h @@ -169,6 +169,12 @@ hiwire_call_va(JsRef idobj, ...); JsRef hiwire_call_bound(JsRef idfunc, JsRef idthis, JsRef idargs); +/** + * Use stack switching to get the result of the promise synchronously. + */ +JsRef +hiwire_syncify(JsRef idpromise); + bool hiwire_HasMethod(JsRef obj, JsRef name); diff --git a/src/core/jsproxy.c b/src/core/jsproxy.c index 51ed029c5..f09b8c6d2 100644 --- a/src/core/jsproxy.c +++ b/src/core/jsproxy.c @@ -2840,6 +2840,43 @@ finally: return result; } +PyObject* +JsProxy_syncify_not_supported(JsProxy* self, PyObject* Py_UNUSED(ignored)) +{ + PyErr_SetString( + PyExc_RuntimeError, + "WebAssembly stack switching not supported in this JavaScript runtime"); + return NULL; +} + +PyObject* +JsProxy_syncify(JsProxy* self, PyObject* Py_UNUSED(ignored)) +{ + JsRef jsresult = NULL; + PyObject* result = NULL; + + jsresult = hiwire_syncify(self->js); + if (jsresult == NULL) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_RuntimeError, "No suspender"); + } + FAIL(); + } + result = js2python(jsresult); + +finally: + hiwire_CLEAR(jsresult); + return result; +} + +static PyMethodDef JsProxy_syncify_MethodDef = { + "syncify", + // We select the appropriate choice between JsProxy_syncify and + // JsProxy_syncify_not_supported in JsProxy_init. + (PyCFunction)NULL, + METH_NOARGS, +}; + // clang-format off static PyNumberMethods JsProxy_NumberMethods = { .nb_bool = JsProxy_Bool @@ -3078,7 +3115,7 @@ JsMethod_Vectorcall(PyObject* self, PyObject* pyresult = NULL; // Recursion error? - FAIL_IF_NONZERO(Py_EnterRecursiveCall(" in JsMethod_Vectorcall")); + FAIL_IF_NONZERO(Py_EnterRecursiveCall(" while calling a JavaScript object")); proxies = JsArray_New(); idargs = JsMethod_ConvertArgs(args, PyVectorcall_NARGS(nargsf), kwnames, proxies); @@ -3883,6 +3920,7 @@ skip_container_slots: methods[cur_method++] = JsProxy_then_MethodDef; methods[cur_method++] = JsProxy_catch_MethodDef; methods[cur_method++] = JsProxy_finally_MethodDef; + methods[cur_method++] = JsProxy_syncify_MethodDef; } if (flags & IS_CALLABLE) { tp_flags |= Py_TPFLAGS_HAVE_VECTORCALL; @@ -4462,6 +4500,14 @@ JsProxy_init(PyObject* core_module) { bool success = false; + bool jspiSupported = EM_ASM_INT({ return Module.jspiSupported; }); + if (jspiSupported) { + JsProxy_syncify_MethodDef.ml_meth = (PyCFunction)JsProxy_syncify; + } else { + JsProxy_syncify_MethodDef.ml_meth = + (PyCFunction)JsProxy_syncify_not_supported; + } + PyObject* asyncio_module = NULL; PyObject* flag_dict = NULL; diff --git a/src/core/pre.js b/src/core/pre.js index 5888c89d5..2a7f4c085 100644 --- a/src/core/pre.js +++ b/src/core/pre.js @@ -4,6 +4,7 @@ const Tests = {}; API.tests = Tests; API.version = "0.25.0.dev0"; Module.hiwire = Hiwire; + function getTypeTag(x) { try { return Object.prototype.toString.call(x); diff --git a/src/core/pyproxy.c b/src/core/pyproxy.c index 232483de0..8ae59aec3 100644 --- a/src/core/pyproxy.c +++ b/src/core/pyproxy.c @@ -12,8 +12,12 @@ #include "pyproxy.h" #include "python2js.h" -#define Py_ENTER() _check_gil() -#define Py_EXIT() +#define Py_ENTER() \ + _check_gil(); \ + const $$s = Module.validSuspender.value; \ + Module.validSuspender.value = false; + +#define Py_EXIT() Module.validSuspender.value = $$s; EM_JS(void, throw_no_gil, (), { throw new API.NoGilError("Attempted to use PyProxy when Python GIL not held"); diff --git a/src/core/pyproxy.ts b/src/core/pyproxy.ts index 454cdf83b..3a93e84f9 100644 --- a/src/core/pyproxy.ts +++ b/src/core/pyproxy.ts @@ -163,6 +163,14 @@ type PyProxyAttrs = { }; const pyproxyAttrsSymbol = Symbol("pyproxy.attrs"); +function pyproxy_getflags(ptrobj: number) { + Py_ENTER(); + try { + return _pyproxy_getflags(ptrobj); + } finally { + Py_EXIT(); + } +} /** * Create a new PyProxy wrapping ptrobj which is a PyObject*. @@ -203,7 +211,7 @@ function pyproxy_new( // register by default gcRegister = true; } - const flags = flags_arg !== undefined ? flags_arg : _pyproxy_getflags(ptr); + const flags = flags_arg !== undefined ? flags_arg : pyproxy_getflags(ptr); if (flags === -1) { _pythonexc2js(); } @@ -508,6 +516,65 @@ Module.callPyObjectKwargs = function ( return result; }; +/** + * A version of callPyObjectKwargs that supports the JSPI. + * + * It returns a promise. Inside Python, JS promises can be syncified, which + * switches the stack to synchronously wait for them to be resolved. + * + * Pretty much everything is the same as callPyObjectKwargs except we use the + * special JSPI-friendly promisingApply wrapper of `__pyproxy_apply`. This + * causes the VM to invent a suspender and call a wrapper module which stores it + * into suspenderGlobal (for later use by hiwire_syncify). Then it calls + * _pyproxy_apply with the same arguments we gave to `promisingApply`. + */ +async function callPyObjectKwargsSuspending( + ptrobj: number, + jsargs: any, + kwargs: any, +) { + if (!Module.jspiSupported) { + throw new Error( + "WebAssembly stack switching not supported in this JavaScript runtime", + ); + } + // We don't do any checking for kwargs, checks are in PyProxy.callKwargs + // which only is used when the keyword arguments come from the user. + let num_pos_args = jsargs.length; + let kwargs_names = Object.keys(kwargs); + let kwargs_values = Object.values(kwargs); + let num_kwargs = kwargs_names.length; + jsargs.push(...kwargs_values); + + let result; + try { + Py_ENTER(); + result = await Module.promisingApply( + ptrobj, + jsargs, + num_pos_args, + kwargs_names, + num_kwargs, + ); + Py_EXIT(); + } catch (e) { + API.fatal_error(e); + } + if (result === null) { + _pythonexc2js(); + } + // Automatically schedule coroutines + if (result && result.type === "coroutine" && result._ensure_future) { + Py_ENTER(); + let is_coroutine = __iscoroutinefunction(ptrobj); + Py_EXIT(); + if (is_coroutine) { + result._ensure_future(); + } + } + return result; +} + Module.callPyObject = function (ptrobj: number, jsargs: any) { return Module.callPyObjectKwargs(ptrobj, jsargs, {}); }; @@ -2428,6 +2495,10 @@ export class PyCallableMethods { return Module.callPyObjectKwargs(_getPtr(this), jsargs, kwargs); } + callSyncifying(...jsargs: any) { + return callPyObjectKwargsSuspending(_getPtr(this), jsargs, {}); + } + /** * The ``bind()`` method creates a new function that, when called, has its * ``this`` keyword set to the provided value, with a given sequence of diff --git a/src/core/stack_switching/create_invokes.mjs b/src/core/stack_switching/create_invokes.mjs index 002f66526..5a95b1f97 100644 --- a/src/core/stack_switching/create_invokes.mjs +++ b/src/core/stack_switching/create_invokes.mjs @@ -36,7 +36,7 @@ import { * } * ``` * - * You can look in src/js/test/unit/invokes/ for a few examples of what this + * You can look at src/js/test/unit/wat/invoke_.wat for a few examples of what this * function produces. * * See diff --git a/src/core/stack_switching/esbuild.config.mjs b/src/core/stack_switching/esbuild.config.mjs index 760815cd3..eb340c575 100644 --- a/src/core/stack_switching/esbuild.config.mjs +++ b/src/core/stack_switching/esbuild.config.mjs @@ -4,10 +4,38 @@ */ import { build } from "esbuild"; +import { readFileSync } from "node:fs"; +import loadWabt from "../../js/node_modules/wabt/index.js"; import { dirname, join } from "node:path"; const __dirname = dirname(new URL(import.meta.url).pathname); +const { parseWat } = await loadWabt(); + +/** + * An esbuild plugin to handle wat imports. It uses the wasm binary toolkit to + * assemble the wat source and returns the assembled binary. + * esbuild automatically base 64 encodes/decodes the result for us. + */ +function watPlugin() { + return { + name: "watPlugin", + setup(build) { + build.onLoad({ filter: /.wat$/ }, async (args) => { + const wasmModule = parseWat( + args.path, + readFileSync(args.path, { encoding: "utf8" }), + ); + const contents = wasmModule.toBinary({}).buffer; + return { + contents, + loader: "binary", + }; + }); + }, + }; +} + const outfile = join(__dirname, "stack_switching.out.js"); const globalName = "StackSwitching"; @@ -16,6 +44,7 @@ const config = { outfile, format: "iife", bundle: true, + plugins: [watPlugin()], globalName, metafile: true, }; diff --git a/src/core/stack_switching/stack_switching.mjs b/src/core/stack_switching/stack_switching.mjs index b89dbe0cd..f832a131e 100644 --- a/src/core/stack_switching/stack_switching.mjs +++ b/src/core/stack_switching/stack_switching.mjs @@ -8,9 +8,14 @@ import { wrapException, adjustWasmImports, } from "./create_invokes.mjs"; +import { initSuspenders } from "./suspenders.mjs"; + +export { promisingApply, createPromising } from "./suspenders.mjs"; export { jsWrapperTag }; +Module.preRun.push(initSuspenders); + if (jsWrapperTag) { Module.adjustWasmImports = adjustWasmImports; Module.wrapException = wrapException; diff --git a/src/core/stack_switching/suspenders.mjs b/src/core/stack_switching/suspenders.mjs new file mode 100644 index 000000000..a6bb5a6c0 --- /dev/null +++ b/src/core/stack_switching/suspenders.mjs @@ -0,0 +1,176 @@ +import wrap_syncifying_wasm from "./wrap_syncifying.wat"; +import { + WasmModule, + CodeSection, + ImportSection, + TypeSection, +} from "./runtime_wasm.mjs"; + +/** + * Set the syncifyHandler used by hiwire_syncify. + * + * syncifyHandler does the work of hiwire_syncify (defined in hiwire). + */ +function setSyncifyHandler() { + const suspending_f = new WebAssembly.Function( + { parameters: ["externref", "i32"], results: ["i32"] }, + async (x) => { + try { + return Hiwire.new_value(await Hiwire.get_value(x)); + } catch (e) { + if (e && e.pyodide_fatal_error) { + throw e; + } + // Error handling is tricky here. We need to wait until after + // unswitching the stack to set the Python error flag. Just store the + // error for the moment. We move this into the error flag in + // hiwire_syncify_handle_error in hiwire.c + Module.syncify_error = e; + } + }, + { suspending: "first" }, + ); + // See wrap_syncifying.wat. + const module = new WebAssembly.Module(new Uint8Array(wrap_syncifying_wasm)); + const instance = new WebAssembly.Instance(module, { + e: { + s: suspenderGlobal, + i: suspending_f, + c: validSuspender, + }, + }); + // Assign to the function pointer so that hiwire_syncify calls our wrapper + // function + HEAP32[_syncifyHandler / 4] = addFunction(instance.exports.o); +} + +let promisingApplyHandler; +export function promisingApply(...args) { + // validSuspender is a flag so that we can ask for permission before trying to + // suspend. + validSuspender.value = true; + return promisingApplyHandler(...args); +} + +// for using wasm types as map keys +function wasmTypeToString(ty) { + return `params:${ty.parameters};results:${ty.results}`; +} + +/** + * This function stores the first argument into suspenderGlobal and then makes + * an onward call with one fewer argument. The suspenderGlobal is later used by + * syncify (see wrap_syncifying.wat) + * + * You can look at src/js/test/unit/wat/promising_.wat for a few examples + * of what this function produces. + */ +export function createPromisingModule(orig_type) { + const mod = new WasmModule(); + const ts = new TypeSection(); + const wrapped_type = structuredClone(orig_type); + wrapped_type.parameters.unshift("externref"); + const orig_sig = ts.addWasm(orig_type); + const wrapped_sig = ts.addWasm(wrapped_type); + mod.addSection(ts); + + const imports = new ImportSection(); + imports.addGlobal("s", "externref"); + const orig = imports.addFunction("i", orig_sig); + mod.addImportSection(imports); + mod.setExportType(wrapped_sig); + + const code = new CodeSection(); + code.local_get(0); + code.global_set(0); + for (let i = 1; i < wrapped_type.parameters.length; i++) { + code.local_get(i); + } + code.call(orig); + mod.addSection(code); + return mod.generate(); +} + +const promisingModuleMap = new Map(); +function getPromisingModule(orig_type) { + const type_str = wasmTypeToString(orig_type); + if (promisingModuleMap.has(type_str)) { + return promisingModuleMap.get(type_str); + } + const module = createPromisingModule(orig_type); + promisingModuleMap.set(type_str, module); + return module; +} + +const promisingFunctionMap = new WeakMap(); +/** + * This creates a wrapper around wasm_func that receives an extra suspender + * argument and returns a promise. The suspender is stored into suspenderGlobal + * so it can be used by syncify (see wrap_syncifying.wat) + */ +export function createPromising(wasm_func) { + if (promisingFunctionMap.has(wasm_func)) { + return promisingFunctionMap.get(wasm_func); + } + const type = WebAssembly.Function.type(wasm_func); + const module = getPromisingModule(type); + const instance = new WebAssembly.Instance(module, { + e: { i: wasm_func, s: suspenderGlobal }, + }); + const result = new WebAssembly.Function( + { parameters: type.parameters, results: ["externref"] }, + instance.exports.o, + { promising: "first" }, + ); + promisingFunctionMap.set(wasm_func, result); + return result; +} + +export let suspenderGlobal; +try { + suspenderGlobal = new WebAssembly.Global( + { value: "externref", mutable: true }, + null, + ); +} catch (e) { + // An error is thrown if externref isn't supported. In this case JSPI is also + // not supported and everything is fine. +} + +let validSuspender; + +/** + * This sets up syncify to work. + * + * We need to make: + * + * - suspenderGlobal where we store the suspender object + * + * - promisingApplyHandler which calls a Python function with stack switching + * enabled (used in callPyObjectKwargsSuspending in pyproxy.ts) + * + * - the syncifyHandler which uses suspenderGlobal to suspend execution, then + * awaits a promise, then resumes execution and returns the promise result + * (used by hiwire_syncify) + * + * If the creation of these fails because JSPI is missing, then we set it up so + * that callKwargsSyncifying and hiwire_syncify will always raise errors and + * everything else can work as normal. + */ +export function initSuspenders() { + // This is what wasm-feature-detect uses to feature detect JSPI. It is not + // 100% clear based on the text of the JSPI proposal that this will actually + // work in the future, but if it breaks we can replace it with something else + // that does work. + Module.jspiSupported = "Suspender" in WebAssembly; + + if (Module.jspiSupported) { + validSuspender = new WebAssembly.Global({ value: "i32", mutable: true }, 0); + promisingApplyHandler = createPromising(wasmExports._pyproxy_apply); + Module.validSuspender = validSuspender; + setSyncifyHandler(); + } else { + // Browser doesn't support JSPI. + Module.validSuspender = { value: 0 }; + } +} diff --git a/src/core/stack_switching/wrap_syncifying.wat b/src/core/stack_switching/wrap_syncifying.wat new file mode 100644 index 000000000..f8ad78670 --- /dev/null +++ b/src/core/stack_switching/wrap_syncifying.wat @@ -0,0 +1,24 @@ +(module + (global $suspender (import "e" "s") (mut externref)) + ;; Status flag to tell us if suspender is usable + (global $check (import "e" "c") (mut i32)) + ;; Wrapped syncify function. Expects suspender as a first argument and a + ;; JsRef to promise as second argument. Returns a JsRef to the result. + (import "e" "i" (func $syncify_promise_import (param externref i32) (result i32))) + ;; Wrapped syncify_promise that handles suspender stuff automatically so + ;; callee doesn't need to worry about it. + (func $syncify_promise_export (export "o") + (param $idpromise i32) (result i32) + (global.get $check) + (i32.eqz) + if + ;; We tried to syncify with no valid suspender, return 0. + ;; JsProxy.c checks for this case and sets Python error flag appropriately. + i32.const 0 + return + end + (global.get $suspender) + (local.get $idpromise) + (call $syncify_promise_import) ;; onwards call args are (suspender, orig argument) + ) +) diff --git a/src/js/api.ts b/src/js/api.ts index b3b56d9a7..2db578cee 100644 --- a/src/js/api.ts +++ b/src/js/api.ts @@ -282,6 +282,20 @@ export class PyodideAPI { return await API.pyodide_code.eval_code_async.callKwargs(code, options); } + static async runPythonSyncifying( + code: string, + options: { globals?: PyProxy; locals?: PyProxy } = {}, + ): Promise { + if (!options.globals) { + options.globals = API.globals; + } + return API.pyodide_code.eval_code.callSyncifying( + code, + options.globals, + options.locals, + ); + } + /** * Registers the JavaScript object ``module`` as a JavaScript module named * ``name``. This module can then be imported from Python using the standard diff --git a/src/js/package-lock.json b/src/js/package-lock.json index 3982c20ec..8379d0881 100644 --- a/src/js/package-lock.json +++ b/src/js/package-lock.json @@ -10,6 +10,7 @@ "license": "Apache-2.0", "dependencies": { "base-64": "^1.0.0", + "wabt": "^1.0.32", "ws": "^8.5.0" }, "devDependencies": { diff --git a/src/js/package.json b/src/js/package.json index 5f498cf79..11a03b246 100644 --- a/src/js/package.json +++ b/src/js/package.json @@ -77,7 +77,7 @@ }, "scripts": { "test": "npm-run-all test:*", - "test:unit": "cross-env TEST_NODE=1 ts-mocha -p tsconfig.test.json test/unit/**/*.test.*", + "test:unit": "cross-env TEST_NODE=1 ts-mocha --node-option=experimental-loader=./test/loader.mjs --node-option=experimental-wasm-stack-switching -p tsconfig.test.json test/unit/**/*.test.*", "test:node": "cross-env TEST_NODE=1 mocha test/integration/**/*.test.js", "test:browser": "mocha test/integration/**/*.test.js", "tsc": "tsc --noEmit", diff --git a/src/js/test/loader.mjs b/src/js/test/loader.mjs new file mode 100644 index 000000000..d9885b8cd --- /dev/null +++ b/src/js/test/loader.mjs @@ -0,0 +1,15 @@ +/** + * An import hook to respond to .wat imports with something degenerate. We + * don't currently unit test the functions that use .wat imports. This is good + * enough for now to keep node from crashing. + */ +export function load(url, context, nextLoad) { + if (url.endsWith(".wat")) { + return { + format: "json", + source: "null", + shortCircuit: true, + }; + } + return nextLoad(url); +} diff --git a/src/js/test/unit/stack_switching.test.mjs b/src/js/test/unit/stack_switching.test.mjs index 087a67891..0268f3adb 100644 --- a/src/js/test/unit/stack_switching.test.mjs +++ b/src/js/test/unit/stack_switching.test.mjs @@ -13,6 +13,11 @@ import { insertSectionPrefix, } from "../../../core/stack_switching/runtime_wasm.mjs"; import { createInvokeModule } from "../../../core/stack_switching/create_invokes.mjs"; +import { + createPromisingModule, + createPromising, + suspenderGlobal, +} from "../../../core/stack_switching/suspenders.mjs"; const __dirname = new URL(".", import.meta.url).pathname; @@ -88,12 +93,9 @@ function findSection(mod, section) { } } -// const source = readFileSync("../core/continuations.js", { encoding: "utf8" }); -// const script = new vm.Script(source); -// script.runInThisContext(); - // Monkey patch to prevent it from creating an actual WebAssembly.Module // as the result so we can assert on the generated bytes. +const origWasmGenerate = WasmModule.prototype.generate; WasmModule.prototype.generate = function () { return new Uint8Array(this._sections.flat()); }; @@ -298,5 +300,48 @@ describe("dynamic wasm generation code", () => { }); } }); + + describe("createPromisingModule", () => { + for (let sig of ["v", "vd", "fd", "dd", "jjjj"]) { + describe(sig, () => { + const result = createPromisingModule(emscriptenSigToWasm(sig)); + const expected = fromWatFile(`promising_${sig}.wat`); + compareModules(result, expected); + }); + } + }); + + it("createPromising", async () => { + WasmModule.prototype.generate = origWasmGenerate; + const bin = fromWat(` + (module + (global $suspender (import "e" "s") (mut externref)) + (import "e" "i" (func $i (param externref) (result i32))) + (func (export "o") (result i32) + global.get $suspender + call $i + ) + ) + `); + const mod = new WebAssembly.Module(bin); + function sleep(ms) { + return new Promise((res) => setTimeout(res, ms)); + } + async function f() { + await sleep(20); + return 7; + } + const i = new WebAssembly.Function( + { parameters: ["externref"], results: ["i32"] }, + f, + { suspending: "first" }, + ); + const inst = new WebAssembly.Instance(mod, { + e: { i, s: suspenderGlobal }, + }); + const w = createPromising(inst.exports.o, suspenderGlobal); + expect(w()).to.instanceOf(Promise); + expect(await w()).to.equal(7); + }); }); }); diff --git a/src/js/test/unit/wat/promising_dd.wat b/src/js/test/unit/wat/promising_dd.wat new file mode 100644 index 000000000..59c535a72 --- /dev/null +++ b/src/js/test/unit/wat/promising_dd.wat @@ -0,0 +1,9 @@ +(module + (import "e" "s" (global (mut externref))) + (import "e" "i" (func (param f64) (result f64))) + (func (param externref) (param f64) (result f64) + local.get 0 + global.set 0 + local.get 1 + call 0) + (export "o" (func 1))) diff --git a/src/js/test/unit/wat/promising_fd.wat b/src/js/test/unit/wat/promising_fd.wat new file mode 100644 index 000000000..51db51f7b --- /dev/null +++ b/src/js/test/unit/wat/promising_fd.wat @@ -0,0 +1,9 @@ +(module + (import "e" "s" (global (mut externref))) + (import "e" "i" (func (param f64) (result f32))) + (func (param externref) (param f64) (result f32) + local.get 0 + global.set 0 + local.get 1 + call 0) + (export "o" (func 1))) diff --git a/src/js/test/unit/wat/promising_jjjj.wat b/src/js/test/unit/wat/promising_jjjj.wat new file mode 100644 index 000000000..846831bb6 --- /dev/null +++ b/src/js/test/unit/wat/promising_jjjj.wat @@ -0,0 +1,11 @@ +(module + (import "e" "s" (global (mut externref))) + (import "e" "i" (func (param i64) (param i64) (param i64) (result i64))) + (func (param externref) (param i64) (param i64) (param i64) (result i64) + local.get 0 + global.set 0 + local.get 1 + local.get 2 + local.get 3 + call 0) + (export "o" (func 1))) diff --git a/src/js/test/unit/wat/promising_v.wat b/src/js/test/unit/wat/promising_v.wat new file mode 100644 index 000000000..24c4eb741 --- /dev/null +++ b/src/js/test/unit/wat/promising_v.wat @@ -0,0 +1,8 @@ +(module + (import "e" "s" (global (mut externref))) + (import "e" "i" (func )) + (func (param externref) + local.get 0 + global.set 0 + call 0) + (export "o" (func 1))) diff --git a/src/js/test/unit/wat/promising_vd.wat b/src/js/test/unit/wat/promising_vd.wat new file mode 100644 index 000000000..534751616 --- /dev/null +++ b/src/js/test/unit/wat/promising_vd.wat @@ -0,0 +1,9 @@ +(module + (import "e" "s" (global (mut externref))) + (import "e" "i" (func (param f64))) + (func (param externref) (param f64) + local.get 0 + global.set 0 + local.get 1 + call 0) + (export "o" (func 1))) diff --git a/src/js/test/unit/wat/promising_ve.wat b/src/js/test/unit/wat/promising_ve.wat new file mode 100644 index 000000000..05db21b23 --- /dev/null +++ b/src/js/test/unit/wat/promising_ve.wat @@ -0,0 +1,8 @@ +(module + (import "e" "s" (global (mut externref))) + (import "e" "i" (func (param f64))) + (func (param externref) (param f64) + local.get 0 + global.set 0 + call 0) + (export "o" (func 1))) diff --git a/src/py/pyodide/webloop.py b/src/py/pyodide/webloop.py index 7ebbeabfe..2611e8f7b 100644 --- a/src/py/pyodide/webloop.py +++ b/src/py/pyodide/webloop.py @@ -167,6 +167,15 @@ class PyodideFuture(Future[T]): self.add_done_callback(wrapper) return result + def syncify(self): + from .ffi import create_proxy + + p = create_proxy(self) + try: + return p.syncify() # type:ignore[attr-defined] + finally: + p.destroy() + class PyodideTask(Task[T], PyodideFuture[T]): """Inherits from both :py:class:`~asyncio.Task` and diff --git a/src/tests/test_syncify.py b/src/tests/test_syncify.py new file mode 100644 index 000000000..41795e738 --- /dev/null +++ b/src/tests/test_syncify.py @@ -0,0 +1,219 @@ +import pytest + + +def test_syncify_not_supported(selenium_standalone_noload): + selenium = selenium_standalone_noload + selenium.run_js( + """ + // Ensure that it's not supported by deleting WebAssembly.Suspender + delete WebAssembly.Suspender; + let pyodide = await loadPyodide({}); + await assertThrowsAsync( + async () => await pyodide.runPythonSyncifying("1+1"), + "Error", + "WebAssembly stack switching not supported in this JavaScript runtime" + ); + await assertThrows( + () => pyodide.runPython("from js import sleep; sleep().syncify()"), + "PythonError", + "RuntimeError: WebAssembly stack switching not supported in this JavaScript runtime" + ); + """ + ) + + +@pytest.mark.xfail_browsers(safari="No JSPI on Safari", firefox="No JSPI on firefox") +def test_syncify1(selenium): + selenium.run_js( + """ + await pyodide.runPythonSyncifying(` + from pyodide.code import run_js + + test = run_js( + ''' + (async function test() { + await sleep(1000); + return 7; + }) + ''' + ) + assert test().syncify() == 7 + del test + `); + """ + ) + + +@pytest.mark.xfail_browsers(safari="No JSPI on Safari", firefox="No JSPI on firefox") +def test_syncify2(selenium): + selenium.run_js( + """ + await pyodide.runPythonSyncifying(` + from pyodide_js import loadPackage + loadPackage("pytest").syncify() + import pytest + import importlib.metadata + with pytest.raises(ModuleNotFoundError): + importlib.metadata.version("micropip") + + from pyodide_js import loadPackage + loadPackage("micropip").syncify() + + assert importlib.metadata.version("micropip") + `); + """ + ) + + +@pytest.mark.xfail_browsers(safari="No JSPI on Safari", firefox="No JSPI on firefox") +def test_syncify_error(selenium): + selenium.run_js( + """ + await pyodide.loadPackage("pytest"); + await pyodide.runPythonSyncifying(` + def temp(): + from pyodide.code import run_js + + asyncThrow = run_js( + ''' + (async function asyncThrow(){ + throw new Error("hi"); + }) + ''' + ) + from pyodide.ffi import JsException + import pytest + with pytest.raises(JsException, match="hi"): + asyncThrow().syncify() + temp() + `); + """ + ) + + +@pytest.mark.xfail_browsers(safari="No JSPI on Safari", firefox="No JSPI on firefox") +def test_syncify_no_suspender(selenium): + selenium.run_js( + """ + await pyodide.loadPackage("pytest"); + await pyodide.runPython(` + from pyodide.code import run_js + import pytest + + test = run_js( + ''' + (async function test() { + await sleep(1000); + return 7; + }) + ''' + ) + with pytest.raises(RuntimeError, match="No suspender"): + test().syncify() + del test + `); + """ + ) + + +@pytest.mark.xfail_browsers(safari="No JSPI on Safari", firefox="No JSPI on firefox") +def test_syncify_getset(selenium): + selenium.run_js( + """ + await pyodide.loadPackage("fpcast-test") + await pyodide.runPythonSyncifying(` + def temp(): + from pyodide.code import run_js + + test = run_js( + ''' + (async function test() { + await sleep(1000); + return 7; + }) + ''' + ) + x = [] + def wrapper(): + x.append(test().syncify()) + + import fpcast_test + t = fpcast_test.TestType() + t.getset_jspi_test = wrapper + t.getset_jspi_test + t.getset_jspi_test = None + assert x == [7, 7] + temp() + `); + """ + ) + + +@pytest.mark.xfail(reason="Will fix in a followup") +def test_syncify_ctypes(): + selenium.run_js( # type: ignore[name-defined] # noqa: F821 + """ + await pyodide.runPythonSyncifying(` + from pyodide.code import run_js + + test = run_js( + ''' + (async function test() { + await sleep(1000); + return 7; + }) + ''' + ) + + def wrapper(): + return test().syncify() + from ctypes import pythonapi, py_object + pythonapi.PyObject_CallNoArgs.argtypes = [py_object] + pythonapi.PyObject_CallNoArgs.restype = py_object + assert pythonapi.PyObject_CallNoArgs(wrapper) == 7 + `); + """ + ) + + +@pytest.mark.xfail_browsers(safari="No JSPI on Safari", firefox="No JSPI on firefox") +def test_cpp_exceptions_and_syncify(selenium): + assert ( + selenium.run_js( + """ + ptr = pyodide.runPython(` + from pyodide.code import run_js + temp = run_js( + ''' + (async function temp() { + await sleep(100); + return 9; + }) + ''' + ) + + def f(): + try: + return temp().syncify() + except Exception as e: + print(e) + return -1 + id(f) + `); + + await pyodide.loadPackage("cpp-exceptions-test") + const Module = pyodide._module; + const catchlib = pyodide._module.LDSO.loadedLibsByName["/usr/lib/cpp-exceptions-test-catch.so"].exports; + async function t(x){ + Module.validSuspender.value = true; + const ptr = await Module.createPromising(catchlib.catch_call_pyobj)(x); + Module.validSuspender.value = false; + const res = Module.UTF8ToString(ptr); + Module._free(ptr); + return res; + } + return await t(ptr) + """ + ) + == "result was: 9" + )