ENG create_proxy and create_once_proxy APIs (#1334)

(for reducing memory leaks)
This commit is contained in:
Hood Chatham 2021-03-19 13:30:20 -07:00 committed by GitHub
parent a5e21ba75a
commit 2d7a9f288e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 612 additions and 53 deletions

View File

@ -22,6 +22,8 @@ Backward compatibility of the API is not guaranteed at this point.
pyodide.console.repr_shorten
pyodide.console.displayhook
pyodide.webloop.WebLoop
pyodide.create_proxy
pyodide.create_once_callable
```

View File

@ -388,17 +388,37 @@ EM_JS_REF(JsRef, hiwire_dir, (JsRef idobj), {
return Module.hiwire.new_value(result);
});
static JsRef
convert_va_args(va_list args)
{
JsRef idargs = hiwire_array();
while (true) {
JsRef idarg = va_arg(args, JsRef);
if (idarg == NULL) {
break;
}
hiwire_push_array(idargs, idarg);
}
va_end(args);
return idargs;
}
EM_JS_REF(JsRef, hiwire_call, (JsRef idfunc, JsRef idargs), {
let jsfunc = Module.hiwire.get_value(idfunc);
let jsargs = Module.hiwire.get_value(idargs);
return Module.hiwire.new_value(jsfunc(... jsargs));
});
EM_JS_REF(JsRef, hiwire_call_OneArg, (JsRef idfunc, JsRef idarg), {
let jsfunc = Module.hiwire.get_value(idfunc);
let jsarg = Module.hiwire.get_value(idarg);
return Module.hiwire.new_value(jsfunc(jsarg));
});
JsRef
hiwire_call_va(JsRef idobj, ...)
{
va_list args;
va_start(args, idobj);
JsRef idargs = convert_va_args(args);
JsRef idresult = hiwire_call(idobj, idargs);
hiwire_decref(idargs);
return idresult;
}
EM_JS_REF(JsRef,
hiwire_call_bound,
@ -427,6 +447,17 @@ EM_JS_REF(JsRef,
return Module.hiwire.new_value(jsobj[jsname](... jsargs));
});
JsRef
hiwire_call_member_va(JsRef idobj, const char* ptrname, ...)
{
va_list args;
va_start(args, ptrname);
JsRef idargs = convert_va_args(args);
JsRef idresult = hiwire_call_member(idobj, ptrname, idargs);
hiwire_decref(idargs);
return idresult;
}
EM_JS_REF(JsRef, hiwire_new, (JsRef idobj, JsRef idargs), {
let jsobj = Module.hiwire.get_value(idobj);
let jsargs = Module.hiwire.get_value(idargs);

View File

@ -364,13 +364,19 @@ hiwire_dir(JsRef idobj);
*
* idargs is a hiwire Array containing the arguments.
*
* Returns: New reference
*/
JsRef
hiwire_call(JsRef idobj, JsRef idargs);
/**
* Call a function
*
* Arguments are specified as a NULL-terminated variable arguments list of
* JsRefs.
*
*/
JsRef
hiwire_call_OneArg(JsRef idfunc, JsRef idarg);
hiwire_call_va(JsRef idobj, ...);
JsRef
hiwire_call_bound(JsRef idfunc, JsRef idthis, JsRef idargs);
@ -378,15 +384,26 @@ hiwire_call_bound(JsRef idfunc, JsRef idthis, JsRef idargs);
/**
* Call a member function.
*
* ptrname is the member name, as a char * to null-terminated UTF8.
* ptrname is the member name, as a null-terminated UTF8.
*
* idargs is a hiwire Array containing the arguments.
*
* Returns: New reference
*/
JsRef
hiwire_call_member(JsRef idobj, const char* ptrname, JsRef idargs);
/**
* Call a member function.
*
* ptrname is the member name, as a null-terminated UTF8.
*
* Arguments are specified as a NULL-terminated variable arguments list of
* JsRefs.
*
*/
JsRef
hiwire_call_member_va(JsRef idobj, const char* ptrname, ...);
/**
* Calls the constructor of a class object.
*

View File

@ -34,6 +34,7 @@
#include "hiwire.h"
#include "js2python.h"
#include "jsproxy.h"
#include "pyproxy.h"
#include "python2js.h"
#include "structmember.h"
@ -625,13 +626,14 @@ JsProxy_Await(JsProxy* self, PyObject* _args)
return NULL;
}
// Main
PyObject* result = NULL;
PyObject* loop = NULL;
PyObject* fut = NULL;
PyObject* set_result = NULL;
PyObject* set_exception = NULL;
JsRef promise_id = NULL;
JsRef promise_handles = NULL;
JsRef promise_result = NULL;
PyObject* result = NULL;
loop = _PyObject_CallNoArg(asyncio_get_event_loop);
FAIL_IF_NULL(loop);
@ -644,26 +646,123 @@ JsProxy_Await(JsProxy* self, PyObject* _args)
set_exception = _PyObject_GetAttrId(fut, &PyId_set_exception);
FAIL_IF_NULL(set_exception);
JsRef promise_id = hiwire_resolve_promise(self->js);
JsRef idargs = hiwire_array();
JsRef idarg;
// TODO: does this leak set_result and set_exception? See #1006.
idarg = python2js(set_result);
hiwire_push_array(idargs, idarg);
hiwire_decref(idarg);
idarg = python2js(set_exception);
hiwire_push_array(idargs, idarg);
hiwire_decref(idarg);
hiwire_decref(hiwire_call_member(promise_id, "then", idargs));
hiwire_decref(promise_id);
hiwire_decref(idargs);
promise_id = hiwire_resolve_promise(self->js);
FAIL_IF_NULL(promise_id);
promise_handles = create_promise_handles(set_result, set_exception);
FAIL_IF_NULL(promise_handles);
promise_result = hiwire_call_member(promise_id, "then", promise_handles);
FAIL_IF_NULL(promise_result);
result = _PyObject_CallMethodId(fut, &PyId___await__, NULL);
finally:
Py_CLEAR(loop);
Py_CLEAR(fut);
Py_CLEAR(set_result);
Py_CLEAR(set_exception);
Py_DECREF(fut);
hiwire_CLEAR(promise_id);
hiwire_CLEAR(promise_handles);
hiwire_CLEAR(promise_result);
return result;
}
/**
* Overload for `then` for JsProxies with a `then` method. Of course without
* this overload, the call would just fall through to the normal `then`
* function. The advantage of this overload is that it automatically releases
* the references to the onfulfilled and onrejected callbacks, which is quite
* hard to do otherwise.
*/
PyObject*
JsProxy_then(JsProxy* self, PyObject* args, PyObject* kwds)
{
PyObject* onfulfilled = NULL;
PyObject* onrejected = NULL;
JsRef promise_handles = NULL;
JsRef result_promise = NULL;
PyObject* result = NULL;
static char* kwlist[] = { "onfulfilled", "onrejected", 0 };
if (!PyArg_ParseTupleAndKeywords(
args, kwds, "|OO:then", kwlist, &onfulfilled, &onrejected)) {
return NULL;
}
if (onfulfilled == Py_None) {
Py_CLEAR(onfulfilled);
}
if (onrejected == Py_None) {
Py_CLEAR(onrejected);
}
promise_handles = create_promise_handles(onfulfilled, onrejected);
FAIL_IF_NULL(promise_handles);
result_promise = hiwire_call_member(self->js, "then", promise_handles);
if (result_promise == NULL) {
Py_CLEAR(onfulfilled);
Py_CLEAR(onrejected);
FAIL();
}
result = JsProxy_create(result_promise);
finally:
hiwire_CLEAR(promise_handles);
hiwire_CLEAR(result_promise);
return result;
}
/**
* Overload for `catch` for JsProxies with a `then` method.
*/
PyObject*
JsProxy_catch(JsProxy* self, PyObject* onrejected)
{
JsRef promise_handles = NULL;
JsRef result_promise = NULL;
PyObject* result = NULL;
// We have to use create_promise_handles so that the handler gets released
// even if the promise resolves successfully.
promise_handles = create_promise_handles(NULL, onrejected);
FAIL_IF_NULL(promise_handles);
result_promise = hiwire_call_member(self->js, "then", promise_handles);
if (result_promise == NULL) {
Py_DECREF(onrejected);
FAIL();
}
result = JsProxy_create(result_promise);
finally:
hiwire_CLEAR(promise_handles);
hiwire_CLEAR(result_promise);
return result;
}
/**
* Overload for `finally` for JsProxies with a `then` method. This isn't
* strictly necessary since one could get the same effect by just calling
* create_once_callable on the argument, but it'd be bad to have `then` and
* `catch` handle freeing the handler automatically but require something extra
* to use `finally`.
*/
PyObject*
JsProxy_finally(JsProxy* self, PyObject* onfinally)
{
JsRef proxy = NULL;
JsRef result_promise = NULL;
PyObject* result = NULL;
// Finally method is called no matter what so we can use
// `create_once_callable`.
proxy = create_once_callable(onfinally);
FAIL_IF_NULL(proxy);
result_promise = hiwire_call_member_va(self->js, "finally", proxy, NULL);
if (result_promise == NULL) {
Py_DECREF(onfinally);
FAIL();
}
result = JsProxy_create(result_promise);
finally:
hiwire_CLEAR(proxy);
hiwire_CLEAR(result_promise);
return result;
}
@ -1068,7 +1167,7 @@ JsProxy_create_subtype(int flags)
// Make sure these stack allocations are large enough to fit!
PyType_Slot slots[20];
int cur_slot = 0;
PyMethodDef methods[5];
PyMethodDef methods[10];
int cur_method = 0;
PyMemberDef members[5];
int cur_member = 0;
@ -1141,6 +1240,35 @@ JsProxy_create_subtype(int flags)
if (flags & IS_AWAITABLE) {
slots[cur_slot++] =
(PyType_Slot){ .slot = Py_am_await, .pfunc = (void*)JsProxy_Await };
methods[cur_method++] = (PyMethodDef){
"then",
(PyCFunction)JsProxy_then,
METH_VARARGS | METH_KEYWORDS,
PyDoc_STR(
"then(onfulfilled : Callable = None, onrejected : Callable = None)"
" -> JsProxy"
"\n\n"
"The promise.then api, wrapped to manage the lifetimes of the "
"arguments"),
};
methods[cur_method++] = (PyMethodDef){
"catch",
(PyCFunction)JsProxy_catch,
METH_O,
PyDoc_STR("catch(onrejected : Callable) -> JsProxy"
"\n\n"
"The promise.catch api, wrapped to manage the lifetime of the "
"argument."),
};
methods[cur_method++] = (PyMethodDef){
"finally_",
(PyCFunction)JsProxy_finally,
METH_O,
PyDoc_STR("finally_(onrejected : Callable) -> JsProxy"
"\n\n"
"The promise.finally api, wrapped to manage the lifetime of "
"the argument."),
};
}
if (flags & IS_CALLABLE) {
tp_flags |= _Py_TPFLAGS_HAVE_VECTORCALL;

View File

@ -109,7 +109,7 @@ main(int argc, char** argv)
TRY_INIT_WITH_CORE_MODULE(error_handling);
TRY_INIT(js2python);
TRY_INIT_WITH_CORE_MODULE(JsProxy);
TRY_INIT(pyproxy);
TRY_INIT_WITH_CORE_MODULE(pyproxy);
TRY_INIT(keyboard_interrupt);
PyObject* module_dict = PyImport_GetModuleDict(); // borrowed

View File

@ -5,6 +5,7 @@
#include "hiwire.h"
#include "js2python.h"
#include "jsproxy.h"
#include "python2js.h"
_Py_IDENTIFIER(result);
@ -501,7 +502,7 @@ FutureDoneCallback_call_resolve(FutureDoneCallback* self, PyObject* result)
JsRef result_js = NULL;
JsRef output = NULL;
result_js = python2js(result);
output = hiwire_call_OneArg(self->resolve_handle, result_js);
output = hiwire_call_va(self->resolve_handle, result_js, NULL);
success = true;
finally:
@ -523,7 +524,7 @@ FutureDoneCallback_call_reject(FutureDoneCallback* self)
// wrap_exception looks up the current exception and wraps it in a Js error.
excval = wrap_exception(false);
FAIL_IF_NULL(excval);
result = hiwire_call_OneArg(self->reject_handle, excval);
result = hiwire_call_va(self->reject_handle, excval, NULL);
success = true;
finally:
@ -741,6 +742,22 @@ EM_JS_NUM(int, pyproxy_init_js, (), {
},
};
Module.callPyObject = function(ptrobj, ...jsargs) {
let idargs = Module.hiwire.new_value(jsargs);
let idresult;
try {
idresult = __pyproxy_apply(ptrobj, idargs);
} catch(e){
Module.fatal_error(e);
} finally {
Module.hiwire.decref(idargs);
}
if(idresult === 0){
_pythonexc2js();
}
return Module.hiwire.pop_value(idresult);
};
// Now a lot of boilerplate to wrap the abstract Object protocol wrappers
// above in Javascript functions.
@ -785,6 +802,12 @@ EM_JS_NUM(int, pyproxy_init_js, (), {
Module.hiwire.decref(idresult);
return result;
}
apply(jsthis, jsargs) {
return Module.callPyObject(_getPtr(this), ...jsargs);
}
call(jsthis, ...jsargs){
return Module.callPyObject(_getPtr(this), ...jsargs);
}
};
// Controlled by HAS_LENGTH, appears for any object with __len__ or sq_length
@ -1107,20 +1130,7 @@ EM_JS_NUM(int, pyproxy_init_js, (), {
return result;
},
apply: function (jsobj, jsthis, jsargs) {
let ptrobj = _getPtr(jsobj);
let idargs = Module.hiwire.new_value(jsargs);
let idresult;
try {
idresult = __pyproxy_apply(ptrobj, idargs);
} catch(e){
Module.fatal_error(e);
} finally {
Module.hiwire.decref(idargs);
}
if(idresult === 0){
_pythonexc2js();
}
return Module.hiwire.pop_value(idresult);
return jsobj.apply(jsthis, jsargs);
},
};
@ -1220,14 +1230,138 @@ EM_JS_NUM(int, pyproxy_init_js, (), {
Module.wrapNamespace = function wrapNamespace(ns){
return new Proxy(ns, NamespaceProxyHandlers);
};
return 0;
});
// clang-format on
int
pyproxy_init()
EM_JS_REF(JsRef, create_once_callable, (PyObject * obj), {
_Py_IncRef(obj);
let alreadyCalled = false;
function wrapper(... args)
{
if (alreadyCalled) {
throw new Error("OnceProxy can only be called once");
}
alreadyCalled = true;
try {
return Module.callPyObject(obj, ... args);
} finally {
_Py_DecRef(obj);
}
}
wrapper.destroy = function()
{
if (alreadyCalled) {
throw new Error("OnceProxy has already been destroyed");
}
alreadyCalled = true;
_Py_DecRef(obj);
};
return Module.hiwire.new_value(wrapper);
});
static PyObject*
create_once_callable_py(PyObject* _mod, PyObject* obj)
{
JsRef ref = create_once_callable(obj);
PyObject* result = JsProxy_create(ref);
hiwire_decref(ref);
return result;
}
// clang-format off
EM_JS_REF(JsRef, create_promise_handles, (
PyObject* handle_result, PyObject* handle_exception
), {
if (handle_result) {
_Py_IncRef(handle_result);
}
if (handle_exception) {
_Py_IncRef(handle_exception);
}
let used = false;
function checkUsed(){
if (used) {
throw new Error("One of the promise handles has already been called.");
}
}
function destroy(){
checkUsed();
used = true;
if(handle_result){
_Py_DecRef(handle_result);
}
if(handle_exception){
_Py_DecRef(handle_exception)
}
}
function onFulfilled(res) {
checkUsed();
try {
if(handle_result){
return Module.callPyObject(handle_result, res);
}
} finally {
destroy();
}
}
function onRejected(err) {
checkUsed();
try {
if(handle_exception){
return Module.callPyObject(handle_exception, err);
}
} finally {
destroy();
}
}
onFulfilled.destroy = destroy;
onRejected.destroy = destroy;
return Module.hiwire.new_value(
[onFulfilled, onRejected]
);
})
// clang-format on
static PyObject*
create_proxy(PyObject* _mod, PyObject* obj)
{
JsRef ref = pyproxy_new(obj);
PyObject* result = JsProxy_create(ref);
hiwire_decref(ref);
return result;
}
static PyMethodDef pyproxy_methods[] = {
{
"create_once_callable",
create_once_callable_py,
METH_O,
PyDoc_STR(
"create_once_callable(obj : Callable) -> JsProxy"
"\n\n"
"Wrap a Python callable in a Javascript function that can be called "
"once. After being called the proxy will decrement the reference count "
"of the Callable. The javascript function also has a `destroy` API that "
"can be used to release the proxy without calling it."),
},
{
"create_proxy",
create_proxy,
METH_O,
PyDoc_STR("create_proxy(obj : Any) -> JsProxy"
"\n\n"
"Create a `JsProxy` of a `PyProxy`. This allows explicit control "
"over the lifetime of the `PyProxy` from Python: call the "
"`destroy` API when done."),
},
{ NULL } /* Sentinel */
};
int
pyproxy_init(PyObject* core)
{
PyModule_AddFunctions(core, pyproxy_methods);
asyncio = PyImport_ImportModule("asyncio");
if (asyncio == NULL) {
return -1;

View File

@ -3,15 +3,35 @@
#define PY_SSIZE_T_CLEAN
#include "Python.h"
/** Makes Python objects usable from Javascript.
/**
* Makes Python objects usable from Javascript.
*/
// This implements the Javascript Proxy handler interface as defined here:
// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Proxy
// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Proxy
JsRef
pyproxy_new(PyObject* obj);
/**
* Wrap a Python callable in a Javascript function that can be called once.
* After being called, the reference count of the python object is automatically
* decremented. The Proxy also has a "destroy" API that can decrement the
* reference count without calling the function.
*/
JsRef
create_once_callable(PyObject* obj);
/**
* Wrap a pair of Python callables in a Javascript function that can be called
* once between the two of them. After being called, the reference counts of
* both python objects are automatically decremented. The wrappers also have a
* "destroy" API that can decrement the reference counts without calling the
* function. Intended for use with `promise.then`.
*/
JsRef
create_promise_handles(PyObject* onfulfilled, PyObject* onrejected);
int
pyproxy_init();

View File

@ -1,5 +1,5 @@
from ._base import open_url, eval_code, eval_code_async, find_imports, as_nested_list
from ._core import JsException # type: ignore
from ._core import JsException, create_once_callable, create_proxy # type: ignore
from ._importhooks import JsFinder
from .webloop import WebLoopPolicy
import asyncio
@ -26,4 +26,6 @@ __all__ = [
"JsException",
"register_js_module",
"unregister_js_module",
"create_once_callable",
"create_proxy",
]

View File

@ -1,7 +1,13 @@
import platform
from typing import Any, Callable
if platform.system() == "Emscripten":
from _pyodide_core import JsProxy, JsException
from _pyodide_core import (
JsProxy,
JsException,
create_proxy,
create_once_callable,
)
else:
# Can add shims here if we are so inclined.
class JsException(Exception): # type: ignore
@ -16,5 +22,23 @@ else:
# Defined in jsproxy.c
# Defined in jsproxy.c
__all__ = ["JsProxy", "JsException"]
def create_once_callable(obj: Callable) -> JsProxy:
"""Wrap a Python callable in a Javascript function that can be called
once. After being called the proxy will decrement the reference count
of the Callable. The javascript function also has a `destroy` API that
can be used to release the proxy without calling it.
"""
return obj
def create_proxy(obj: Any) -> JsProxy:
"""Create a `JsProxy` of a `PyProxy`.
This allows explicit control over the lifetime of the `PyProxy` from
Python: call the `destroy` API when done.
"""
return obj
__all__ = ["JsProxy", "JsException", "create_proxy", "create_once_callable"]

View File

@ -132,11 +132,12 @@ class WebLoop(asyncio.AbstractEventLoop):
This uses `setTimeout(callback, delay)`
"""
from js import setTimeout
from . import create_once_callable
if delay < 0:
raise ValueError("Can't schedule in the past")
h = asyncio.Handle(callback, args, self, context=context)
setTimeout(h._run, delay * 1000)
setTimeout(create_once_callable(h._run), delay * 1000)
return h
def call_at(

View File

@ -35,6 +35,122 @@ def test_await_jsproxy(selenium):
)
def test_then_jsproxy(selenium):
selenium.run(
"""
def prom(res, rej):
global resolve
global reject
resolve = res
reject = rej
from js import Promise
result = None
err = None
finally_occurred = False
def onfulfilled(value):
global result
result = value
def onrejected(value):
global err
err = value
def onfinally():
global finally_occurred
finally_occurred = True
"""
)
selenium.run(
"""
p = Promise.new(prom)
p.then(onfulfilled, onrejected)
resolve(10)
"""
)
time.sleep(0.01)
selenium.run(
"""
assert result == 10
assert err is None
result = None
"""
)
selenium.run(
"""
p = Promise.new(prom)
p.then(onfulfilled, onrejected)
reject(10)
"""
)
time.sleep(0.01)
selenium.run(
"""
assert result is None
assert err == 10
err = None
"""
)
selenium.run(
"""
p = Promise.new(prom)
p.catch(onrejected)
resolve(10)
"""
)
time.sleep(0.01)
selenium.run("assert err is None")
selenium.run(
"""
p = Promise.new(prom)
p.catch(onrejected)
reject(10)
"""
)
time.sleep(0.01)
selenium.run(
"""
assert err == 10
err = None
"""
)
selenium.run(
"""
p = Promise.new(prom)
p.finally_(onfinally)
resolve(10)
"""
)
time.sleep(0.01)
selenium.run(
"""
assert finally_occurred
finally_occurred = False
"""
)
selenium.run(
"""
p = Promise.new(prom)
p.finally_(onfinally)
reject(10)
"""
)
time.sleep(0.01)
selenium.run(
"""
assert finally_occurred
finally_occurred = False
"""
)
def test_await_fetch(selenium):
selenium.run(
"""

View File

@ -255,3 +255,87 @@ def test_run_python_last_exc(selenium):
`);
"""
)
def test_create_once_callable(selenium):
selenium.run_js(
"""
window.call7 = function call7(f){
return f(7);
}
pyodide.runPython(`
from pyodide import create_once_callable, JsException
from js import call7;
from unittest import TestCase
raises = TestCase().assertRaisesRegex
class Square:
def __call__(self, x):
return x*x
def __del__(self):
global destroyed
destroyed = True
f = Square()
import sys
assert sys.getrefcount(f) == 2
proxy = create_once_callable(f)
assert sys.getrefcount(f) == 3
assert call7(proxy) == 49
assert sys.getrefcount(f) == 2
with raises(JsException, "can only be called once"):
call7(proxy)
destroyed = False
del f
assert destroyed == True
del proxy # causes a fatal error =(
`);
"""
)
def test_create_proxy(selenium):
selenium.run_js(
"""
window.testAddListener = function(f){
window.listener = f;
}
window.testCallListener = function(f){
return window.listener();
}
window.testRemoveListener = function(f){
return window.listener === f;
}
pyodide.runPython(`
from pyodide import create_proxy
from js import testAddListener, testCallListener, testRemoveListener;
class Test:
def __call__(self):
return 7
def __del__(self):
global destroyed
destroyed = True
f = Test()
import sys
assert sys.getrefcount(f) == 2
proxy = create_proxy(f)
assert sys.getrefcount(f) == 3
assert proxy() == 7
testAddListener(proxy)
assert sys.getrefcount(f) == 3
assert testCallListener() == 7
assert sys.getrefcount(f) == 3
assert testCallListener() == 7
assert sys.getrefcount(f) == 3
assert testRemoveListener(f)
assert sys.getrefcount(f) == 3
proxy.destroy()
assert sys.getrefcount(f) == 2
destroyed = False
del f
assert destroyed == True
`);
"""
)