Rooba
/
pyodide
mirror of https://github.com/pyodide/pyodide.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add dict_converter parameter to toJs (#1742)

This commit is contained in:
Hood Chatham 2021-07-23 22:33:53 +00:00 committed by GitHub
parent 8493215aba
commit 2262165570
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 261 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
docs/project/changelog.md
View File

@ -108,6 +108,18 @@ substitutions:
now takes `depth` as a named argument. Also `to_js` and `to_py` only take
depth as a keyword argument.
{pr}`1721`
- {{ API }} `toJs` and `to_js` now take an option `pyproxies`, if a Javascript
Array is passed for this, then any proxies created during conversion will be
placed into this array. This allows easy cleanup later. The `create_pyproxies`
option can be used to disable creation of pyproxies during conversion
(instead a `ConversionError` is raised).
{pr}`1726`
- {{ API }} `toJs` and `to_js` now take an option `dict_converter` which will be
called on a Javascript iterable of two-element Arrays as the final step of
converting dictionaries. For instance, pass `Object.fromEntries` to convert to
an object or `Array.from` to convert to an array of pairs.
{pr}`1742`
### pyodide-build

26
src/core/pyproxy.js
View File

@ -361,14 +361,25 @@ class PyProxyClass {
* generated structure. The most common use case is to create a new empty
* list, pass the list as `pyproxies`, and then later iterate over `pyproxies`
* to destroy all of created proxies.
* @param {bool} [options.create_pyproxies] If false, `toJs` will throw a
* @param {bool} [options.create_pyproxies] If false, ``toJs`` will throw a
* ``ConversionError`` rather than producing a ``PyProxy``.
* @param {bool} [options.dict_converter] A function to be called on an
* iterable of pairs ``[key, value]``. Convert this iterable of pairs to the
* desired output. For instance, ``Object.fromEntries`` would convert the dict
* to an object, ``Array.from`` converts it to an array of entries, and ``(it) =>
* new Map(it)`` converts it to a ``Map`` (which is the default behavior).
* @return {any} The Javascript object resulting from the conversion.
*/
toJs({ depth = -1, pyproxies, create_pyproxies = true } = {}) {
toJs({
depth = -1,
pyproxies,
create_pyproxies = true,
dict_converter,
} = {}) {
let ptrobj = _getPtr(this);
let idresult;
let proxies_id;
let dict_converter_id = 0;
if (!create_pyproxies) {
proxies_id = 0;
} else if (pyproxies) {
@ -376,12 +387,21 @@ class PyProxyClass {
} else {
proxies_id = Module.hiwire.new_value([]);
}
if (dict_converter) {
dict_converter_id = Module.hiwire.new_value(dict_converter);
}
try {
idresult = Module._python2js_with_depth(ptrobj, depth, proxies_id);
idresult = Module._python2js_custom_dict_converter(
ptrobj,
depth,
proxies_id,
dict_converter_id
);
} catch (e) {
Module.fatal_error(e);
} finally {
Module.hiwire.decref(proxies_id);
Module.hiwire.decref(dict_converter_id);
}
if (idresult === 0) {
Module._pythonexc2js();

195
src/core/python2js.c
View File

@ -20,15 +20,24 @@ _python2js_immutable(PyObject* x);
int
_python2js_add_to_cache(PyObject* cache, PyObject* pyparent, JsRef jsparent);
typedef struct
struct ConversionContext_s;
typedef struct ConversionContext_s
{
PyObject* cache;
int depth;
JsRef proxies;
JsRef jscontext;
JsRef (*dict_new)(struct ConversionContext_s context);
int (*dict_add_keyvalue)(struct ConversionContext_s context,
JsRef target,
JsRef key,
JsRef value);
JsRef (*dict_postprocess)(struct ConversionContext_s context, JsRef dict);
} ConversionContext;
JsRef
_python2js(PyObject* x, ConversionContext context);
_python2js(ConversionContext context, PyObject* x);
///////////////////////////////////////////////////////////////////////////////
//
@ -112,7 +121,7 @@ _python2js_unicode(PyObject* x)
* returns NULL, we must assume that the cache has been corrupted and bail out.
*/
static JsRef
_python2js_sequence(PyObject* x, ConversionContext context)
_python2js_sequence(ConversionContext context, PyObject* x)
{
bool success = false;
PyObject* pyitem = NULL;
@ -127,7 +136,7 @@ _python2js_sequence(PyObject* x, ConversionContext context)
for (Py_ssize_t i = 0; i < length; ++i) {
PyObject* pyitem = PySequence_GetItem(x, i);
FAIL_IF_NULL(pyitem);
jsitem = _python2js(pyitem, context);
jsitem = _python2js(context, pyitem);
FAIL_IF_NULL(jsitem);
JsArray_Push(jsarray, jsitem);
Py_CLEAR(pyitem);
@ -148,7 +157,7 @@ finally:
* returns NULL, we must assume that the cache has been corrupted and bail out.
*/
static JsRef
_python2js_dict(PyObject* x, ConversionContext context)
_python2js_dict(ConversionContext context, PyObject* x)
{
bool success = false;
JsRef jskey = NULL;
@ -156,7 +165,8 @@ _python2js_dict(PyObject* x, ConversionContext context)
// result:
JsRef jsdict = NULL;
jsdict = JsMap_New();
jsdict = context.dict_new(context);
FAIL_IF_NULL(jsdict);
FAIL_IF_MINUS_ONE(_python2js_add_to_cache(context.cache, x, jsdict));
PyObject *pykey, *pyval;
Py_ssize_t pos = 0;
@ -168,12 +178,18 @@ _python2js_dict(PyObject* x, ConversionContext context)
conversion_error, "Cannot use %R as a key for a Javascript Map", pykey);
FAIL();
}
jsval = _python2js(pyval, context);
jsval = _python2js(context, pyval);
FAIL_IF_NULL(jsval);
FAIL_IF_MINUS_ONE(JsMap_Set(jsdict, jskey, jsval));
FAIL_IF_MINUS_ONE(context.dict_add_keyvalue(context, jsdict, jskey, jsval));
hiwire_CLEAR(jskey);
hiwire_CLEAR(jsval);
}
if (context.dict_postprocess) {
JsRef temp = context.dict_postprocess(context, jsdict);
FAIL_IF_NULL(temp);
hiwire_CLEAR(jsdict);
jsdict = temp;
}
success = true;
finally:
hiwire_CLEAR(jskey);
@ -195,7 +211,7 @@ finally:
* can't convert).
*/
static JsRef
_python2js_set(PyObject* x, ConversionContext context)
_python2js_set(ConversionContext context, PyObject* x)
{
bool success = false;
PyObject* iter = NULL;
@ -294,18 +310,18 @@ _python2js_proxy(PyObject* x)
* we want to convert at least the outermost layer.
*/
static JsRef
_python2js_deep(PyObject* x, ConversionContext context)
_python2js_deep(ConversionContext context, PyObject* x)
{
RETURN_IF_HAS_VALUE(_python2js_immutable(x));
RETURN_IF_HAS_VALUE(_python2js_proxy(x));
if (PyList_Check(x) || PyTuple_Check(x)) {
return _python2js_sequence(x, context);
return _python2js_sequence(context, x);
}
if (PyDict_Check(x)) {
return _python2js_dict(x, context);
return _python2js_dict(context, x);
}
if (PySet_Check(x)) {
return _python2js_set(x, context);
return _python2js_set(context, x);
}
if (PyObject_CheckBuffer(x)) {
return _python2js_buffer(x);
@ -369,7 +385,7 @@ finally:
* the cache. It leaves any real work to python2js or _python2js_deep.
*/
JsRef
_python2js(PyObject* x, ConversionContext context)
_python2js(ConversionContext context, PyObject* x)
{
PyObject* id = PyLong_FromSize_t((size_t)x);
FAIL_IF_NULL(id);
@ -383,7 +399,7 @@ _python2js(PyObject* x, ConversionContext context)
return python2js_track_proxies(x, context.proxies);
} else {
context.depth--;
return _python2js_deep(x, context);
return _python2js_deep(context, x);
}
finally:
return NULL;
@ -436,7 +452,7 @@ python2js_track_proxies(PyObject* x, JsRef proxies)
}
/**
* Do a shallow conversion from python2js. Convert immutable types with
* Do a translation from Python to Javascript. Convert immutable types with
* equivalent Javascript immutable types, but all other types are proxied.
*/
JsRef
@ -445,23 +461,32 @@ python2js(PyObject* x)
return python2js_inner(x, NULL, false);
}
// taking function pointers to EM_JS functions leads to linker errors.
static JsRef
_JsMap_New(ConversionContext context)
{
return JsMap_New();
}
static int
_JsMap_Set(ConversionContext context, JsRef map, JsRef key, JsRef value)
{
return JsMap_Set(map, key, value);
}
/**
* Do a deep conversion from Python to Javascript, converting lists, dicts, and
* sets down to depth "depth".
* Do a conversion from Python to Javascript using settings from
* ConversionContext
*/
JsRef
python2js_with_depth(PyObject* x, int depth, JsRef proxies)
python2js_with_context(ConversionContext context, PyObject* x)
{
PyObject* cache = PyDict_New();
if (cache == NULL) {
return NULL;
}
ConversionContext context = {
.cache = cache,
.depth = depth,
.proxies = proxies,
};
JsRef result = _python2js(x, context);
context.cache = cache;
JsRef result = _python2js(context, x);
// Destroy the cache. Because the cache has raw JsRefs inside, we need to
// manually dealloc them.
PyObject *pykey, *pyval;
@ -486,6 +511,91 @@ python2js_with_depth(PyObject* x, int depth, JsRef proxies)
return result;
}
/**
* Do a conversion from Python to Javascript, converting lists, dicts, and sets
* down to depth "depth".
*/
JsRef
python2js_with_depth(PyObject* x, int depth, JsRef proxies)
{
ConversionContext context = {
.depth = depth,
.proxies = proxies,
.dict_new = _JsMap_New,
.dict_add_keyvalue = _JsMap_Set,
};
return python2js_with_context(context, x);
}
static JsRef
_JsArray_New(ConversionContext context)
{
return JsArray_New();
}
EM_JS_NUM(int,
_JsArray_PushEntry_helper,
(JsRef array, JsRef key, JsRef value),
{
Module.hiwire.get_value(array).push(
[ Module.hiwire.get_value(key), Module.hiwire.get_value(value) ]);
})
static int
_JsArray_PushEntry(ConversionContext context,
JsRef array,
JsRef key,
JsRef value)
{
return _JsArray_PushEntry_helper(array, key, value);
}
EM_JS_REF(JsRef, _JsArray_PostProcess_helper, (JsRef jscontext, JsRef array), {
return Module.hiwire.new_value(
Module.hiwire.get_value(jscontext).dict_converter(
Module.hiwire.get_value(array)));
})
static JsRef
_JsArray_PostProcess(ConversionContext context, JsRef array)
{
return _JsArray_PostProcess_helper(context.jscontext, array);
}
/**
* dict_converter should be a Javascript function that converts an Iterable of
* pairs into the desired Javascript object. If dict_converter is NULL, we use
* python2js_with_depth which converts dicts to Map (the default)
*/
JsRef
python2js_custom_dict_converter(PyObject* x,
int depth,
JsRef proxies,
JsRef dict_converter)
{
if (dict_converter == NULL) {
// No custom converter provided, go back to default convertion to Map.
return python2js_with_depth(x, depth, proxies);
}
JsRef jscontext = (JsRef)EM_ASM_INT(
{
return Module.hiwire.new_value(
{ dict_converter : Module.hiwire.get_value($0) });
},
dict_converter);
ConversionContext context = {
.depth = depth,
.proxies = proxies,
.dict_new = _JsArray_New,
.dict_add_keyvalue = _JsArray_PushEntry,
.dict_postprocess = _JsArray_PostProcess,
.jscontext = jscontext,
};
JsRef result = python2js_with_context(context, x);
hiwire_CLEAR(jscontext);
return result;
}
static PyObject*
to_js(PyObject* self,
PyObject* const* args,
@ -496,20 +606,22 @@ to_js(PyObject* self,
int depth = -1;
PyObject* pyproxies = NULL;
bool create_proxies = true;
PyObject* py_dict_converter = NULL;
static const char* const _keywords[] = {
"", "depth", "pyproxies", "create_pyproxies", 0
"", "depth", "pyproxies", "create_pyproxies", "dict_converter", 0
};
// See argparse docs on format strings:
// https://docs.python.org/3/c-api/arg.html?highlight=pyarg_parse#parsing-arguments
// O|$iOp:to_js
// O - Object
// | - start of optional args
// $ - start of kwonly args
// i - signed integer
// O - Object
// p - predicate (ie bool)
// :to_js - name of this function for error messages
static struct _PyArg_Parser _parser = { "O|$iOp:to_js", _keywords, 0 };
// O|$iOpO:to_js
// O - Object
// | - start of optional args
// $ - start of kwonly args
// i - signed integer
// O - Object
// p - predicate (ie bool)
// O - Object
// :to_js - name of this function for error messages
static struct _PyArg_Parser _parser = { "O|$iOpO:to_js", _keywords, 0 };
if (kwnames != NULL && !_PyArg_ParseStackAndKeywords(args,
nargs,
kwnames,
@ -517,7 +629,8 @@ to_js(PyObject* self,
&obj,
&depth,
&pyproxies,
&create_proxies)) {
&create_proxies,
&py_dict_converter)) {
return NULL;
}
@ -530,6 +643,7 @@ to_js(PyObject* self,
return obj;
}
JsRef proxies = NULL;
JsRef js_dict_converter = NULL;
JsRef js_result = NULL;
PyObject* py_result = NULL;
@ -550,7 +664,11 @@ to_js(PyObject* self,
} else {
proxies = JsArray_New();
}
js_result = python2js_with_depth(obj, depth, proxies);
if (py_dict_converter) {
js_dict_converter = python2js(py_dict_converter);
}
js_result =
python2js_custom_dict_converter(obj, depth, proxies, js_dict_converter);
FAIL_IF_NULL(js_result);
if (hiwire_is_pyproxy(js_result)) {
// Oops, just created a PyProxy. Wrap it I guess?
@ -559,8 +677,9 @@ to_js(PyObject* self,
py_result = js2python(js_result);
}
finally:
hiwire_CLEAR(js_result);
hiwire_CLEAR(proxies);
hiwire_CLEAR(js_dict_converter);
hiwire_CLEAR(js_result);
return py_result;
}

14
src/py/_pyodide/_core_docs.py
View File

@ -1,6 +1,6 @@
# type: ignore
from typing import Any, Callable
from typing import Any, Callable, Iterable
# All docstrings for public `core` APIs should be extracted from here. We use
# the utilities in `docstring.py` and `docstring.c` to format them
@ -130,7 +130,8 @@ try:
*,
depth: int = -1,
pyproxies: JsProxy = None,
create_pyproxies: bool = True
create_pyproxies: bool = True,
dict_converter: Callable[[Iterable[JsProxy]], JsProxy] = None,
) -> JsProxy:
"""Convert the object to Javascript.
@ -160,6 +161,15 @@ try:
create_pyproxies: bool, default=True
If you set this to False, :any:`to_js` will raise an error
dict_converter: Callable[[Iterable[JsProxy]], JsProxy], defauilt = None
This converter if provided recieves a (Javascript) iterable of
(Javascript) pairs [key, value]. It is expected to return the
desired result of the dict conversion. Some suggested values for
this argument:
js.Map.new -- similar to the default behavior
js.Array.from -- convert to an array of entries
js.Object.fromEntries -- convert to a Javascript object
"""
return obj

57
src/tests/test_typeconversions.py
View File

@ -403,6 +403,63 @@ def test_wrong_way_conversions(selenium):
)
def test_dict_converter(selenium):
assert (
selenium.run_js(
"""
self.arrayFrom = Array.from;
return pyodide.runPython(`
from js import arrayFrom
from pyodide import to_js
res = to_js({ x : x + 2 for x in range(5)}, dict_converter=arrayFrom)
res
`)
"""
)
== [[0, 2], [1, 3], [2, 4], [3, 5], [4, 6]]
)
assert (
selenium.run_js(
"""
let px = pyodide.runPython("{ x : x + 2 for x in range(5)}");
let result = px.toJs({dict_converter : Array.from});
px.destroy();
return result;
"""
)
== [[0, 2], [1, 3], [2, 4], [3, 5], [4, 6]]
)
assert (
selenium.run_js(
"""
return pyodide.runPython(`
from js import Object
from pyodide import to_js
res = to_js({ x : x + 2 for x in range(5)}, dict_converter=Object.fromEntries)
res
`);
"""
)
== {"0": 2, "1": 3, "2": 4, "3": 5, "4": 6}
)
assert (
selenium.run_js(
"""
let px = pyodide.runPython("{ x : x + 2 for x in range(5)}");
let result = px.toJs({dict_converter : Object.fromEntries});
px.destroy();
return result;
"""
)
== {"0": 2, "1": 3, "2": 4, "3": 5, "4": 6}
)
selenium.run("del res; del arrayFrom; del Object")
def test_python2js_long_ints(selenium):
assert selenium.run("2**30") == 2 ** 30
assert selenium.run("2**31") == 2 ** 31