Rooba
/
pyodide
mirror of https://github.com/pyodide/pyodide.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Handling for BigInt, more type conversion tests (#1407)

This commit is contained in:
Hood Chatham 2021-04-08 19:30:27 -04:00 committed by GitHub
parent b342a39722
commit 05a84ba3e9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 293 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
docs/project/changelog.md
View File

@ -47,7 +47,7 @@ substitutions:
Made a `PyProxy` of a Python generator into a Javascript generator:
`proxy.next(val)` is translated to `gen.send(val)`.
[#1180](https://github.com/pyodide/pyodide/pull/1180)
- Updated `PyProxy` so that if the wrapped Python object supports `__getitem__`
- {{ API }} Updated `PyProxy` so that if the wrapped Python object supports `__getitem__`
access, then the wrapper has `get`, `set`, `has`, and `delete` methods which do
`obj[key]`, `obj[key] = val`, `key in obj` and `del obj[key]` respectively.
[#1175](https://github.com/pyodide/pyodide/pull/1175)
@ -56,6 +56,9 @@ substitutions:
- {{ API }} Added `PyProxy.getBuffer` API to allow direct access to Python
buffers as Javascript TypedArrays.
[1215](https://github.com/pyodide/pyodide/pull/1215)
- {{ Enhancement }} Javascript `BigInt`s are converted into Python `int` and
Python `int`s larger than 2^53 are converted into `BigInt`.
[1407](https://github.com/pyodide/pyodide/pull/1407)
### Fixed
- {{ Fix }} getattr and dir on JsProxy now report consistent results and include all

33
docs/usage/type-conversions.md
View File

@ -45,13 +45,15 @@ is proxied into Javascript, then translation back unwraps the proxy, and the
result of the round trip conversion `is` the original object (in the sense that
they live at the same memory address).
Translating an object from Javascript to Python and then back to
Javascript gives an object that is `===` to the original object (with the
exception of `NaN` because `NaN !== NaN`, and of `null` which after a round trip
is converted to `undefined`). Furthermore, if the object is proxied into Python,
then translation back unwraps the proxy, and the result of the round trip
conversion is the original object (in the sense that they live at the same
memory address).
Translating an object from Javascript to Python and then back to Javascript
gives an object that is `===` to the original object. Furthermore, if the object
is proxied into Python, then translation back unwraps the proxy, and the result
of the round trip conversion is the original object (in the sense that they live
at the same memory address). There are a few exceptions:
1. `NaN` is converted to `NaN` after a round trip but `NaN !== NaN`,
2. `null` is converted to `undefined` after a round trip, and
3. a `BigInt` will be converted to a `Number` after a round trip unless its
absolute value is greater than `Number.MAX_SAFE_INTEGER` (i.e., 2^53).
## Implicit conversions
@ -66,25 +68,34 @@ The following immutable types are implicitly converted from Javascript to
Python:
| Python | Javascript |
|-----------------|---------------------|
| `int` | `Number` |
|-----------------|-----------------------|
| `int` | `Number` or `BigInt`* |
| `float` | `Number` |
| `str` | `String` |
| `bool` | `Boolean` |
| `None` | `undefined` |
* An `int` is converted to a `Number` if the `int` is between -2^{53} and 2^{53}
inclusive, otherwise it is converted to a `BigInt`. (If the browser does not
support `BigInt` then a `Number` will be used instead. In this case,
conversion of large integers from Python to Javascript is lossy.)
### Javascript to Python
The following immutable types are implicitly converted from Python to
Javascript:
| Javascript | Python |
|-----------------|---------------------------------|
| `Number` | `int` or `float` as appropriate |
|-----------------|----------------------------------|
| `Number` | `int` or `float` as appropriate* |
| `BigInt` | `int` |
| `String` | `str` |
| `Boolean` | `bool` |
| `undefined` | `None` |
| `null` | `None` |
* A number is converted to an `int` if it is between -2^{53} and 2^{53}
inclusive and its fractional part is zero. Otherwise it is converted to a
float.
## Proxying

22
src/core/hiwire.c
View File

@ -144,6 +144,12 @@ EM_JS_NUM(int, hiwire_init, (), {
// clang-format on
};
if (globalThis.BigInt) {
Module.BigInt = BigInt;
} else {
Module.BigInt = Number;
}
/**
* Determine type and endianness of data from format. This is a helper
* function for converting buffers from Python to Javascript, used in
@ -274,6 +280,22 @@ EM_JS_REF(JsRef, hiwire_int, (int val), {
return Module.hiwire.new_value(val);
});
EM_JS_REF(JsRef, hiwire_int_from_hex, (const char* s), {
let result;
// clang-format off
// Check if number starts with a minus sign
if (HEAP8[s] === 45) {
// clang-format on
result = -Module.BigInt(UTF8ToString(s + 1));
} else {
result = Module.BigInt(UTF8ToString(s));
}
if (-Number.MAX_SAFE_INTEGER < result && result < Number.MAX_SAFE_INTEGER) {
result = Number(result);
}
return Module.hiwire.new_value(result);
});
EM_JS_REF(JsRef, hiwire_double, (double val), {
return Module.hiwire.new_value(val);
});

8
src/core/hiwire.h
View File

@ -59,6 +59,14 @@ extern const JsRef Js_novalue;
int
hiwire_init();
/**
* Convert a string of hexadecimal digits to a Number or BigInt depending on
* whether it is less than MAX_SAFE_INTEGER or not. The string is assumed to
* begin with an optional sign followed by 0x followed by one or more digits.
*/
JsRef
hiwire_int_from_hex(const char* s);
/**
* Increase the reference count on an object.
*

27
src/core/js2python.c
View File

@ -21,17 +21,6 @@ _js2python_get_ptr(PyObject* obj)
return PyUnicode_DATA(obj);
}
PyObject*
_js2python_number(double val)
{
double i;
if (modf(val, &i) == 0.0)
return PyLong_FromDouble(i);
return PyFloat_FromDouble(val);
}
PyObject*
_js2python_none()
{
@ -139,6 +128,14 @@ EM_JS_NUM(errcode, js2python_init, (), {
return result;
};
Module.__js2python_bigint = function(value)
{
let ptr = stringToNewUTF8(value.toString(16));
let result = _PyLong_FromString(ptr, 0, 16);
_free(ptr);
return result;
};
Module.__js2python_convertImmutable = function(value)
{
let type = typeof value;
@ -146,7 +143,13 @@ EM_JS_NUM(errcode, js2python_init, (), {
if (type === 'string') {
return Module.__js2python_string(value);
} else if (type === 'number') {
return __js2python_number(value);
if(Number.isSafeInteger(value)){
return _PyLong_FromDouble(value);
} else {
return _PyFloat_FromDouble(value);
}
} else if(type === "bigint"){
return Module.__js2python_bigint(value);
} else if (value === undefined || value === null) {
return __js2python_none();
} else if (value === true) {

2
src/core/pyproxy.js
View File

@ -70,7 +70,7 @@ TEMP_EMJS_HELPER(() => {0, /* Magic, see comment */
Module.PyProxy = {
_getPtr,
isPyProxy : function(jsobj) {
return jsobj && jsobj.$$ !== undefined && jsobj.$$.type === 'PyProxy';
return !!jsobj && jsobj.$$ !== undefined && jsobj.$$.type === 'PyProxy';
},
};

13
src/core/python2js.c
View File

@ -45,9 +45,16 @@ _python2js_long(PyObject* x)
long x_long = PyLong_AsLongAndOverflow(x, &overflow);
if (x_long == -1) {
if (overflow) {
PyObject* py_float = PyNumber_Float(x);
FAIL_IF_NULL(py_float);
return _python2js_float(py_float);
// Backup approach for large integers: convert via hex string.
//
// Unfortunately Javascript doesn't offer a good way to convert a numbers
// to / from Uint8Arrays.
PyObject* hex_py = PyNumber_ToBase(x, 16);
FAIL_IF_NULL(hex_py);
const char* hex_str = PyUnicode_AsUTF8(hex_py);
JsRef result = hiwire_int_from_hex(hex_str);
Py_DECREF(hex_py);
return result;
}
FAIL_IF_ERR_OCCURRED();
}

211
src/tests/test_typeconversions.py
View File

@ -1,7 +1,7 @@
# See also test_pyproxy, test_jsproxy, and test_python.
import pytest
from hypothesis import given, settings
from hypothesis.strategies import text
from hypothesis import given, settings, assume, strategies
from hypothesis.strategies import text, from_type
from conftest import selenium_context_manager
@ -26,6 +26,189 @@ def test_string_conversion(selenium_module_scope, s):
)
@given(
n=strategies.one_of(
strategies.integers(min_value=-(2 ** 53), max_value=2 ** 53),
strategies.floats(allow_nan=False),
)
)
@settings(deadline=600)
def test_number_conversions(selenium_module_scope, n):
with selenium_context_manager(selenium_module_scope) as selenium:
import json
s = json.dumps(n)
selenium.run_js(
f"""
window.x_js = eval({s!r}); // JSON.parse apparently doesn't work
pyodide.runPython(`
import json
x_py = json.loads({s!r})
`);
"""
)
assert selenium.run_js(f"""return pyodide.runPython('x_py') === x_js;""")
assert selenium.run(
"""
from js import x_js
x_js == x_py
"""
)
def test_nan_conversions(selenium):
selenium.run_js(
"""
window.a = NaN;
pyodide.runPython(`
from js import a
from math import isnan, nan
assert isnan(a)
`);
let b = pyodide.runPython("nan");
if(!Number.isNaN(b)){
throw new Error();
}
"""
)
@given(n=strategies.integers())
@settings(deadline=600)
def test_bigint_conversions(selenium_module_scope, n):
with selenium_context_manager(selenium_module_scope) as selenium:
h = hex(n)
selenium.run_js(
"""
window.assert = function assert(cb){
if(cb() !== true){
throw new Error(`Assertion failed: ${cb.toString().slice(6)}`);
}
};
"""
)
selenium.run_js(f"window.h = {h!r};")
selenium.run_js(
"""
let negative = false;
let h2 = h;
if(h2.startsWith('-')){
h2 = h2.slice(1);
negative = true;
}
window.n = BigInt(h2);
if(negative){
window.n = -n;
}
pyodide.runPython(`
from js import n, h
n2 = int(h, 16)
assert n == n2
`);
let n2 = pyodide.globals.get("n2");
let n3 = Number(n2);
if(Number.isSafeInteger(n3)){
assert(() => typeof n2 === "number");
assert(() => n2 === Number(n));
} else {
assert(() => typeof n2 === "bigint");
assert(() => n2 === n);
}
"""
)
# Generate an object of any type
@given(obj=from_type(type).flatmap(from_type))
@settings(deadline=600)
def test_hyp_py2js2py(selenium_module_scope, obj):
with selenium_context_manager(selenium_module_scope) as selenium:
import pickle
# When we compare x == x, there are three possible outcomes:
# 1. returns True
# 2. returns False (e.g., nan)
# 3. raises an exception
#
# Hypothesis *will* test this function on objects in case 2 and 3, so we
# have to defend against them here.
try:
assume(obj == obj)
except:
assume(False)
try:
obj_bytes = list(pickle.dumps(obj))
except:
assume(False)
selenium.run(
f"""
import pickle
x1 = pickle.loads(bytes({obj_bytes!r}))
"""
)
selenium.run_js(
"""
window.x2 = pyodide.globals.get("x1");
pyodide.runPython(`
from js import x2
if x1 != x2:
print(f"Assertion Error: {x1!r} != {x2!r}")
assert False
`);
"""
)
def test_big_integer_py2js2py(selenium):
a = 9992361673228537
selenium.run_js(
f"""
window.a = pyodide.runPython("{a}")
pyodide.runPython(`
from js import a
assert a == {a}
`);
"""
)
a = -a
selenium.run_js(
f"""
window.a = pyodide.runPython("{a}")
pyodide.runPython(`
from js import a
assert a == {a}
`);
"""
)
# Generate an object of any type
@given(obj=from_type(type).flatmap(from_type))
@settings(deadline=600)
def test_hyp_tojs_no_crash(selenium_module_scope, obj):
with selenium_context_manager(selenium_module_scope) as selenium:
import pickle
try:
obj_bytes = list(pickle.dumps(obj))
except:
assume(False)
selenium.run(
f"""
import pickle
x = pickle.loads(bytes({obj_bytes!r}))
"""
)
selenium.run_js(
"""
let x = pyodide.globals.get("x");
if(x && x.toJs){
x.toJs();
}
"""
)
def test_python2js(selenium):
assert selenium.run_js('return pyodide.runPython("None") === undefined')
assert selenium.run_js('return pyodide.runPython("True") === true')
@ -38,14 +221,12 @@ def test_python2js(selenium):
assert selenium.run_js('return pyodide.runPython("\'ιωδιούχο\'") === "ιωδιούχο"')
assert selenium.run_js('return pyodide.runPython("\'碘化物\'") === "碘化物"')
assert selenium.run_js('return pyodide.runPython("\'🐍\'") === "🐍"')
# TODO: replace with suitable test for the behavior of bytes objects once we
# get the new behavior specified.
# assert selenium.run_js(
# "let x = pyodide.runPython(\"b'bytes'\");\n"
# "return (x instanceof window.Uint8ClampedArray) && "
# "(x.length === 5) && "
# "(x[0] === 98)"
# )
assert selenium.run_js(
"let x = pyodide.runPython(\"b'bytes'\").toJs();\n"
"return (x instanceof window.Uint8Array) && "
"(x.length === 5) && "
"(x[0] === 98)"
)
assert selenium.run_js(
"""
let proxy = pyodide.runPython("[1, 2, 3]");
@ -80,6 +261,16 @@ def test_python2js_long_ints(selenium):
assert selenium.run("2**32 / 2**4") == (2 ** 32 / 2 ** 4)
assert selenium.run("-2**30") == -(2 ** 30)
assert selenium.run("-2**31") == -(2 ** 31)
assert selenium.run_js(
"""
return pyodide.runPython("2**64") === 2n**64n;
"""
)
assert selenium.run_js(
"""
return pyodide.runPython("-(2**64)") === -(2n**64n);
"""
)
def test_pythonexc2js(selenium):