diff --git a/docs/project/changelog.md b/docs/project/changelog.md index 54379bdc0..181c84b70 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -47,7 +47,7 @@ substitutions: Made a `PyProxy` of a Python generator into a Javascript generator: `proxy.next(val)` is translated to `gen.send(val)`. [#1180](https://github.com/pyodide/pyodide/pull/1180) -- Updated `PyProxy` so that if the wrapped Python object supports `__getitem__` +- {{ API }} Updated `PyProxy` so that if the wrapped Python object supports `__getitem__` access, then the wrapper has `get`, `set`, `has`, and `delete` methods which do `obj[key]`, `obj[key] = val`, `key in obj` and `del obj[key]` respectively. [#1175](https://github.com/pyodide/pyodide/pull/1175) @@ -56,6 +56,9 @@ substitutions: - {{ API }} Added `PyProxy.getBuffer` API to allow direct access to Python buffers as Javascript TypedArrays. [1215](https://github.com/pyodide/pyodide/pull/1215) +- {{ Enhancement }} Javascript `BigInt`s are converted into Python `int` and + Python `int`s larger than 2^53 are converted into `BigInt`. + [1407](https://github.com/pyodide/pyodide/pull/1407) ### Fixed - {{ Fix }} getattr and dir on JsProxy now report consistent results and include all diff --git a/docs/usage/type-conversions.md b/docs/usage/type-conversions.md index d33cff4c0..f28a8512b 100644 --- a/docs/usage/type-conversions.md +++ b/docs/usage/type-conversions.md @@ -45,13 +45,15 @@ is proxied into Javascript, then translation back unwraps the proxy, and the result of the round trip conversion `is` the original object (in the sense that they live at the same memory address). -Translating an object from Javascript to Python and then back to -Javascript gives an object that is `===` to the original object (with the -exception of `NaN` because `NaN !== NaN`, and of `null` which after a round trip -is converted to `undefined`). Furthermore, if the object is proxied into Python, -then translation back unwraps the proxy, and the result of the round trip -conversion is the original object (in the sense that they live at the same -memory address). +Translating an object from Javascript to Python and then back to Javascript +gives an object that is `===` to the original object. Furthermore, if the object +is proxied into Python, then translation back unwraps the proxy, and the result +of the round trip conversion is the original object (in the sense that they live +at the same memory address). There are a few exceptions: +1. `NaN` is converted to `NaN` after a round trip but `NaN !== NaN`, +2. `null` is converted to `undefined` after a round trip, and +3. a `BigInt` will be converted to a `Number` after a round trip unless its + absolute value is greater than `Number.MAX_SAFE_INTEGER` (i.e., 2^53). ## Implicit conversions @@ -65,26 +67,35 @@ the same type as the original object, we proxy `tuple` and `bytes` objects. The following immutable types are implicitly converted from Javascript to Python: -| Python | Javascript | -|-----------------|---------------------| -| `int` | `Number` | -| `float` | `Number` | -| `str` | `String` | -| `bool` | `Boolean` | -| `None` | `undefined` | +| Python | Javascript | +|-----------------|-----------------------| +| `int` | `Number` or `BigInt`* | +| `float` | `Number` | +| `str` | `String` | +| `bool` | `Boolean` | +| `None` | `undefined` | + +* An `int` is converted to a `Number` if the `int` is between -2^{53} and 2^{53} + inclusive, otherwise it is converted to a `BigInt`. (If the browser does not + support `BigInt` then a `Number` will be used instead. In this case, + conversion of large integers from Python to Javascript is lossy.) ### Javascript to Python The following immutable types are implicitly converted from Python to Javascript: -| Javascript | Python | -|-----------------|---------------------------------| -| `Number` | `int` or `float` as appropriate | -| `String` | `str` | -| `Boolean` | `bool` | -| `undefined` | `None` | -| `null` | `None` | +| Javascript | Python | +|-----------------|----------------------------------| +| `Number` | `int` or `float` as appropriate* | +| `BigInt` | `int` | +| `String` | `str` | +| `Boolean` | `bool` | +| `undefined` | `None` | +| `null` | `None` | +* A number is converted to an `int` if it is between -2^{53} and 2^{53} + inclusive and its fractional part is zero. Otherwise it is converted to a + float. ## Proxying diff --git a/src/core/hiwire.c b/src/core/hiwire.c index 622500684..09714b4ee 100644 --- a/src/core/hiwire.c +++ b/src/core/hiwire.c @@ -144,6 +144,12 @@ EM_JS_NUM(int, hiwire_init, (), { // clang-format on }; + if (globalThis.BigInt) { + Module.BigInt = BigInt; + } else { + Module.BigInt = Number; + } + /** * Determine type and endianness of data from format. This is a helper * function for converting buffers from Python to Javascript, used in @@ -274,6 +280,22 @@ EM_JS_REF(JsRef, hiwire_int, (int val), { return Module.hiwire.new_value(val); }); +EM_JS_REF(JsRef, hiwire_int_from_hex, (const char* s), { + let result; + // clang-format off + // Check if number starts with a minus sign + if (HEAP8[s] === 45) { + // clang-format on + result = -Module.BigInt(UTF8ToString(s + 1)); + } else { + result = Module.BigInt(UTF8ToString(s)); + } + if (-Number.MAX_SAFE_INTEGER < result && result < Number.MAX_SAFE_INTEGER) { + result = Number(result); + } + return Module.hiwire.new_value(result); +}); + EM_JS_REF(JsRef, hiwire_double, (double val), { return Module.hiwire.new_value(val); }); diff --git a/src/core/hiwire.h b/src/core/hiwire.h index 87f010d78..0b03c1984 100644 --- a/src/core/hiwire.h +++ b/src/core/hiwire.h @@ -59,6 +59,14 @@ extern const JsRef Js_novalue; int hiwire_init(); +/** + * Convert a string of hexadecimal digits to a Number or BigInt depending on + * whether it is less than MAX_SAFE_INTEGER or not. The string is assumed to + * begin with an optional sign followed by 0x followed by one or more digits. + */ +JsRef +hiwire_int_from_hex(const char* s); + /** * Increase the reference count on an object. * diff --git a/src/core/js2python.c b/src/core/js2python.c index 68b9bdff1..082a1c88e 100644 --- a/src/core/js2python.c +++ b/src/core/js2python.c @@ -21,17 +21,6 @@ _js2python_get_ptr(PyObject* obj) return PyUnicode_DATA(obj); } -PyObject* -_js2python_number(double val) -{ - double i; - - if (modf(val, &i) == 0.0) - return PyLong_FromDouble(i); - - return PyFloat_FromDouble(val); -} - PyObject* _js2python_none() { @@ -139,6 +128,14 @@ EM_JS_NUM(errcode, js2python_init, (), { return result; }; + Module.__js2python_bigint = function(value) + { + let ptr = stringToNewUTF8(value.toString(16)); + let result = _PyLong_FromString(ptr, 0, 16); + _free(ptr); + return result; + }; + Module.__js2python_convertImmutable = function(value) { let type = typeof value; @@ -146,7 +143,13 @@ EM_JS_NUM(errcode, js2python_init, (), { if (type === 'string') { return Module.__js2python_string(value); } else if (type === 'number') { - return __js2python_number(value); + if(Number.isSafeInteger(value)){ + return _PyLong_FromDouble(value); + } else { + return _PyFloat_FromDouble(value); + } + } else if(type === "bigint"){ + return Module.__js2python_bigint(value); } else if (value === undefined || value === null) { return __js2python_none(); } else if (value === true) { diff --git a/src/core/pyproxy.js b/src/core/pyproxy.js index 08e3a8596..a8555e387 100644 --- a/src/core/pyproxy.js +++ b/src/core/pyproxy.js @@ -70,7 +70,7 @@ TEMP_EMJS_HELPER(() => {0, /* Magic, see comment */ Module.PyProxy = { _getPtr, isPyProxy : function(jsobj) { - return jsobj && jsobj.$$ !== undefined && jsobj.$$.type === 'PyProxy'; + return !!jsobj && jsobj.$$ !== undefined && jsobj.$$.type === 'PyProxy'; }, }; diff --git a/src/core/python2js.c b/src/core/python2js.c index 5bb8eee8c..c9136ebc8 100644 --- a/src/core/python2js.c +++ b/src/core/python2js.c @@ -45,9 +45,16 @@ _python2js_long(PyObject* x) long x_long = PyLong_AsLongAndOverflow(x, &overflow); if (x_long == -1) { if (overflow) { - PyObject* py_float = PyNumber_Float(x); - FAIL_IF_NULL(py_float); - return _python2js_float(py_float); + // Backup approach for large integers: convert via hex string. + // + // Unfortunately Javascript doesn't offer a good way to convert a numbers + // to / from Uint8Arrays. + PyObject* hex_py = PyNumber_ToBase(x, 16); + FAIL_IF_NULL(hex_py); + const char* hex_str = PyUnicode_AsUTF8(hex_py); + JsRef result = hiwire_int_from_hex(hex_str); + Py_DECREF(hex_py); + return result; } FAIL_IF_ERR_OCCURRED(); } diff --git a/src/tests/test_typeconversions.py b/src/tests/test_typeconversions.py index 1ff0e0e71..fea2c3c6b 100644 --- a/src/tests/test_typeconversions.py +++ b/src/tests/test_typeconversions.py @@ -1,7 +1,7 @@ # See also test_pyproxy, test_jsproxy, and test_python. import pytest -from hypothesis import given, settings -from hypothesis.strategies import text +from hypothesis import given, settings, assume, strategies +from hypothesis.strategies import text, from_type from conftest import selenium_context_manager @@ -26,6 +26,189 @@ def test_string_conversion(selenium_module_scope, s): ) +@given( + n=strategies.one_of( + strategies.integers(min_value=-(2 ** 53), max_value=2 ** 53), + strategies.floats(allow_nan=False), + ) +) +@settings(deadline=600) +def test_number_conversions(selenium_module_scope, n): + with selenium_context_manager(selenium_module_scope) as selenium: + import json + + s = json.dumps(n) + selenium.run_js( + f""" + window.x_js = eval({s!r}); // JSON.parse apparently doesn't work + pyodide.runPython(` + import json + x_py = json.loads({s!r}) + `); + """ + ) + assert selenium.run_js(f"""return pyodide.runPython('x_py') === x_js;""") + assert selenium.run( + """ + from js import x_js + x_js == x_py + """ + ) + + +def test_nan_conversions(selenium): + selenium.run_js( + """ + window.a = NaN; + pyodide.runPython(` + from js import a + from math import isnan, nan + assert isnan(a) + `); + let b = pyodide.runPython("nan"); + if(!Number.isNaN(b)){ + throw new Error(); + } + """ + ) + + +@given(n=strategies.integers()) +@settings(deadline=600) +def test_bigint_conversions(selenium_module_scope, n): + with selenium_context_manager(selenium_module_scope) as selenium: + h = hex(n) + selenium.run_js( + """ + window.assert = function assert(cb){ + if(cb() !== true){ + throw new Error(`Assertion failed: ${cb.toString().slice(6)}`); + } + }; + """ + ) + selenium.run_js(f"window.h = {h!r};") + selenium.run_js( + """ + let negative = false; + let h2 = h; + if(h2.startsWith('-')){ + h2 = h2.slice(1); + negative = true; + } + window.n = BigInt(h2); + if(negative){ + window.n = -n; + } + pyodide.runPython(` + from js import n, h + n2 = int(h, 16) + assert n == n2 + `); + let n2 = pyodide.globals.get("n2"); + let n3 = Number(n2); + if(Number.isSafeInteger(n3)){ + assert(() => typeof n2 === "number"); + assert(() => n2 === Number(n)); + } else { + assert(() => typeof n2 === "bigint"); + assert(() => n2 === n); + } + """ + ) + + +# Generate an object of any type +@given(obj=from_type(type).flatmap(from_type)) +@settings(deadline=600) +def test_hyp_py2js2py(selenium_module_scope, obj): + with selenium_context_manager(selenium_module_scope) as selenium: + import pickle + + # When we compare x == x, there are three possible outcomes: + # 1. returns True + # 2. returns False (e.g., nan) + # 3. raises an exception + # + # Hypothesis *will* test this function on objects in case 2 and 3, so we + # have to defend against them here. + try: + assume(obj == obj) + except: + assume(False) + try: + obj_bytes = list(pickle.dumps(obj)) + except: + assume(False) + selenium.run( + f""" + import pickle + x1 = pickle.loads(bytes({obj_bytes!r})) + """ + ) + selenium.run_js( + """ + window.x2 = pyodide.globals.get("x1"); + pyodide.runPython(` + from js import x2 + if x1 != x2: + print(f"Assertion Error: {x1!r} != {x2!r}") + assert False + `); + """ + ) + + +def test_big_integer_py2js2py(selenium): + a = 9992361673228537 + selenium.run_js( + f""" + window.a = pyodide.runPython("{a}") + pyodide.runPython(` + from js import a + assert a == {a} + `); + """ + ) + a = -a + selenium.run_js( + f""" + window.a = pyodide.runPython("{a}") + pyodide.runPython(` + from js import a + assert a == {a} + `); + """ + ) + + +# Generate an object of any type +@given(obj=from_type(type).flatmap(from_type)) +@settings(deadline=600) +def test_hyp_tojs_no_crash(selenium_module_scope, obj): + with selenium_context_manager(selenium_module_scope) as selenium: + import pickle + + try: + obj_bytes = list(pickle.dumps(obj)) + except: + assume(False) + selenium.run( + f""" + import pickle + x = pickle.loads(bytes({obj_bytes!r})) + """ + ) + selenium.run_js( + """ + let x = pyodide.globals.get("x"); + if(x && x.toJs){ + x.toJs(); + } + """ + ) + + def test_python2js(selenium): assert selenium.run_js('return pyodide.runPython("None") === undefined') assert selenium.run_js('return pyodide.runPython("True") === true') @@ -38,14 +221,12 @@ def test_python2js(selenium): assert selenium.run_js('return pyodide.runPython("\'ιωδιούχο\'") === "ιωδιούχο"') assert selenium.run_js('return pyodide.runPython("\'碘化物\'") === "碘化物"') assert selenium.run_js('return pyodide.runPython("\'🐍\'") === "🐍"') - # TODO: replace with suitable test for the behavior of bytes objects once we - # get the new behavior specified. - # assert selenium.run_js( - # "let x = pyodide.runPython(\"b'bytes'\");\n" - # "return (x instanceof window.Uint8ClampedArray) && " - # "(x.length === 5) && " - # "(x[0] === 98)" - # ) + assert selenium.run_js( + "let x = pyodide.runPython(\"b'bytes'\").toJs();\n" + "return (x instanceof window.Uint8Array) && " + "(x.length === 5) && " + "(x[0] === 98)" + ) assert selenium.run_js( """ let proxy = pyodide.runPython("[1, 2, 3]"); @@ -80,6 +261,16 @@ def test_python2js_long_ints(selenium): assert selenium.run("2**32 / 2**4") == (2 ** 32 / 2 ** 4) assert selenium.run("-2**30") == -(2 ** 30) assert selenium.run("-2**31") == -(2 ** 31) + assert selenium.run_js( + """ + return pyodide.runPython("2**64") === 2n**64n; + """ + ) + assert selenium.run_js( + """ + return pyodide.runPython("-(2**64)") === -(2n**64n); + """ + ) def test_pythonexc2js(selenium):