Handling for BigInt, more type conversion tests (#1407)

docs/project/changelog.md

@@ -47,7 +47,7 @@ substitutions:
   Made a `PyProxy` of a Python generator into a Javascript generator:
   `proxy.next(val)` is translated to `gen.send(val)`.
   [#1180](https://github.com/pyodide/pyodide/pull/1180)
-- Updated `PyProxy` so that if the wrapped Python object supports `__getitem__`
+- {{ API }} Updated `PyProxy` so that if the wrapped Python object supports `__getitem__`
   access, then the wrapper has `get`, `set`, `has`, and `delete` methods which do
   `obj[key]`, `obj[key] = val`, `key in obj` and `del obj[key]` respectively.
   [#1175](https://github.com/pyodide/pyodide/pull/1175)
@@ -56,6 +56,9 @@ substitutions:
 - {{ API }} Added `PyProxy.getBuffer` API to allow direct access to Python
   buffers as Javascript TypedArrays.
   [1215](https://github.com/pyodide/pyodide/pull/1215)
+- {{ Enhancement }} Javascript `BigInt`s are converted into Python `int` and
+  Python `int`s larger than 2^53 are converted into `BigInt`.
+  [1407](https://github.com/pyodide/pyodide/pull/1407)
 
 ### Fixed
 - {{ Fix }} getattr and dir on JsProxy now report consistent results and include all

docs/usage/type-conversions.md

@@ -45,13 +45,15 @@ is proxied into Javascript, then translation back unwraps the proxy, and the
 result of the round trip conversion `is` the original object (in the sense that
 they live at the same memory address).
 
-Translating an object from Javascript to Python and then back to
+Translating an object from Javascript to Python and then back to Javascript
-Javascript gives an object that is `===` to the original object (with the
+gives an object that is `===` to the original object. Furthermore, if the object
-exception of `NaN` because `NaN !== NaN`, and of `null` which after a round trip
+is proxied into Python, then translation back unwraps the proxy, and the result
-is converted to `undefined`). Furthermore, if the object is proxied into Python,
+of the round trip conversion is the original object (in the sense that they live
-then translation back unwraps the proxy, and the result of the round trip
+at the same memory address). There are a few exceptions:
-conversion is the original object (in the sense that they live at the same
+1. `NaN` is converted to `NaN` after a round trip but `NaN !== NaN`,
-memory address).
+2. `null` is converted to `undefined` after a round trip, and
+3. a `BigInt` will be converted to a `Number` after a round trip unless its
+   absolute value is greater than `Number.MAX_SAFE_INTEGER` (i.e., 2^53).
 
 ## Implicit conversions
 
@@ -65,26 +67,35 @@ the same type as the original object, we proxy `tuple` and `bytes` objects.
 The following immutable types are implicitly converted from Javascript to
 Python:
 
-| Python          | Javascript          |
+| Python          | Javascript            |
-|-----------------|---------------------|
+|-----------------|-----------------------|
-| `int`           | `Number`            |
+| `int`           | `Number` or `BigInt`* |
-| `float`         | `Number`            |
+| `float`         | `Number`              |
-| `str`           | `String`            |
+| `str`           | `String`              |
-| `bool`          | `Boolean`           |
+| `bool`          | `Boolean`             |
-| `None`          | `undefined`         |
+| `None`          | `undefined`           |
+
+* An `int` is converted to a `Number` if the `int` is between -2^{53} and 2^{53}
+  inclusive, otherwise it is converted to a `BigInt`. (If the browser does not
+  support `BigInt` then a `Number` will be used instead. In this case,
+  conversion of large integers from Python to Javascript is lossy.)
 
 ### Javascript to Python
 The following immutable types are implicitly converted from Python to
 Javascript:
 
-| Javascript      | Python                          |
+| Javascript      | Python                           |
-|-----------------|---------------------------------|
+|-----------------|----------------------------------|
-| `Number`        | `int` or `float` as appropriate |
+| `Number`        | `int` or `float` as appropriate* |
-| `String`        | `str`                           |
+| `BigInt`        | `int`                            |
-| `Boolean`       | `bool`                          |
+| `String`        | `str`                            |
-| `undefined`     | `None`                          |
+| `Boolean`       | `bool`                           |
-| `null`          | `None`                          |
+| `undefined`     | `None`                           |
+| `null`          | `None`                           |
 
+* A number is converted to an `int` if it is between -2^{53} and 2^{53}
+  inclusive and its fractional part is zero. Otherwise it is converted to a
+  float.
 
 ## Proxying
 

src/core/hiwire.c

@@ -144,6 +144,12 @@ EM_JS_NUM(int, hiwire_init, (), {
     // clang-format on
   };
 
+  if (globalThis.BigInt) {
+    Module.BigInt = BigInt;
+  } else {
+    Module.BigInt = Number;
+  }
+
   /**
    * Determine type and endianness of data from format. This is a helper
    * function for converting buffers from Python to Javascript, used in
@@ -274,6 +280,22 @@ EM_JS_REF(JsRef, hiwire_int, (int val), {
   return Module.hiwire.new_value(val);
 });
 
+EM_JS_REF(JsRef, hiwire_int_from_hex, (const char* s), {
+  let result;
+  // clang-format off
+  // Check if number starts with a minus sign
+  if (HEAP8[s] === 45) {
+    // clang-format on
+    result = -Module.BigInt(UTF8ToString(s + 1));
+  } else {
+    result = Module.BigInt(UTF8ToString(s));
+  }
+  if (-Number.MAX_SAFE_INTEGER < result && result < Number.MAX_SAFE_INTEGER) {
+    result = Number(result);
+  }
+  return Module.hiwire.new_value(result);
+});
+
 EM_JS_REF(JsRef, hiwire_double, (double val), {
   return Module.hiwire.new_value(val);
 });

src/core/hiwire.h

@@ -59,6 +59,14 @@ extern const JsRef Js_novalue;
 int
 hiwire_init();
 
+/**
+ * Convert a string of hexadecimal digits to a Number or BigInt depending on
+ * whether it is less than MAX_SAFE_INTEGER or not. The string is assumed to
+ * begin with an optional sign followed by 0x followed by one or more digits.
+ */
+JsRef
+hiwire_int_from_hex(const char* s);
+
 /**
  * Increase the reference count on an object.
  *

src/core/js2python.c

@@ -21,17 +21,6 @@ _js2python_get_ptr(PyObject* obj)
   return PyUnicode_DATA(obj);
 }
 
-PyObject*
-_js2python_number(double val)
-{
-  double i;
-
-  if (modf(val, &i) == 0.0)
-    return PyLong_FromDouble(i);
-
-  return PyFloat_FromDouble(val);
-}
-
 PyObject*
 _js2python_none()
 {
@@ -139,6 +128,14 @@ EM_JS_NUM(errcode, js2python_init, (), {
     return result;
   };
 
+  Module.__js2python_bigint = function(value)
+  {
+    let ptr = stringToNewUTF8(value.toString(16));
+    let result = _PyLong_FromString(ptr, 0, 16);
+    _free(ptr);
+    return result;
+  };
+
   Module.__js2python_convertImmutable = function(value)
   {
     let type = typeof value;
@@ -146,7 +143,13 @@ EM_JS_NUM(errcode, js2python_init, (), {
     if (type === 'string') {
       return Module.__js2python_string(value);
     } else if (type === 'number') {
-      return __js2python_number(value);
+      if(Number.isSafeInteger(value)){
+        return _PyLong_FromDouble(value);
+      } else {
+        return _PyFloat_FromDouble(value);
+      }
+    } else if(type === "bigint"){
+      return Module.__js2python_bigint(value);
     } else if (value === undefined || value === null) {
       return __js2python_none();
     } else if (value === true) {

src/core/pyproxy.js

@@ -70,7 +70,7 @@ TEMP_EMJS_HELPER(() => {0, /* Magic, see comment */
   Module.PyProxy = {
     _getPtr,
     isPyProxy : function(jsobj) {
-      return jsobj && jsobj.$$ !== undefined && jsobj.$$.type === 'PyProxy';
+      return !!jsobj && jsobj.$$ !== undefined && jsobj.$$.type === 'PyProxy';
     },
   };
 

src/core/python2js.c

@@ -45,9 +45,16 @@ _python2js_long(PyObject* x)
   long x_long = PyLong_AsLongAndOverflow(x, &overflow);
   if (x_long == -1) {
     if (overflow) {
-      PyObject* py_float = PyNumber_Float(x);
+      // Backup approach for large integers: convert via hex string.
-      FAIL_IF_NULL(py_float);
+      //
-      return _python2js_float(py_float);
+      // Unfortunately Javascript doesn't offer a good way to convert a numbers
+      // to / from Uint8Arrays.
+      PyObject* hex_py = PyNumber_ToBase(x, 16);
+      FAIL_IF_NULL(hex_py);
+      const char* hex_str = PyUnicode_AsUTF8(hex_py);
+      JsRef result = hiwire_int_from_hex(hex_str);
+      Py_DECREF(hex_py);
+      return result;
     }
     FAIL_IF_ERR_OCCURRED();
   }

src/tests/test_typeconversions.py

@@ -1,7 +1,7 @@
 # See also test_pyproxy, test_jsproxy, and test_python.
 import pytest
-from hypothesis import given, settings
+from hypothesis import given, settings, assume, strategies
-from hypothesis.strategies import text
+from hypothesis.strategies import text, from_type
 from conftest import selenium_context_manager
 
 
@@ -26,6 +26,189 @@ def test_string_conversion(selenium_module_scope, s):
         )
 
 
+@given(
+    n=strategies.one_of(
+        strategies.integers(min_value=-(2 ** 53), max_value=2 ** 53),
+        strategies.floats(allow_nan=False),
+    )
+)
+@settings(deadline=600)
+def test_number_conversions(selenium_module_scope, n):
+    with selenium_context_manager(selenium_module_scope) as selenium:
+        import json
+
+        s = json.dumps(n)
+        selenium.run_js(
+            f"""
+            window.x_js = eval({s!r}); // JSON.parse apparently doesn't work
+            pyodide.runPython(`
+                import json
+                x_py = json.loads({s!r})
+            `);
+            """
+        )
+        assert selenium.run_js(f"""return pyodide.runPython('x_py') === x_js;""")
+        assert selenium.run(
+            """
+            from js import x_js
+            x_js == x_py
+            """
+        )
+
+
+def test_nan_conversions(selenium):
+    selenium.run_js(
+        """
+        window.a = NaN;
+        pyodide.runPython(`
+            from js import a
+            from math import isnan, nan
+            assert isnan(a)
+        `);
+        let b = pyodide.runPython("nan");
+        if(!Number.isNaN(b)){
+            throw new Error();
+        }
+        """
+    )
+
+
+@given(n=strategies.integers())
+@settings(deadline=600)
+def test_bigint_conversions(selenium_module_scope, n):
+    with selenium_context_manager(selenium_module_scope) as selenium:
+        h = hex(n)
+        selenium.run_js(
+            """
+            window.assert = function assert(cb){
+                if(cb() !== true){
+                    throw new Error(`Assertion failed: ${cb.toString().slice(6)}`);
+                }
+            };
+            """
+        )
+        selenium.run_js(f"window.h = {h!r};")
+        selenium.run_js(
+            """
+            let negative = false;
+            let h2 = h;
+            if(h2.startsWith('-')){
+                h2 = h2.slice(1);
+                negative = true;
+            }
+            window.n = BigInt(h2);
+            if(negative){
+                window.n = -n;
+            }
+            pyodide.runPython(`
+                from js import n, h
+                n2 = int(h, 16)
+                assert n == n2
+            `);
+            let n2 = pyodide.globals.get("n2");
+            let n3 = Number(n2);
+            if(Number.isSafeInteger(n3)){
+                assert(() => typeof n2 === "number");
+                assert(() => n2 === Number(n));
+            } else {
+                assert(() => typeof n2 === "bigint");
+                assert(() => n2 === n);
+            }
+            """
+        )
+
+
+# Generate an object of any type
+@given(obj=from_type(type).flatmap(from_type))
+@settings(deadline=600)
+def test_hyp_py2js2py(selenium_module_scope, obj):
+    with selenium_context_manager(selenium_module_scope) as selenium:
+        import pickle
+
+        # When we compare x == x, there are three possible outcomes:
+        # 1. returns True
+        # 2. returns False (e.g., nan)
+        # 3. raises an exception
+        #
+        # Hypothesis *will* test this function on objects in case 2 and 3, so we
+        # have to defend against them here.
+        try:
+            assume(obj == obj)
+        except:
+            assume(False)
+        try:
+            obj_bytes = list(pickle.dumps(obj))
+        except:
+            assume(False)
+        selenium.run(
+            f"""
+            import pickle
+            x1 = pickle.loads(bytes({obj_bytes!r}))
+            """
+        )
+        selenium.run_js(
+            """
+            window.x2 = pyodide.globals.get("x1");
+            pyodide.runPython(`
+                from js import x2
+                if x1 != x2:
+                    print(f"Assertion Error: {x1!r} != {x2!r}")
+                    assert False
+            `);
+            """
+        )
+
+
+def test_big_integer_py2js2py(selenium):
+    a = 9992361673228537
+    selenium.run_js(
+        f"""
+        window.a = pyodide.runPython("{a}")
+        pyodide.runPython(`
+            from js import a
+            assert a == {a}
+        `);
+        """
+    )
+    a = -a
+    selenium.run_js(
+        f"""
+        window.a = pyodide.runPython("{a}")
+        pyodide.runPython(`
+            from js import a
+            assert a == {a}
+        `);
+        """
+    )
+
+
+# Generate an object of any type
+@given(obj=from_type(type).flatmap(from_type))
+@settings(deadline=600)
+def test_hyp_tojs_no_crash(selenium_module_scope, obj):
+    with selenium_context_manager(selenium_module_scope) as selenium:
+        import pickle
+
+        try:
+            obj_bytes = list(pickle.dumps(obj))
+        except:
+            assume(False)
+        selenium.run(
+            f"""
+            import pickle
+            x = pickle.loads(bytes({obj_bytes!r}))
+            """
+        )
+        selenium.run_js(
+            """
+            let x = pyodide.globals.get("x");
+            if(x && x.toJs){
+                x.toJs();
+            }
+            """
+        )
+
+
 def test_python2js(selenium):
     assert selenium.run_js('return pyodide.runPython("None") === undefined')
     assert selenium.run_js('return pyodide.runPython("True") === true')
@@ -38,14 +221,12 @@ def test_python2js(selenium):
     assert selenium.run_js('return pyodide.runPython("\'ιωδιούχο\'") === "ιωδιούχο"')
     assert selenium.run_js('return pyodide.runPython("\'碘化物\'") === "碘化物"')
     assert selenium.run_js('return pyodide.runPython("\'🐍\'") === "🐍"')
-    # TODO: replace with suitable test for the behavior of bytes objects once we
+    assert selenium.run_js(
-    # get the new behavior specified.
+        "let x = pyodide.runPython(\"b'bytes'\").toJs();\n"
-    # assert selenium.run_js(
+        "return (x instanceof window.Uint8Array) && "
-    #     "let x = pyodide.runPython(\"b'bytes'\");\n"
+        "(x.length === 5) && "
-    #     "return (x instanceof window.Uint8ClampedArray) && "
+        "(x[0] === 98)"
-    #     "(x.length === 5) && "
+    )
-    #     "(x[0] === 98)"
-    # )
     assert selenium.run_js(
         """
         let proxy = pyodide.runPython("[1, 2, 3]");
@@ -80,6 +261,16 @@ def test_python2js_long_ints(selenium):
     assert selenium.run("2**32 / 2**4") == (2 ** 32 / 2 ** 4)
     assert selenium.run("-2**30") == -(2 ** 30)
     assert selenium.run("-2**31") == -(2 ** 31)
+    assert selenium.run_js(
+        """
+        return pyodide.runPython("2**64") === 2n**64n;
+        """
+    )
+    assert selenium.run_js(
+        """
+        return pyodide.runPython("-(2**64)") === -(2n**64n);
+        """
+    )
 
 
 def test_pythonexc2js(selenium):