Add "wrong way" conversion functions (#1436)

conftest.py

@@ -110,6 +110,19 @@ class SeleniumWrapper:
             )
         self.driver.get(f"http://{server_hostname}:{server_port}/test.html")
         self.run_js("Error.stackTraceLimit = Infinity;", pyodide_checks=False)
+        self.run_js(
+            """
+            window.assert = function assert(cb, message=""){
+                if(message !== ""){
+                    message = "\\n" + message;
+                }
+                if(cb() !== true){
+                    throw new Error(`Assertion failed: ${cb.toString().slice(6)}${message}`);
+                }
+            };
+            """,
+            pyodide_checks=False,
+        )
         if load_pyodide:
             self.run_js("await loadPyodide({ indexURL : './'});")
             self.save_state()

docs/usage/type-conversions.md

@@ -320,7 +320,7 @@ function destroyToJsResult(x){
 }
 ```
 
-
+(type-translations-jsproxy-to-py)=
 ### Javascript to Python
 Explicit conversion of a {any}`JsProxy` into a native Python object is done with the
 {any}`JsProxy.to_py` method. By default, the `to_py` method does a recursive "deep"

src/core/docstring.c

@@ -38,6 +38,25 @@ finally:
   return success ? 0 : -1;
 }
 
+int
+add_methods_and_set_docstrings(PyObject* module,
+                               PyMethodDef* methods,
+                               PyObject* docstring_source)
+{
+  bool success = false;
+
+  int i = 0;
+  while (methods[i].ml_name != NULL) {
+    FAIL_IF_MINUS_ONE(set_method_docstring(&methods[i], docstring_source));
+    i++;
+  }
+  FAIL_IF_MINUS_ONE(PyModule_AddFunctions(module, methods));
+
+  success = true;
+finally:
+  return success ? 0 : -1;
+}
+
 int
 docstring_init()
 {

src/core/docstring.h

@@ -4,6 +4,11 @@
 int
 set_method_docstring(PyMethodDef* method, PyObject* parent);
 
+int
+add_methods_and_set_docstrings(PyObject* module,
+                               PyMethodDef* methods,
+                               PyObject* docstring_source);
+
 int
 docstring_init();
 

src/core/main.c

@@ -125,6 +125,7 @@ main(int argc, char** argv)
   TRY_INIT(hiwire);
   TRY_INIT(docstring);
   TRY_INIT(js2python);
+  TRY_INIT_WITH_CORE_MODULE(python2js);
   TRY_INIT(python2js_buffer);
   TRY_INIT_WITH_CORE_MODULE(JsProxy);
   TRY_INIT_WITH_CORE_MODULE(pyproxy);

src/core/pyproxy.c

@@ -772,69 +772,8 @@ finally:
   }
 }
 
-///////////////////////////////////////////////////////////////////////////////
-//
-// Javascript code
-//
-// The rest of the file is in Javascript. It would probably be better to move it
-// into a .js file.
-//
-
-/**
- * In the case that the Python object is callable, PyProxyClass inherits from
- * Function so that PyProxy objects can be callable.
- *
- * The following properties on a Python object will be shadowed in the proxy in
- * the case that the Python object is callable:
- *  - "arguments" and
- *  - "caller"
- *
- * Inheriting from Function has the unfortunate side effect that we MUST expose
- * the members "proxy.arguments" and "proxy.caller" because they are
- * nonconfigurable, nonwritable, nonenumerable own properties. They are just
- * always `null`.
- *
- * We also get the properties "length" and "name" which are configurable so we
- * delete them in the constructor. "prototype" is not configurable so we can't
- * delete it, however it *is* writable so we set it to be undefined. We must
- * still make "prototype in proxy" be true though.
- */
 EM_JS_REF(JsRef, pyproxy_new, (PyObject * ptrobj), {
-  // Technically, this leaks memory, since we're holding on to a reference
-  // to the proxy forever.  But we have that problem anyway since we don't
-  // have a destructor in Javascript to free the Python object.
-  // _pyproxy_destroy, which is a way for users to manually delete the proxy,
-  // also deletes the proxy from this set.
-  if (Module.PyProxies.hasOwnProperty(ptrobj)) {
-    return Module.hiwire.new_value(Module.PyProxies[ptrobj]);
-  }
-  let flags = _pyproxy_getflags(ptrobj);
-  let cls = Module.getPyProxyClass(flags);
-  // Reflect.construct calls the constructor of Module.PyProxyClass but sets the
-  // prototype as cls.prototype. This gives us a way to dynamically create
-  // subclasses of PyProxyClass (as long as we don't need to use the "new
-  // cls(ptrobj)" syntax).
-  let target;
-  if (flags & IS_CALLABLE) {
-    // To make a callable proxy, we must call the Function constructor.
-    // In this case we are effectively subclassing Function.
-    target = Reflect.construct(Function, [], cls);
-    // Remove undesireable properties added by Function constructor. Note: we
-    // can't remove "arguments" or "caller" because they are not configurable
-    // and not writable
-    delete target.length;
-    delete target.name;
-    // prototype isn't configurable so we can't delete it but it's writable.
-    target.prototype = undefined;
-  } else {
-    target = Object.create(cls.prototype);
-  }
-  Object.defineProperty(
-    target, "$$", { value : { ptr : ptrobj, type : 'PyProxy' } });
-  _Py_IncRef(ptrobj);
-  let proxy = new Proxy(target, Module.PyProxyHandlers);
-  Module.PyProxies[ptrobj] = proxy;
-  return Module.hiwire.new_value(proxy);
+  return Module.hiwire.new_value(Module.pyproxy_new(ptrobj));
 });
 
 EM_JS_REF(JsRef, create_once_callable, (PyObject * obj), {
@@ -935,7 +874,7 @@ create_proxy(PyObject* _mod, PyObject* obj)
   return result;
 }
 
-static PyMethodDef pyproxy_methods[] = {
+static PyMethodDef methods[] = {
   {
     "create_once_callable",
     create_once_callable_py,
@@ -956,17 +895,11 @@ int
 pyproxy_init(PyObject* core)
 {
   bool success = false;
-  int i = 0;
 
-  PyObject* _pyodide_core = NULL;
-  _pyodide_core = PyImport_ImportModule("_pyodide._core");
-  FAIL_IF_NULL(_pyodide_core);
-
-  while (pyproxy_methods[i].ml_name != NULL) {
-    FAIL_IF_MINUS_ONE(set_method_docstring(&pyproxy_methods[i], _pyodide_core));
-    i++;
-  }
-  FAIL_IF_MINUS_ONE(PyModule_AddFunctions(core, pyproxy_methods));
+  PyObject* docstring_source = PyImport_ImportModule("_pyodide._core");
+  FAIL_IF_NULL(docstring_source);
+  FAIL_IF_MINUS_ONE(
+    add_methods_and_set_docstrings(core, methods, docstring_source));
   asyncio = PyImport_ImportModule("asyncio");
   FAIL_IF_NULL(asyncio);
   FAIL_IF_MINUS_ONE(PyType_Ready(&FutureDoneCallbackType));
@@ -974,6 +907,6 @@ pyproxy_init(PyObject* core)
 
   success = true;
 finally:
-  Py_CLEAR(_pyodide_core);
-  return 0;
+  Py_CLEAR(docstring_source);
+  return success ? 0 : -1;
 }

src/core/pyproxy.js

@@ -5,6 +5,64 @@ JS_FILE(pyproxy_init_js, () => {0,0; /* Magic, see include_js_file.h */
   Module.PyProxies = {};
   // clang-format on
 
+  /**
+   * In the case that the Python object is callable, PyProxyClass inherits from
+   * Function so that PyProxy objects can be callable.
+   *
+   * The following properties on a Python object will be shadowed in the proxy
+   * in the case that the Python object is callable:
+   *  - "arguments" and
+   *  - "caller"
+   *
+   * Inheriting from Function has the unfortunate side effect that we MUST
+   * expose the members "proxy.arguments" and "proxy.caller" because they are
+   * nonconfigurable, nonwritable, nonenumerable own properties. They are just
+   * always `null`.
+   *
+   * We also get the properties "length" and "name" which are configurable so we
+   * delete them in the constructor. "prototype" is not configurable so we can't
+   * delete it, however it *is* writable so we set it to be undefined. We must
+   * still make "prototype in proxy" be true though.
+   * @private
+   */
+  Module.pyproxy_new = function(ptrobj) {
+    // Technically, this leaks memory, since we're holding on to a reference
+    // to the proxy forever.  But we have that problem anyway since we don't
+    // have a destructor in Javascript to free the Python object.
+    // _pyproxy_destroy, which is a way for users to manually delete the proxy,
+    // also deletes the proxy from this set.
+    if (Module.PyProxies.hasOwnProperty(ptrobj)) {
+      return Module.PyProxies[ptrobj];
+    }
+    let flags = _pyproxy_getflags(ptrobj);
+    let cls = Module.getPyProxyClass(flags);
+    // Reflect.construct calls the constructor of Module.PyProxyClass but sets
+    // the prototype as cls.prototype. This gives us a way to dynamically create
+    // subclasses of PyProxyClass (as long as we don't need to use the "new
+    // cls(ptrobj)" syntax).
+    let target;
+    if (flags & IS_CALLABLE) {
+      // To make a callable proxy, we must call the Function constructor.
+      // In this case we are effectively subclassing Function.
+      target = Reflect.construct(Function, [], cls);
+      // Remove undesireable properties added by Function constructor. Note: we
+      // can't remove "arguments" or "caller" because they are not configurable
+      // and not writable
+      delete target.length;
+      delete target.name;
+      // prototype isn't configurable so we can't delete it but it's writable.
+      target.prototype = undefined;
+    } else {
+      target = Object.create(cls.prototype);
+    }
+    Object.defineProperty(target, "$$",
+                          {value : {ptr : ptrobj, type : 'PyProxy'}});
+    _Py_IncRef(ptrobj);
+    let proxy = new Proxy(target, Module.PyProxyHandlers);
+    Module.PyProxies[ptrobj] = proxy;
+    return proxy;
+  };
+
   function _getPtr(jsobj) {
     let ptr = jsobj.$$.ptr;
     if (ptr === null) {

src/core/python2js.c

@@ -1,7 +1,9 @@
 #define PY_SSIZE_T_CLEAN
 #include "Python.h"
 
+#include "docstring.h"
 #include "hiwire.h"
+#include "js2python.h"
 #include "jsproxy.h"
 #include "pyproxy.h"
 #include "python2js.h"
@@ -428,3 +430,57 @@ python2js_with_depth(PyObject* x, int depth)
   }
   return result;
 }
+
+static PyObject*
+to_js(PyObject* _mod, PyObject* args)
+{
+  PyObject* obj;
+  int depth = -1;
+  if (!PyArg_ParseTuple(args, "O|i:to_js", &obj, &depth)) {
+    return NULL;
+  }
+  if (obj == Py_None || PyBool_Check(obj) || PyLong_Check(obj) ||
+      PyFloat_Check(obj) || PyUnicode_Check(obj) || JsProxy_Check(obj) ||
+      JsException_Check(obj)) {
+    // No point in converting these, it'd be dumb to proxy them so they'd just
+    // get converted back by `js2python` at the end
+    Py_INCREF(obj);
+    return obj;
+  }
+  JsRef js_result = python2js_with_depth(obj, depth);
+  PyObject* py_result;
+  if (js_result == NULL) {
+    return NULL;
+  }
+  if (hiwire_is_pyproxy(js_result)) {
+    // Oops, just created a PyProxy. Wrap it I guess?
+    py_result = JsProxy_create(js_result);
+  } else {
+    py_result = js2python(js_result);
+  }
+  hiwire_CLEAR(js_result);
+  return py_result;
+}
+
+static PyMethodDef methods[] = {
+  {
+    "to_js",
+    to_js,
+    METH_VARARGS,
+  },
+  { NULL } /* Sentinel */
+};
+
+int
+python2js_init(PyObject* core)
+{
+  bool success = false;
+  PyObject* docstring_source = PyImport_ImportModule("_pyodide._core");
+  FAIL_IF_NULL(docstring_source);
+  FAIL_IF_MINUS_ONE(
+    add_methods_and_set_docstrings(core, methods, docstring_source));
+  success = true;
+finally:
+  Py_CLEAR(docstring_source);
+  return success ? 0 : -1;
+}

src/core/python2js.h

@@ -27,4 +27,7 @@ python2js(PyObject* x);
 JsRef
 python2js_with_depth(PyObject* x, int depth);
 
+int
+python2js_init(PyObject* core);
+
 #endif /* PYTHON2JS_H */

src/pyodide-py/_pyodide/_core.py

@@ -12,6 +12,7 @@ _save_name = __name__
 __name__ = "pyodide"
 try:
     # From jsproxy.c
+
     class JsException(Exception):
         """
         A wrapper around a Javascript Error to allow it to be thrown in Python.
@@ -43,8 +44,14 @@ try:
         def new(self, *args, **kwargs) -> "JsProxy":
             """Construct a new instance of the Javascript object"""
 
-        def to_py(self) -> Any:
-            """Convert the :class:`JsProxy` to a native Python object as best as possible"""
+        def to_py(self, depth: int = -1) -> Any:
+            """Convert the :class:`JsProxy` to a native Python object as best as
+            possible.
+
+            By default does a deep conversion, if a shallow conversion is
+            desired, you can use ``proxy.to_py(1)``. See
+            :ref:`type-translations-jsproxy-to-py` for more information.
+            """
             pass
 
         def then(self, onfulfilled: Callable, onrejected: Callable) -> "Promise":
@@ -94,6 +101,21 @@ try:
         """
         return obj
 
+    # from python2js
+
+    def to_js(obj: Any, depth: int = -1) -> JsProxy:
+        """Convert the object to Javascript.
+
+        This is similar to :any:`PyProxy.toJs`, but for use from Python. If the
+        object would be implicitly translated to Javascript, it will be returned
+        unchanged. If the object cannot be converted into Javascript, this
+        method will return a :any:`JsProxy` of a :any:`PyProxy`, as if you had
+        used :any:`pyodide.create_proxy`.
+
+        See :ref:`type-translations-pyproxy-to-js` for more information.
+        """
+        return obj
+
 
 finally:
     __name__ = _save_name

src/pyodide-py/pyodide/__init__.py

@@ -1,5 +1,5 @@
 from ._base import open_url, eval_code, eval_code_async, find_imports
-from ._core import JsProxy, JsException, create_once_callable, create_proxy  # type: ignore
+from ._core import JsProxy, JsException, create_once_callable, create_proxy, to_js  # type: ignore
 from ._importhooks import jsfinder
 from .webloop import WebLoopPolicy
 from . import _state  # type: ignore # noqa
@@ -24,6 +24,7 @@ __all__ = [
     "find_imports",
     "JsProxy",
     "JsException",
+    "to_js",
     "register_js_module",
     "unregister_js_module",
     "create_once_callable",

src/pyodide-py/pyodide/_core.py

@@ -5,6 +5,12 @@ if "_pyodide_core" not in sys.modules:
 
     sys.modules["_pyodide_core"] = _pyodide_core
 
-from _pyodide_core import JsProxy, JsException, create_proxy, create_once_callable
+from _pyodide_core import (
+    JsProxy,
+    JsException,
+    create_proxy,
+    create_once_callable,
+    to_js,
+)
 
-__all__ = ["JsProxy", "JsException", "create_proxy", "create_once_callable"]
+__all__ = ["JsProxy", "JsException", "create_proxy", "create_once_callable", "to_js"]

src/pyodide.js

@@ -384,6 +384,8 @@ globalThis.loadPyodide = async function(config = {}) {
   // clang-format off
   let PUBLIC_API = [
     'globals',
+    'pyodide_py',
+    'version',
     'loadPackage',
     'loadPackagesFromImports',
     'loadedPackages',
@@ -391,11 +393,10 @@ globalThis.loadPyodide = async function(config = {}) {
     'pyimport',
     'runPython',
     'runPythonAsync',
-    'version',
     'registerJsModule',
     'unregisterJsModule',
     'setInterruptBuffer',
-    'pyodide_py',
+    'toPy',
     'PythonError',
   ];
   // clang-format on
@@ -712,6 +713,64 @@ globalThis.loadPyodide = async function(config = {}) {
   };
   // clang-format on
 
+  /**
+   * Convert the Javascript object to a Python object as best as possible.
+   *
+   * This is similar to :any:`JsProxy.to_py` but for use from Javascript. If the
+   * object is immutable or a :any:`PyProxy`, it will be returned unchanged. If
+   * the object cannot be converted into Python, it will be returned unchanged.
+   *
+   * See :ref:`type-translations-jsproxy-to-py` for more information.
+   *
+   * @param {*} obj
+   * @param {number} depth Optional argument to limit the depth of the
+   * conversion.
+   * @returns {PyProxy} The object converted to Python.
+   */
+  Module.toPy = function(obj, depth = -1) {
+    // No point in converting these, it'd be dumb to proxy them so they'd just
+    // get converted back by `js2python` at the end
+    // clang-format off
+    switch (typeof obj) {
+      case "string":
+      case "number":
+      case "boolean":
+      case "bigint":
+      case "undefined":
+        return obj;
+    }
+    // clang-format on
+    if (!obj || Module.isPyProxy(obj)) {
+      return obj;
+    }
+    let obj_id = 0;
+    let py_result = 0;
+    let result = 0;
+    try {
+      obj_id = Module.hiwire.new_value(obj);
+      py_result = Module.__js2python_convert(obj_id, new Map(), depth);
+      // clang-format off
+      if(py_result === 0){
+        // clang-format on
+        Module._pythonexc2js();
+      }
+      if (Module._JsProxy_Check(py_result)) {
+        // Oops, just created a JsProxy. Return the original object.
+        return obj;
+        // return Module.pyproxy_new(py_result);
+      }
+      result = Module._python2js(py_result);
+      // clang-format off
+      if (result === 0) {
+        // clang-format on
+        Module._pythonexc2js();
+      }
+    } finally {
+      Module.hiwire.decref(obj_id);
+      Module._Py_DecRef(py_result);
+    }
+    return Module.hiwire.pop_value(result);
+  };
   /**
    * Is the argument a :any:`PyProxy`?
    * @param jsobj {any} Object to test.

src/tests/test_typeconversions.py

@@ -254,6 +254,69 @@ def test_python2js(selenium):
     )
 
 
+def test_wrong_way_conversions(selenium):
+    selenium.run_js(
+        """
+        assert(() => pyodide.toPy(5) === 5);
+        assert(() => pyodide.toPy(5n) === 5n);
+        assert(() => pyodide.toPy("abc") === "abc");
+        class Test {};
+        let t = new Test();
+        assert(() => pyodide.toPy(t) === t);
+
+        window.a1 = [1,2,3];
+        window.b1 = pyodide.toPy(a1);
+        window.a2 = { a : 1, b : 2, c : 3};
+        window.b2 = pyodide.toPy(a2);
+        pyodide.runPython(`
+            from js import a1, b1, a2, b2
+            assert a1.to_py() == b1
+            assert a2.to_py() == b2
+        `);
+        window.b1.destroy();
+        window.b2.destroy();
+        """
+    )
+
+    selenium.run_js(
+        """
+        window.a = [1,2,3];
+        window.b = pyodide.runPython(`
+            import pyodide
+            pyodide.to_js([1, 2, 3])
+        `);
+        assert(() => JSON.stringify(a) == JSON.stringify(b));
+        """
+    )
+
+    selenium.run_js(
+        """
+        window.t3 = pyodide.runPython(`
+            class Test: pass
+            t1 = Test()
+            t2 = pyodide.to_js(t1)
+            t2
+        `);
+        pyodide.runPython(`
+            from js import t3
+            assert t1 is t3
+            t2.destroy();
+        `);
+        """
+    )
+
+    selenium.run_js(
+        """
+        pyodide.runPython(`
+            s = "avafhjpa"
+            t = 55
+            assert pyodide.to_js(s) is s
+            assert pyodide.to_js(t) is t
+        `);
+        """
+    )
+
+
 def test_python2js_long_ints(selenium):
     assert selenium.run("2**30") == 2 ** 30
     assert selenium.run("2**31") == 2 ** 31
@@ -704,10 +767,10 @@ def test_python2js_with_depth(selenium):
         set(
             selenium.run_js(
                 """
-        return Array.from(pyodide.runPython(`
-            { 1, "1" }
-        `).toJs().values())
-        """
+                return Array.from(pyodide.runPython(`
+                    { 1, "1" }
+                `).toJs().values())
+                """
             )
         )
         == {1, "1"}
@@ -717,10 +780,10 @@ def test_python2js_with_depth(selenium):
         dict(
             selenium.run_js(
                 """
-        return Array.from(pyodide.runPython(`
-            { 1 : 7, "1" : 9 }
-        `).toJs().entries())
-        """
+                return Array.from(pyodide.runPython(`
+                    { 1 : 7, "1" : 9 }
+                `).toJs().entries())
+                """
             )
         )
         == {1: 7, "1": 9}