Make a pyproxy of an awaitable py object an awaitable js object (#1170)

docs/project/changelog.md

@@ -57,14 +57,18 @@
   raise a `KeyboardInterrupt` by writing to the interrupt buffer.
   [#1148](https://github.com/iodide-project/pyodide/pull/1148) and
   [#1173](https://github.com/iodide-project/pyodide/pull/1173)
-- A `JsProxy` of a Javascript `Promise` or other awaitable object is now a 
-  Python awaitable.
-  [#880](https://github.com/iodide-project/pyodide/pull/880)
 - Added a Python event loop to support asyncio by scheduling coroutines to run 
   as jobs on the browser event loop. This event loop is available by default and 
   automatically enabled by any relevant asyncio API, so for instance 
   `asyncio.ensure_future` works without any configuration.
   [#1158](https://github.com/iodide-project/pyodide/pull/1158)
+- A `PyProxy` of a Python coroutine or awaitable is now an awaitable javascript
+  object. Awaiting a coroutine will schedule it to run on the Python event loop
+  using `asyncio.ensure_future`.
+  [#1170](https://github.com/iodide-project/pyodide/pull/1170)
+- A `JsProxy` of a Javascript `Promise` or other awaitable object is now a 
+  Python awaitable.
+  [#880](https://github.com/iodide-project/pyodide/pull/880)
 - Made PyProxy of an iterable Python object an iterable Js object: defined the
   `[Symbol.iterator]` method, can be used like `for(let x of proxy)`.
   Made a PyProxy of a Python iterator an iterator: `proxy.next()` is
@@ -87,10 +91,6 @@
 - JsBoundMethod is now a subclass of JsProxy, which fixes nested attribute
   access and various other strange bugs.
   [#1124](https://github.com/iodide-project/pyodide/pull/1124)
-- In console.html: sync behavior, full stdout/stderr support, clean namespace,
-  bigger font, correct result representation, clean traceback
-  [#1125](https://github.com/iodide-project/pyodide/pull/1125) and
-  [#1141](https://github.com/iodide-project/pyodide/pull/1141)
 - Javascript functions imported like `from js import fetch` no longer trigger
   "invalid invocation" errors (issue
   [#461](https://github.com/iodide-project/pyodide/issues/461)) and
@@ -99,11 +99,15 @@
   [#1126](https://github.com/iodide-project/pyodide/pull/1126)
 - Javascript bound method calls now work correctly with keyword arguments.
   [#1138](https://github.com/iodide-project/pyodide/pull/1138)
+- In console.html: sync behavior, full stdout/stderr support, clean namespace,
+  bigger font, correct result representation, clean traceback
+  [#1125](https://github.com/iodide-project/pyodide/pull/1125) and
+  [#1141](https://github.com/iodide-project/pyodide/pull/1141)
 - Switched from ̀Jedi to rlcompleter for completion in
   `pyodide.console.InteractiveConsole` and so in `console.html`. This fixes
   some completion issues (see
   [#821](https://github.com/iodide-project/pyodide/issues/821) and
-  [#1160](https://github.com/iodide-project/pyodide/issues/821)
+  [#1160](https://github.com/iodide-project/pyodide/issues/1160)
 
 ## Version 0.16.1
 *December 25, 2020*

src/core/hiwire.c

@@ -322,6 +322,12 @@ EM_JS_REF(JsRef, hiwire_call, (JsRef idfunc, JsRef idargs), {
   return Module.hiwire.new_value(jsfunc(... jsargs));
 });
 
+EM_JS_REF(JsRef, hiwire_call_OneArg, (JsRef idfunc, JsRef idarg), {
+  let jsfunc = Module.hiwire.get_value(idfunc);
+  let jsarg = Module.hiwire.get_value(idarg);
+  return Module.hiwire.new_value(jsfunc(jsarg));
+});
+
 EM_JS_REF(JsRef,
           hiwire_call_bound,
           (JsRef idfunc, JsRef idthis, JsRef idargs),

src/core/hiwire.h

@@ -371,6 +371,9 @@ hiwire_dir(JsRef idobj);
 JsRef
 hiwire_call(JsRef idobj, JsRef idargs);
 
+JsRef
+hiwire_call_OneArg(JsRef idfunc, JsRef idarg);
+
 JsRef
 hiwire_call_bound(JsRef idfunc, JsRef idthis, JsRef idargs);
 

src/core/pyproxy.c

@@ -7,6 +7,12 @@
 #include "js2python.h"
 #include "python2js.h"
 
+_Py_IDENTIFIER(result);
+_Py_IDENTIFIER(ensure_future);
+_Py_IDENTIFIER(add_done_callback);
+
+static PyObject* asyncio;
+
 JsRef
 _pyproxy_repr(PyObject* pyobj)
 {
@@ -223,6 +229,176 @@ _pyproxy_destroy(PyObject* ptrobj)
   EM_ASM({ delete Module.PyProxies[$0]; }, ptrobj);
 }
 
+/**
+ * Test if a PyObject is awaitable.
+ * Uses _PyCoro_GetAwaitableIter like in the implementation of the GET_AWAITABLE
+ * opcode (see ceval.c). Unfortunately this is not a public API (see issue
+ * https://bugs.python.org/issue24510) so it could be a source of instability.
+ *
+ * :param pyobject: The Python object.
+ * :return: 1 if the python code "await obj" would succeed, 0 otherwise. Never
+ * fails.
+ */
+bool
+_pyproxy_is_awaitable(PyObject* pyobject)
+{
+  PyObject* awaitable = _PyCoro_GetAwaitableIter(pyobject);
+  PyErr_Clear();
+  bool result = awaitable != NULL;
+  Py_CLEAR(awaitable);
+  return result;
+}
+
+// clang-format off
+/**
+ * A simple Callable python object. Intended to be called with a single argument
+ * which is the future that was resolved.
+ */
+typedef struct {
+    PyObject_HEAD
+    /** Will call this function with the result if the future succeeded */
+    JsRef resolve_handle;
+    /** Will call this function with the error if the future succeeded */
+    JsRef reject_handle;
+} FutureDoneCallback;
+// clang-format on
+
+static void
+FutureDoneCallback_dealloc(FutureDoneCallback* self)
+{
+  hiwire_CLEAR(self->resolve_handle);
+  hiwire_CLEAR(self->reject_handle);
+  Py_TYPE(self)->tp_free((PyObject*)self);
+}
+
+/**
+ * Helper method: if the future resolved successfully, call resolve_handle on
+ * the result.
+ */
+int
+FutureDoneCallback_call_resolve(FutureDoneCallback* self, PyObject* result)
+{
+  bool success = false;
+  JsRef result_js = NULL;
+  JsRef output = NULL;
+  result_js = python2js(result);
+  output = hiwire_call_OneArg(self->resolve_handle, result_js);
+
+  success = true;
+finally:
+  hiwire_CLEAR(result_js);
+  hiwire_CLEAR(output);
+  return success ? 0 : -1;
+}
+
+/**
+ * Helper method: if the future threw an error, call reject_handle on a
+ * converted exception. The caller leaves the python error indicator set.
+ */
+int
+FutureDoneCallback_call_reject(FutureDoneCallback* self)
+{
+  bool success = false;
+  JsRef excval = NULL;
+  JsRef result = NULL;
+  // wrap_exception looks up the current exception and wraps it in a Js error.
+  excval = wrap_exception();
+  FAIL_IF_NULL(excval);
+  result = hiwire_call_OneArg(self->reject_handle, excval);
+
+  success = true;
+finally:
+  hiwire_CLEAR(excval);
+  hiwire_CLEAR(result);
+  return success ? 0 : -1;
+}
+
+/**
+ * Intended to be called with a single argument which is the future that was
+ * resolved. Resolves the promise as appropriate based on the result of the
+ * future.
+ */
+PyObject*
+FutureDoneCallback_call(FutureDoneCallback* self,
+                        PyObject* args,
+                        PyObject* kwargs)
+{
+  PyObject* fut;
+  if (!PyArg_UnpackTuple(args, "future_done_callback", 1, 1, &fut)) {
+    return NULL;
+  }
+  PyObject* result = _PyObject_CallMethodIdObjArgs(fut, &PyId_result, NULL);
+  int errcode;
+  if (result != NULL) {
+    errcode = FutureDoneCallback_call_resolve(self, result);
+  } else {
+    errcode = FutureDoneCallback_call_reject(self);
+  }
+  if (errcode == 0) {
+    Py_RETURN_NONE;
+  } else {
+    return NULL;
+  }
+}
+
+// clang-format off
+static PyTypeObject FutureDoneCallbackType = {
+    .tp_name = "FutureDoneCallback",
+    .tp_doc = "Callback for internal use to allow awaiting a future from javascript",
+    .tp_basicsize = sizeof(FutureDoneCallback),
+    .tp_itemsize = 0,
+    .tp_flags = Py_TPFLAGS_DEFAULT,
+    .tp_dealloc = (destructor) FutureDoneCallback_dealloc,
+    .tp_call = (ternaryfunc) FutureDoneCallback_call,
+};
+// clang-format on
+
+static PyObject*
+FutureDoneCallback_cnew(JsRef resolve_handle, JsRef reject_handle)
+{
+  FutureDoneCallback* self =
+    (FutureDoneCallback*)FutureDoneCallbackType.tp_alloc(
+      &FutureDoneCallbackType, 0);
+  self->resolve_handle = hiwire_incref(resolve_handle);
+  self->reject_handle = hiwire_incref(reject_handle);
+  return (PyObject*)self;
+}
+
+/**
+ * Intended to be called with a single argument which is the future that was
+ * resolved. Resolves the promise as appropriate based on the result of the
+ * future.
+ *
+ * :param pyobject: An awaitable python object
+ * :param resolve_handle: The resolve javascript method for a promise
+ * :param reject_handle: The reject javascript method for a promise
+ * :return: 0 on success, -1 on failure
+ */
+int
+_pyproxy_ensure_future(PyObject* pyobject,
+                       JsRef resolve_handle,
+                       JsRef reject_handle)
+{
+  bool success = false;
+  PyObject* future = NULL;
+  PyObject* callback = NULL;
+  PyObject* retval = NULL;
+  future =
+    _PyObject_CallMethodIdObjArgs(asyncio, &PyId_ensure_future, pyobject, NULL);
+  FAIL_IF_NULL(future);
+  callback = FutureDoneCallback_cnew(resolve_handle, reject_handle);
+  retval = _PyObject_CallMethodIdObjArgs(
+    future, &PyId_add_done_callback, callback, NULL);
+  FAIL_IF_NULL(retval);
+
+  success = true;
+finally:
+  Py_CLEAR(future);
+  Py_CLEAR(callback);
+  Py_CLEAR(retval);
+  return success ? 0 : -1;
+}
+
 EM_JS_REF(JsRef, pyproxy_new, (PyObject * ptrobj), {
   // Technically, this leaks memory, since we're holding on to a reference
   // to the proxy forever.  But we have that problem anyway since we don't
@@ -249,11 +425,15 @@ EM_JS_REF(JsRef, pyproxy_new, (PyObject * ptrobj), {
   }
   // clang-format on
   Module.PyProxies[ptrobj] = proxy;
+  let is_awaitable = __pyproxy_is_awaitable(ptrobj);
+  if (is_awaitable) {
+    Object.assign(target, Module.PyProxyAwaitableMethods);
+  }
 
   return Module.hiwire.new_value(proxy);
 });
 
-EM_JS(int, pyproxy_init, (), {
+EM_JS_NUM(int, pyproxy_init_js, (), {
   // clang-format off
   Module.PyProxies = {};
   function _getPtr(jsobj) {
@@ -314,7 +494,7 @@ EM_JS(int, pyproxy_init, (), {
     },
   };
 
-  // See: 
+  // See:
   // https://docs.python.org/3/c-api/iter.html
   // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Iteration_protocols
   // This avoids allocating a PyProxy wrapper for the temporary iterator.
@@ -338,7 +518,7 @@ EM_JS(int, pyproxy_init, (), {
   Module.PyProxyIteratorMethods = {
     [Symbol.iterator] : function() {
       return this;
-    },    
+    },
     next : function(arg) {
       let idresult;
       // Note: arg is optional, if arg is not supplied, it will be undefined
@@ -350,7 +530,7 @@ EM_JS(int, pyproxy_init, (), {
         Module.fatal_error(e);
       } finally {
         Module.hiwire.decref(idarg);
-      }      
+      }
 
       let done = false;
       if(idresult === 0){
@@ -473,6 +653,61 @@ EM_JS(int, pyproxy_init, (), {
     },
   };
 
+  Module.PyProxyAwaitableMethods = {
+    _ensure_future : function(){
+      let resolve_handle_id = 0;
+      let reject_handle_id = 0;
+      let resolveHandle;
+      let rejectHandle;
+      let promise;
+      try {
+        promise = new Promise((resolve, reject) => {
+          resolveHandle = resolve;
+          rejectHandle = reject;
+        });
+        resolve_handle_id = Module.hiwire.new_value(resolveHandle);
+        reject_handle_id = Module.hiwire.new_value(rejectHandle);
+        let ptrobj = _getPtr(this);
+        let errcode = __pyproxy_ensure_future(ptrobj, resolve_handle_id, reject_handle_id);
+        if(errcode === -1){
+          _pythonexc2js();
+        }
+      } finally {
+        Module.hiwire.decref(resolve_handle_id);
+        Module.hiwire.decref(reject_handle_id);
+      }
+      return promise;
+    },
+    then : function(onFulfilled, onRejected){
+      let promise = this._ensure_future();
+      return promise.then(onFulfilled, onRejected);
+    },
+    catch : function(onRejected){
+      let promise = this._ensure_future();
+      return promise.catch(onRejected);
+    },
+    finally : function(onFinally){
+      let promise = this._ensure_future();
+      return promise.finally(onFinally);
+    }
+  };
+
   return 0;
 // clang-format on
 });
+
+int
+pyproxy_init()
+{
+  asyncio = PyImport_ImportModule("asyncio");
+  if (asyncio == NULL) {
+    return -1;
+  }
+  if (PyType_Ready(&FutureDoneCallbackType)) {
+    return -1;
+  }
+  if (pyproxy_init_js()) {
+    return -1;
+  }
+  return 0;
+}

src/core/python2js.c

@@ -16,6 +16,49 @@ _Py_IDENTIFIER(format_exception);
 static JsRef
 _python2js_unicode(PyObject* x);
 
+EM_JS_REF(JsRef, pyproxy_to_js_error, (JsRef pyproxy), {
+  return Module.hiwire.new_value(
+    new Module.PythonError(Module.hiwire.get_value(pyproxy)));
+});
+
+JsRef
+wrap_exception()
+{
+  bool success = true;
+  PyObject* type = NULL;
+  PyObject* value = NULL;
+  PyObject* traceback = NULL;
+  JsRef pyexc_proxy = NULL;
+  JsRef jserror = NULL;
+
+  PyErr_Fetch(&type, &value, &traceback);
+  PyErr_NormalizeException(&type, &value, &traceback);
+  if (type == NULL || type == Py_None || value == NULL || value == Py_None) {
+    PyErr_SetString(PyExc_TypeError, "No exception type or value");
+    FAIL();
+  }
+
+  if (traceback == NULL) {
+    traceback = Py_None;
+    Py_INCREF(traceback);
+  }
+  PyException_SetTraceback(value, traceback);
+
+  pyexc_proxy = pyproxy_new(value);
+  jserror = pyproxy_to_js_error(pyexc_proxy);
+
+  success = true;
+finally:
+  Py_CLEAR(type);
+  Py_CLEAR(value);
+  Py_CLEAR(traceback);
+  hiwire_CLEAR(pyexc_proxy);
+  if (!success) {
+    hiwire_CLEAR(jserror);
+  }
+  return jserror;
+}
+
 void _Py_NO_RETURN
 pythonexc2js()
 {
@@ -393,6 +436,18 @@ python2js_init()
   FAIL_IF_NULL(globals);
 
   EM_ASM({
+    class PythonError extends Error
+    {
+      constructor(pythonError)
+      {
+        let message = "Python Error";
+        super(message);
+        this.name = this.constructor.name;
+        this.pythonError = pythonError;
+      }
+    };
+    Module.PythonError = PythonError;
+
     Module.test_python2js_with_depth = function(name, depth)
     {
       let pyname = stringToNewUTF8(name);

src/core/python2js.h

@@ -9,6 +9,9 @@
 // clang-format on
 #include "hiwire.h"
 
+JsRef
+wrap_exception();
+
 /** Convert the active Python exception into a Javascript Error object
  *  and print it to the console.
  */

src/pyodide-py/pyodide/__init__.py

@@ -1,4 +1,4 @@
-from ._base import open_url, eval_code, find_imports, as_nested_list
+from ._base import open_url, eval_code, eval_code_async, find_imports, as_nested_list
 from ._core import JsException  # type: ignore
 from ._importhooks import JsFinder
 from .webloop import WebLoopPolicy
@@ -20,6 +20,7 @@ __version__ = "0.16.1"
 __all__ = [
     "open_url",
     "eval_code",
+    "eval_code_async",
     "find_imports",
     "as_nested_list",
     "JsException",

src/pyodide-py/pyodide/_base.py

@@ -229,10 +229,25 @@ class CodeRunner:
             return eval(last_expr, self.globals, self.locals)
 
     async def run_async(self, code: str) -> Any:
-        """ //!\\ WARNING //!\\
-        This is not working yet. For use once we add an EventLoop.
+        """Runs a code string asynchronously.
 
-        Note: see `_eval_code_async`.
+        Uses
+        [PyCF_ALLOW_TOP_LEVEL_AWAIT](https://docs.python.org/3/library/ast.html#ast.PyCF_ALLOW_TOP_LEVEL_AWAIT)
+        to compile to code.
+
+        Parameters
+        ----------
+        code
+           the Python code to run.
+
+        Returns
+        -------
+        If the last nonwhitespace character of code is a semicolon,
+        return `None`.
+        If the last statement is an expression, return the
+        result of the expression.
+        Use the `return_mode` and `quiet_trailing_semicolon` parameters in the
+        constructor to modify this default behavior.
         """
         mod, last_expr = self._split_and_compile(
             code, flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT  # type: ignore
@@ -316,14 +331,49 @@ async def eval_code_async(
     quiet_trailing_semicolon: bool = True,
     filename: str = "<exec>",
 ) -> Any:
-    """ //!\\ WARNING //!\\
-    This is not working yet. For use once we add an EventLoop.
+    """Runs a code string asynchronously.
 
-    Note: once async is working, one should:
-      - rename `_eval_code_async` in `eval_code_async` (remove leading '_')
-      - remove exceptions here and in `CodeRunner.run_async`
-      - add docstrings here and in `CodeRunner.run_async`
-      - add tests
+    Uses
+    [PyCF_ALLOW_TOP_LEVEL_AWAIT](https://docs.python.org/3/library/ast.html#ast.PyCF_ALLOW_TOP_LEVEL_AWAIT)
+    to compile to code.
+
+    Parameters
+    ----------
+    code
+       the Python code to run.
+    globals
+        The global scope in which to execute code. This is used as the `exec`
+        `globals` parameter. See
+        [the exec documentation](https://docs.python.org/3/library/functions.html#exec)
+        for more info.
+    locals
+        The local scope in which to execute code. This is used as the `exec`
+        `locals` parameter. As with `exec`, if `locals` is absent, it is set equal
+        to `globals`. See
+        [the exec documentation](https://docs.python.org/3/library/functions.html#exec)
+        for more info.
+    return_mode
+        Specifies what should be returned, must be one of 'last_expr',
+        'last_expr_or_assign' or `None`. On other values an exception is raised.
+
+        'last_expr' -- return the last expression
+        'last_expr_or_assign' -- return the last expression or the last
+        (named) assignment.
+        'none' -- always return `None`.
+    quiet_trailing_semicolon
+        whether a trailing semicolon should 'quiet' the result or not.
+        Setting this to `True` (default) mimic the CPython's interpret
+        behavior ; whereas setting it to `False` mimic the IPython's
+    filename:
+       file from which the code was read.
+
+    Returns
+    -------
+    If the last nonwhitespace character of code is a semicolon return `None`.
+    If the last statement is an expression, return the
+    result of the expression.
+    Use the `return_mode` and `quiet_trailing_semicolon` parameters to modify
+    this default behavior.
     """
     return await CodeRunner(
         globals=globals,

src/tests/test_asyncio.py

@@ -261,3 +261,90 @@ def test_eval_code_await_error(selenium):
             r2 = c.send(r1.result())
             """
         )
+
+
+def test_await_pyproxy_eval_async(selenium):
+    assert (
+        selenium.run_js(
+            """
+        let c = pyodide._module.pyodide_py._base.eval_code_async("1+1");
+        return await c;
+        """
+        )
+        == 2
+    )
+
+    assert (
+        selenium.run_js(
+            """
+        let finally_occurred = false;
+        let c = pyodide._module.pyodide_py._base.eval_code_async("1+1");
+        let result = await c.finally(() => { finally_occurred = true; });
+        return [result, finally_occurred];
+        """
+        )
+        == [2, True]
+    )
+
+    assert (
+        selenium.run_js(
+            """
+        let finally_occurred = false;
+        let err_occurred = false;
+        let c = pyodide._module.pyodide_py._base.eval_code_async("raise ValueError('hi')");
+        try {
+            let result = await c.finally(() => { finally_occurred = true; });
+        } catch(e){
+            err_occurred = e.constructor.name === "PythonError";
+        }
+        return [finally_occurred, err_occurred];
+        """
+        )
+        == [True, True]
+    )
+
+    assert selenium.run_js(
+        """
+        let c = pyodide._module.pyodide_py._base.eval_code_async("raise ValueError('hi')");
+        return await c.catch(e => e.constructor.name === "PythonError");
+        """
+    )
+
+    assert selenium.run_js(
+        """
+        let packages = await pyodide._module.pyodide_py._base.eval_code_async(`
+            from js import fetch
+            await (await fetch('packages.json')).json()
+        `);
+        return (!!packages.dependencies) && (!!packages.import_name_to_package_name);
+        """
+    )
+
+    assert selenium.run_js(
+        """
+        let c = pyodide._module.pyodide_py._base.eval_code_async("1+1");
+        await c;
+        let err_occurred = false;
+        try {
+            // Triggers: cannot await already awaited coroutine
+            await c;
+        } catch(e){
+            err_occurred = true;
+        }
+        return err_occurred;
+        """
+    )
+
+
+def test_await_pyproxy_async_def(selenium):
+    assert selenium.run_js(
+        """
+        let packages = await pyodide.runPython(`
+            from js import fetch
+            async def temp():
+                return await (await fetch('packages.json')).json()
+            temp()  
+        `);
+        return (!!packages.dependencies) && (!!packages.import_name_to_package_name);
+        """
+    )

src/tests/test_pyproxy.py

@@ -201,3 +201,64 @@ def test_pyproxy_iter(selenium):
         """
     )
     assert result == result2
+
+
+def test_pyproxy_mixins(selenium):
+    result = selenium.run_js(
+        """
+        let [noimpls, awaitable, iterable, iterator, awaititerable, awaititerator] = pyodide.runPython(`
+            class NoImpls: pass
+
+            class Await:
+                def __await__(self):
+                    return iter([])
+            
+            class Iter:
+                def __iter__(self):
+                    return iter([])
+            
+            class Next:
+                def __next__(self):
+                    pass
+            
+            class AwaitIter(Await, Iter): pass
+
+            class AwaitNext(Await, Next): pass
+
+            [NoImpls(), Await(), Iter(), Next(), AwaitIter(), AwaitNext()]
+        `);
+        let name_proxy = {noimpls, awaitable, iterable, iterator, awaititerable, awaititerator};
+        let result = {};
+        for(let [name, x] of Object.entries(name_proxy)){
+            let impls = { 
+                "then" : x.then !== undefined,
+                "catch" : x.catch !== undefined,
+                "finally_" : x.finally !== undefined,
+                "iterable" : x[Symbol.iterator] !== undefined,
+                "iterator" : x.next !== undefined
+            }
+            result[name] = impls;
+        }
+        return result;
+        """
+    )
+    assert result == dict(
+        noimpls=dict(
+            then=False, catch=False, finally_=False, iterable=False, iterator=False
+        ),
+        awaitable=dict(
+            then=True, catch=True, finally_=True, iterable=False, iterator=False
+        ),
+        iterable=dict(
+            then=False, catch=False, finally_=False, iterable=True, iterator=False
+        ),
+        iterator=dict(
+            then=False, catch=False, finally_=False, iterable=True, iterator=True
+        ),
+        awaititerable=dict(
+            then=True, catch=True, finally_=True, iterable=True, iterator=False
+        ),
+        awaititerator=dict(
+            then=True, catch=True, finally_=True, iterable=True, iterator=True
+        ),
+    )