Distinguish between sync and async JavaScript iterators when possible (#3339)

If a JavaScript has a `next` method and exactly one of `Symbol.iterator` or `Symbol.asyncIterator`
we use that to tell us whether we think `next` is sync or async. If both or neither of these
Symbols are present then we define both `__next__` and `__anext__`.

docs/project/changelog.md

@@ -158,7 +158,7 @@ substitutions:
 - {{ Enhancement }} It is now possible to use aynchronous JavaScript iterables,
   iterators and generators from Python. This includes support for `aiter` for async interables,
   `anext` and `asend` for async iterators, and `athrow` and `aclose` for async generators.
-  {pr}`3285`, {pr}`3299`
+  {pr}`3285`, {pr}`3299`, {pr}`3339`
 
 - {{ Enhancement }} Added a mypy typeshed for some common functionality for the
   `js` module.

src/core/jsproxy.c

@@ -60,6 +60,7 @@
 #define IS_ASYNC_ITERABLE (1 << 15)
 #define IS_GENERATOR (1 << 16)
 #define IS_ASYNC_GENERATOR (1 << 17)
+#define IS_ASYNC_ITERATOR   (1<<18)
 
 // clang-format on
 
@@ -3446,7 +3447,8 @@ JsProxy_create_subtype(int flags)
     goto skip_container_slots;
   }
 
-  if (flags & IS_ITERABLE) {
+  if ((flags & IS_ITERABLE) && !(flags & IS_ITERATOR)) {
+    // If it is an iterator we should use SelfIter instead.
     if (mapping) {
       // Prefer `obj.keys()` over `obj[Symbol.iterator]()`
       slots[cur_slot++] =
@@ -3457,11 +3459,21 @@ JsProxy_create_subtype(int flags)
         (PyType_Slot){ .slot = Py_tp_iter, .pfunc = (void*)JsProxy_GetIter };
     }
   }
-  if (flags & IS_ASYNC_ITERABLE) {
+  if ((flags & IS_ASYNC_ITERABLE) && !(flags & IS_ASYNC_ITERATOR)) {
     // This uses `obj[Symbol.asyncIterator]()`
+    // If it is an iterator we should use SelfIter instead.
     slots[cur_slot++] = (PyType_Slot){ .slot = Py_am_aiter,
                                        .pfunc = (void*)JsProxy_GetAsyncIter };
   }
+
+  // If it's an iterator, we aren't sure whether it is an async iterator or a
+  // sync iterator -- they both define a next method, you have to see whether
+  // the result is  a promise or not to learn whether we are async. But most
+  // iterators also define `Symbol.iterator` to return themself, and most async
+  // iterators define `Symbol.asyncIterator` to return themself. So if one of
+  // these is defined but not the other, we use this to decide what type we are.
+
+  // Iterator methods
   if (flags & IS_ITERATOR) {
     // We're not sure whether it is an async iterator or a sync iterator. So add
     // both methods and raise at runtime if someone uses the wrong one.
@@ -3473,12 +3485,16 @@ JsProxy_create_subtype(int flags)
       (PyType_Slot){ .slot = Py_tp_iternext, .pfunc = (void*)JsProxy_IterNext };
     slots[cur_slot++] =
       (PyType_Slot){ .slot = Py_am_send, .pfunc = (void*)JsProxy_am_send };
+    methods[cur_method++] = JsGenerator_send_MethodDef;
+  }
+
+  // Async iterator methods
+  if (flags & IS_ASYNC_ITERATOR) {
     slots[cur_slot++] =
       (PyType_Slot){ .slot = Py_am_aiter, .pfunc = (void*)PyObject_SelfIter };
     slots[cur_slot++] =
       (PyType_Slot){ .slot = Py_am_anext, .pfunc = (void*)JsGenerator_anext };
     // Send works okay on any js object that has a "next" method
-    methods[cur_method++] = JsGenerator_send_MethodDef;
     methods[cur_method++] = JsGenerator_asend_MethodDef;
   }
   if (flags & IS_GENERATOR) {
@@ -3733,7 +3749,8 @@ EM_JS_NUM(int, compute_typeflags, (JsRef idobj), {
   SET_FLAG_IF(IS_AWAITABLE, typeof obj.then === 'function')
   SET_FLAG_IF(IS_ITERABLE, typeof obj[Symbol.iterator] === 'function')
   SET_FLAG_IF(IS_ASYNC_ITERABLE, typeof obj[Symbol.asyncIterator] === 'function')
-  SET_FLAG_IF(IS_ITERATOR, typeof obj.next === 'function')
+  SET_FLAG_IF(IS_ITERATOR, typeof obj.next === 'function' && (typeof obj[Symbol.iterator] === 'function' || typeof obj[Symbol.asyncIterator] !== 'function') );
+  SET_FLAG_IF(IS_ASYNC_ITERATOR, typeof obj.next === 'function' && (typeof obj[Symbol.iterator] !== 'function' || typeof obj[Symbol.asyncIterator] === 'function') );
   SET_FLAG_IF(HAS_LENGTH,
     (typeof obj.size === "number") ||
     (typeof obj.length === "number" && typeof obj !== "function"));
@@ -4016,6 +4033,7 @@ JsProxy_init(PyObject* core_module)
   AddFlag(IS_ASYNC_ITERABLE);
   AddFlag(IS_GENERATOR);
   AddFlag(IS_ASYNC_GENERATOR);
+  AddFlag(IS_ASYNC_ITERATOR);
 
 #undef AddFlag
   FAIL_IF_MINUS_ONE(PyObject_SetAttrString(core_module, "js_flags", flag_dict));

src/py/_pyodide/_core_docs.py

@@ -625,9 +625,8 @@ class JsMutableMap(JsMap):
 class JsIterator(JsProxy):
     """A JsProxy of a JavaScript iterator.
 
-    An object is a JsIterator if it has a `next` method. We can't tell if it's
+    An object is a JsIterator if it has a `next` method and either has a
-    synchronously iterable or asynchronously iterable, so we implement both and
+    Symbol.iterator or has no Symbol.asyncIterator.
-    if you try to use the wrong one it will fail at runtime.
     """
 
     _js_type_flags = ["IS_ITERATOR"]
@@ -636,22 +635,8 @@ class JsIterator(JsProxy):
         """Send a value into the iterator. This is a wrapper around
         ``jsobj.next(value)``.
 
-        We can't tell whether a JavaScript iterator is a synchronous iterator,
+        If the object is not actually a synchronous iterator, then ``send`` will raise a
-        an asynchronous iterator, or just some object with a "next" method, so
+        TypeError (but only after calling ``jsobj.next()``!).
-        we include both ``send`` and ``asend``. If the object is not a
-        synchronous iterator, then ``send`` will raise a TypeError (but only
-        after calling ``jsobj.next()``!).
-        """
-
-    def asend(self, value: Any) -> Any:
-        """Send a value into the asynchronous iterator. This is a wrapper around
-        ``jsobj.next(value)``.
-
-        We can't tell whether a JavaScript iterator is a synchronous iterator,
-        an asynchronous iterator, or just some object with a "next" method, so
-        we include both ``send`` and ``asend``. If the object is not a
-        asynchronous iterator, then ``asend`` will raise a TypeError (but only
-        after calling ``jsobj.next()``!).
         """
 
     def __next__(self):
@@ -660,6 +645,24 @@ class JsIterator(JsProxy):
     def __iter__(self):
         pass
 
+
+class JsAsyncIterator(JsProxy):
+    """A JsProxy of a JavaScript async iterator.
+
+    An object is a JsAsyncIterator if it has a `next` method and either has a
+    Symbol.asyncIterator or has no Symbol.iterator.
+    """
+
+    _js_type_flags = ["IS_ASYNC_ITERATOR"]
+
+    def asend(self, value: Any) -> Any:
+        """Send a value into the asynchronous iterator. This is a wrapper around
+        ``jsobj.next(value)``.
+
+        If the object is not actually an asynchronous iterator, then ``asend``
+        will raise a TypeError (but only after calling ``jsobj.next()``!).
+        """
+
     def __aiter__(self):
         pass
 

src/py/pyodide/_core.py

@@ -31,6 +31,7 @@ from _pyodide._core_docs import (
     JsArray,
     JsAsyncGenerator,
     JsAsyncIterable,
+    JsAsyncIterator,
     JsBuffer,
     JsDoubleProxy,
     JsFetchResponse,
@@ -66,4 +67,5 @@ __all__ = [
     "JsFetchResponse",
     "JsMap",
     "JsMutableMap",
+    "JsAsyncIterator",
 ]

src/tests/test_jsproxy.py

@@ -1754,7 +1754,7 @@ def test_gen_send(selenium):
     import pytest
 
     from pyodide.code import run_js
-    from pyodide.ffi import JsGenerator
+    from pyodide.ffi import JsAsyncGenerator, JsAsyncIterator, JsGenerator, JsIterator
 
     f = run_js(
         """
@@ -1769,6 +1769,9 @@ def test_gen_send(selenium):
 
     it = f()
     assert isinstance(it, JsGenerator)
+    assert not isinstance(it, JsAsyncGenerator)
+    assert isinstance(it, JsIterator)
+    assert not isinstance(it, JsAsyncIterator)
 
     assert it.send(None) == 2
     assert it.send(2) == 4
@@ -1784,7 +1787,7 @@ def test_gen_send_type_errors(selenium):
     import pytest
 
     from pyodide.code import run_js
-    from pyodide.ffi import JsGenerator, JsIterator
+    from pyodide.ffi import JsAsyncIterator, JsGenerator, JsIterator
 
     g = run_js(
         """
@@ -1792,6 +1795,7 @@ def test_gen_send_type_errors(selenium):
         """
     )
     assert isinstance(g, JsIterator)
+    assert isinstance(g, JsAsyncIterator)
     assert not isinstance(g, JsGenerator)
     with pytest.raises(
         TypeError, match='Result should have type "object" not "number"'
@@ -1922,7 +1926,7 @@ async def test_agen_aiter(selenium):
     import pytest
 
     from pyodide.code import run_js
-    from pyodide.ffi import JsIterator
+    from pyodide.ffi import JsAsyncGenerator, JsAsyncIterator, JsGenerator, JsIterator
 
     f = run_js(
         """
@@ -1934,28 +1938,29 @@ async def test_agen_aiter(selenium):
         """
     )
     b = f()
-    assert isinstance(b, JsIterator)
+    assert isinstance(b, JsAsyncIterator)
+    assert not isinstance(b, JsIterator)
+    assert isinstance(b, JsAsyncGenerator)
+    assert not isinstance(b, JsGenerator)
     assert await anext(b) == 2
     assert await anext(b) == 3
     with pytest.raises(StopAsyncIteration):
         await anext(b)
 
-    with pytest.raises(TypeError, match="Result was a promise, use anext.* instead."):
-        next(f())
-
     g = run_js(
         """
         (function *(){
             yield 2;
             yield 3;
             return 7;
-        })
+        })()
         """
     )
-    with pytest.raises(
+
-        TypeError, match="Result of anext.. was not a promise, use next.. instead."
+    assert not isinstance(g, JsAsyncIterator)
-    ):
+    assert isinstance(g, JsIterator)
-        anext(g())
+    assert not isinstance(g, JsAsyncGenerator)
+    assert isinstance(g, JsGenerator)
 
 
 @run_in_pyodide
@@ -1963,7 +1968,7 @@ async def test_agen_aiter2(selenium):
     import pytest
 
     from pyodide.code import run_js
-    from pyodide.ffi import JsAsyncIterable, JsIterable, JsIterator
+    from pyodide.ffi import JsAsyncIterable, JsAsyncIterator, JsIterable, JsIterator
 
     iterable = run_js(
         """
@@ -1981,7 +1986,8 @@ async def test_agen_aiter2(selenium):
         iter(iterable)  # type:ignore[call-overload]
 
     it = aiter(iterable)
-    assert isinstance(it, JsIterator)
+    assert isinstance(it, JsAsyncIterator)
+    assert not isinstance(it, JsIterator)
 
     assert await anext(it) == 1
     assert await anext(it) == 2
@@ -1995,7 +2001,7 @@ async def test_agen_asend(selenium):
     import pytest
 
     from pyodide.code import run_js
-    from pyodide.ffi import JsIterator
+    from pyodide.ffi import JsAsyncIterator, JsIterator
 
     it = run_js(
         """
@@ -2008,7 +2014,8 @@ async def test_agen_asend(selenium):
         """
     )
 
-    assert isinstance(it, JsIterator)
+    assert isinstance(it, JsAsyncIterator)
+    assert not isinstance(it, JsIterator)
 
     assert await it.asend(None) == 2
     assert await it.asend(2) == 4