DOCS Documentation about type translation of errors (#1435)

docs/sphinx_pyodide/sphinx_pyodide/jsdoc.py

@@ -190,6 +190,9 @@ def get_jsdoc_summary_directive(app):
             It seems like colons need escaping for some reason.
             """
             colon_esc = "esccolon\\\xafhoa:"
+            # extract_summary seems to have trouble if there are Sphinx
+            # directives in descr
+            descr, _, _ = descr.partition("\n..")
             return extract_summary(
                 [descr.replace(":", colon_esc)], self.state.document
             ).replace(colon_esc, ":")

docs/sphinx_pyodide/sphinx_pyodide/lexers.py

@@ -1,4 +1,4 @@
-from pygments.lexer import bygroups, inherit, using
+from pygments.lexer import bygroups, inherit, using, default
 from pygments.lexers import PythonLexer
 from pygments.lexers.javascript import JavascriptLexer
 from pygments.lexers.html import HtmlLexer
@@ -9,13 +9,12 @@ class PyodideLexer(JavascriptLexer):
     tokens = {
         "root": [
             (
-                rf"""(pyodide)(\.)(runPython|runPythonAsync)(\()(`)""",
+                r"(pyodide)(\.)(runPython|runPythonAsync)(\()",
                 bygroups(
                     Token.Name,
                     Token.Operator,
                     Token.Name,
                     Token.Punctuation,
-                    Token.Literal.String.Single,
                 ),
                 "python-code",
             ),
@@ -23,13 +22,15 @@ class PyodideLexer(JavascriptLexer):
         ],
         "python-code": [
             (
-                r"(.+?)(`)(\))",
+                rf"({quotemark})((?:\\\\|\\[^\\]|[^{quotemark}\\])*)({quotemark})",
                 bygroups(
-                    using(PythonLexer), Token.Literal.String.Single, Token.Punctuation
+                    Token.Literal.String, using(PythonLexer), Token.Literal.String
                 ),
                 "#pop",
             )
-        ],
+            for quotemark in ["'", '"', "`"]
+        ]
+        + [default("#pop")],
     }
 
 

docs/usage/api/python-api.md

@@ -2,6 +2,20 @@
 
 Backward compatibility of the API is not guaranteed at this point.
 
+**Javascript Modules**
+
+By default there are two Javascript modules. More can be added with
+{any}`pyodide.registerJsModule`. You can import these modules using the Python
+``import`` statement in the normal way.
+
+```{eval-rst}
+.. list-table::
+
+   *  - ``js``
+      - The global Javascript scope.
+   *  - :js:mod:`pyodide_js <pyodide>`
+      - The Javascript pyodide module.
+```
 
 ```{eval-rst}
 .. currentmodule:: pyodide

docs/usage/faq.md

@@ -24,7 +24,11 @@ but not in Firefox.
 
 
 ## How can I change the behavior of {any}`runPython <pyodide.runPython>` and {any}`runPythonAsync <pyodide.runPythonAsync>`?
-The definitions of {any}`runPython <pyodide.runPython>` and {any}`runPythonAsync <pyodide.runPythonAsync>` are very simple:
+You can directly call Python functions from Javascript. For many purposes it
+makes sense to make your own Python function as an entrypoint and call that
+instead of using `runPython`. The definitions of {any}`runPython
+<pyodide.runPython>` and {any}`runPythonAsync <pyodide.runPythonAsync>` are very
+simple:
 ```javascript
 function runPython(code){
   pyodide.pyodide_py.eval_code(code, pyodide.globals);

docs/usage/quickstart.md

@@ -68,7 +68,7 @@ Create and save a test `index.html` page with the following contents:
             import sys
             sys.version
         `));
-        console.log(pyodide.runPython(`print(1 + 2)`));
+        console.log(pyodide.runPython("print(1 + 2)"));
       }
       main();
     </script>
@@ -168,8 +168,7 @@ pyodide.globals.set("alert", alert);
 pyodide.globals.set("square", x => x*x);
 
 // You can test your new Python function in the console by running
-pyodide.runPython(`square(3)`);
-
+pyodide.runPython("square(3)");
 ```
 
 Feel free to play around with the code using the browser console and the above example.

docs/usage/type-conversions.md

@@ -371,7 +371,7 @@ numpy_array = np.asarray(array)
 
 A PyProxy of any Python object supporting the
 [Python Buffer protocol](https://docs.python.org/3/c-api/buffer.html) will have
-a method called :any`getBuffer`. This can be used to retrieve a reference to a
+a method called {any}`getBuffer <PyProxy.getBuffer>`. This can be used to retrieve a reference to a
 Javascript typed array that points to the data backing the Python object,
 combined with other metadata about the buffer format. The metadata is suitable
 for use with a Javascript ndarray library if one is present. For instance, if
@@ -394,21 +394,39 @@ try {
 }
 ```
 
-## Importing Python objects into Javascript
+## Importing Objects
+It is possible to access objects in one languge from the global scope in the
+other language. It is also possible to create custom namespaces and access
+objects on the custom namespaces.
+
+### Importing Python objects into Javascript
 
 A Python object in the `__main__` global scope can imported into Javascript
-using the `pyodide.globals.get` method. Given the name of the Python object
-to import, it returns the object translated to Javascript.
+using the {any}`pyodide.globals.get <PyProxy.get>` method. Given the name of the
+Python object to import, it returns the object translated to Javascript.
 
 ```js
 let sys = pyodide.globals.get('sys');
 ```
+As always, if the result is a `PyProxy` and you care about not leaking the
+Python object, you must destroy it when you are done. It's also possible to set
+values in the Python global scope with {any}`pyodide.globals.set <PyProxy.set>`
+or remove them with {any}`pyodide.globals.delete <PyProxy.delete>`:
+```pyodide
+pyodide.globals.set("x", 2);
+pyodide.runPython("print(x)"); // Prints 2
+```
 
-As always, if the result is a `PyProxy` and you care about not leaking the Python
-object, you must destroy it when you are done.
+If you execute code with a custom globals dictionary, you can use a similar
+approach:
+```pyodide
+let my_py_namespace = pyodide.globals.get("dict")();
+pyodide.runPython("x=2", my_py_namespace);
+let x = my_py_namespace.get("x");
+```
 
 (type-translations_using-js-obj-from-py)=
-## Importing Javascript objects into Python
+### Importing Javascript objects into Python
 
 Javascript objects in the
 [`globalThis`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/globalThis)
@@ -416,8 +434,7 @@ global scope can be imported into Python using the `js` module.
 
 When importing a name from the `js` module, the `js` module looks up Javascript
 attributes of the `globalThis` scope and translates the Javascript objects into
-Python. You can create your own custom Javascript modules using
-{any}`pyodide.registerJsModule`.
+Python.
 
 ```py
 import js
@@ -425,3 +442,68 @@ js.document.title = 'New window title'
 from js.document.location import reload as reload_page
 reload_page()
 ```
+You can also assign to Javascript global variables in this way:
+```pyodide
+pyodide.runPython("js.x = 2");
+console.log(window.x); // 2
+```
+You can create your own custom Javascript modules using
+{any}`pyodide.registerJsModule` and they will behave like the `js` module except
+with a custom scope:
+```pyodide
+let my_js_namespace = { x : 3 };
+pyodide.registerJsModule("my_js_namespace", my_js_namespace);
+pyodide.runPython(`
+    from my_js_namespace import x
+    print(x) # 3
+    my_js_namespace.y = 7
+`);
+console.log(my_js_namespace.y); // 7
+```
+
+(type-translations-errors)=
+## Translating Errors
+All entrypoints and exit points from Python code are wrapped in Javascript `try`
+blocks. At the boundary between Python and Javascript, errors are caught,
+converted between languages, and rethrown.
+
+Javascript errors are wrapped in a {any}`JsException <pyodide.JsException>`.
+Python exceptions are converted to a {any}`PythonError <pyodide.PythonError>`.
+At present if an exception crosses between Python and Javascript several times,
+the resulting error message won't be as useful as one might hope.
+
+In order to reduce memory leaks, the {any}`PythonError <pyodide.PythonError>`
+has a formatted traceback, but no reference to the original Python exception.
+The original exception has references to the stack frame and leaking it will
+leak all the local variables from that stack frame. The actual Python exception
+will be stored in
+[`sys.last_value`](https://docs.python.org/3/library/sys.html#sys.last_value) so
+if you need access to it (for instance to produce a traceback with certain
+functions filtered out), use that.
+
+`````{admonition} Avoid Stack Frames
+:class: warning
+If you make a {any}`PyProxy` of ``sys.last_value``, you should be especially
+careful to {any}`destroy() <PyProxy.destroy>` it when you are done with it or
+you may leak a large amount of memory if you don't.
+`````
+The easiest way is to only handle the exception in Python:
+```pyodide
+pyodide.runPython(`
+def reformat_exception():
+    from traceback import format_exception
+    # Format a modified exception here
+    # this just prints it normally but you could for instance filter some frames
+    return "".join(
+        traceback.format_exception(sys.last_type, sys.last_value, sys.last_traceback)
+    )
+`);
+let reformat_exception = pyodide.globals.get("reformat_exception");
+try {
+    pyodide.runPython(some_code);
+} catch(e){
+    // replace error message
+    e.message = reformat_exception();
+    throw e;
+}
+```

src/pyodide-py/_pyodide/_core.py

@@ -14,9 +14,14 @@ try:
     # From jsproxy.c
     class JsException(Exception):
         """
-        A wrapper around a Javascript ``Error`` to allow the ``Error`` to be thrown in Python.
+        A wrapper around a Javascript Error to allow it to be thrown in Python.
+        See :ref:`type-translations-errors`.
         """
 
+        @property
+        def js_error(self):
+            """The original Javascript error"""
+
     class JsProxy:
         """A proxy to make a Javascript object behave like a Python object
 

src/pyodide.js

@@ -393,7 +393,8 @@ globalThis.loadPyodide = async function(config = {}) {
     'registerJsModule',
     'unregisterJsModule',
     'setInterruptBuffer',
-    'pyodide_py'
+    'pyodide_py',
+    'PythonError',
   ];
   // clang-format on
 
@@ -457,7 +458,7 @@ globalThis.loadPyodide = async function(config = {}) {
    *
    * @type {PyProxy}
    */
-  Module.pyodide_py = {}; // Hack to make jsdoc behave
+  Module.pyodide_py = {}; // actually defined in runPythonSimple below
 
   /**
    *
@@ -469,7 +470,42 @@ globalThis.loadPyodide = async function(config = {}) {
    *
    * @type {PyProxy}
    */
-  Module.globals = {}; // Hack to make jsdoc behave
+  Module.globals = {}; // actually defined in runPythonSimple below
+
+  // clang-format off
+  /**
+   * A Javascript error caused by a Python exception.
+   *
+   * In order to reduce the risk of large memory leaks, the ``PythonError``
+   * contains no reference to the Python exception that caused it. You can find
+   * the actual Python exception that caused this error as `sys.last_value
+   * <https://docs.python.org/3/library/sys.html#sys.last_value>`_.
+   *
+   * See :ref:`type-translations-errors` for more information.
+   *
+   * .. admonition:: Avoid Stack Frames
+   *    :class: warning
+   *
+   *    If you make a ``PyProxy`` of ``sys.last_value``, you should be
+   *    especially careful to :any:`destroy() <PyProxy.destroy>`. You may leak a
+   *    large amount of memory including the local variables of all the stack
+   *    frames in the traceback if you don't. The easiest way is to only handle
+   *    the exception in Python.
+   *
+   * @class
+   */
+  Module.PythonError = class PythonError {
+    // actually defined in error_handling.c. TODO: would be good to move this
+    // documentation and the definition of PythonError to error_handling.js
+    constructor(){
+      /**
+       * The Python traceback.
+       * @type {string}
+       */
+      this.message;
+    }
+  };
+  // clang-format on
 
   /**
    *