Rooba
/
pyodide
mirror of https://github.com/pyodide/pyodide.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Finish InteractiveConsole revamp (#1635) 

Co-authored-by: Roman Yurchak <rth.yurchak@gmail.com>
This commit is contained in:
Hood Chatham 2021-07-23 13:33:41 +00:00 committed by GitHub
parent 15f6fd264c
commit 986016a957
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 783 additions and 652 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/conf.py
View File

@ -151,3 +151,4 @@ def delete_attrs(cls):
delete_attrs(pyodide.webloop.WebLoop)
delete_attrs(pyodide.webloop.WebLoopPolicy)
delete_attrs(pyodide.console.PyodideConsole)

11
docs/project/changelog.md
View File

@ -13,7 +13,7 @@ substitutions:
## [Unreleased]
- {{ API }} {any}`loadPyodide` no longer automatically stores the API into a
- {{ API }} {any}`loadPyodide <globalThis.loadPyodide>` no longer automatically stores the API into a
global variable called `pyodide`. To get old behavior, say `globalThis.pyodide
= await loadPyodide({...})`.
{pr}`1597`
@ -38,6 +38,13 @@ substitutions:
type. This is particularly important if the error is a `KeyboardInterrupt`.
{pr}`1447`
- {{ Enhancement }} Added {any}`Console` class closely based on the Python standard
library `code.InteractiveConsole` but with support for top level await and
stream redirection. Also added the subclass {any}`PyodideConsole` which
automatically uses {any}`pyodide.loadPackagesFromImports` on the code before running
it.
{pr}`1125`, {pr}`1155`, {pr}`1635`
- {{ Update }} Pyodide now runs Python 3.9.5.
{pr}`1637`
@ -59,7 +66,7 @@ substitutions:
- The following standard library modules are now available as standalone packages
- distlib
They are loaded by default in {any}`globalThis.loadPyodide`, however this behavior
They are loaded by default in {any}`loadPyodide <globalThis.loadPyodide>`, however this behavior
can be disabled with the `fullStdLib` parameter set to `false`.
All optional stdlib modules can then be loaded as needed with
{any}`pyodide.loadPackage`. {pr}`1543`

479
src/py/_pyodide/console.py Normal file
View File

@ -0,0 +1,479 @@
import ast
import asyncio
from asyncio import ensure_future, Future
from codeop import Compile, CommandCompiler, _features # type: ignore
from contextlib import (
contextmanager,
redirect_stdout,
redirect_stderr,
ExitStack,
)
from contextlib import _RedirectStream # type: ignore
import rlcompleter
import platform
import sys
import traceback
from typing import Literal
from typing import (
Optional,
Callable,
Any,
List,
Tuple,
Union,
Tuple,
)
from _pyodide._base import should_quiet, CodeRunner
__all__ = ["repr_shorten", "BANNER", "Console", "PyodideConsole"]
def _banner():
"""A banner similar to the one printed by the real Python interpreter."""
# copied from https://github.com/python/cpython/blob/799f8489d418b7f9207d333eac38214931bd7dcc/Lib/code.py#L214
cprt = 'Type "help", "copyright", "credits" or "license" for more information.'
version = platform.python_version()
build = f"({', '.join(platform.python_build())})"
return f"Python {version} {build} on WebAssembly VM\n{cprt}"
BANNER = _banner()
del _banner
class redirect_stdin(_RedirectStream):
_stream = "stdin"
class _WriteStream:
"""A utility class so we can specify our own handlers for writes to sdout, stderr"""
def __init__(self, write_handler, name=None):
self.write_handler = write_handler
self.name = name
def write(self, text):
self.write_handler(text)
def flush(self):
pass
class _ReadStream:
"""A utility class so we can specify our own handler for reading from stdin"""
def __init__(self, read_handler, name=None):
self.read_handler = read_handler
self.name = name
def readline(self, n=-1):
return self.read_handler(n)
def flush(self):
pass
class _Compile(Compile):
"""Compile code with CodeRunner, and remember future imports
Instances of this class behave much like the built-in compile function,
but if one is used to compile text containing a future statement, it
"remembers" and compiles all subsequent program texts with the statement in
force. It uses CodeRunner instead of the built in compile.
"""
def __init__(
self,
*,
return_mode="last_expr",
quiet_trailing_semicolon=True,
flags=0x0,
):
super().__init__()
self.flags |= flags
self.return_mode = return_mode
self.quiet_trailing_semicolon = quiet_trailing_semicolon
def __call__(self, source, filename, symbol) -> CodeRunner: # type: ignore
return_mode = self.return_mode
if self.quiet_trailing_semicolon and should_quiet(source):
return_mode = None
code_runner = CodeRunner(
source,
mode=symbol,
filename=filename,
return_mode=return_mode,
flags=self.flags,
).compile()
for feature in _features:
if code_runner.code.co_flags & feature.compiler_flag:
self.flags |= feature.compiler_flag
return code_runner
class _CommandCompiler(CommandCompiler):
"""Compile code with CodeRunner, and remember future imports, return None if
code is incomplete.
Instances of this class have __call__ methods identical in signature to
compile; the difference is that if the instance compiles program text
containing a __future__ statement, the instance 'remembers' and compiles all
subsequent program texts with the statement in force.
If the source is determined to be incomplete, will suppress the SyntaxError
and return ``None``.
"""
def __init__(
self,
*,
return_mode="last_expr",
quiet_trailing_semicolon=True,
flags=0x0,
):
self.compiler = _Compile(
return_mode=return_mode,
quiet_trailing_semicolon=quiet_trailing_semicolon,
flags=flags,
)
def __call__(self, source, filename="<console>", symbol="single") -> Optional[CodeRunner]: # type: ignore
return super().__call__(source, filename, symbol) # type: ignore
INCOMPLETE: Literal["incomplete"] = "incomplete"
SYNTAX_ERROR: Literal["syntax-error"] = "syntax-error"
COMPLETE: Literal["complete"] = "complete"
class ConsoleFuture(Future):
"""A future with extra fields used as the return value for :any:`Console` apis.
Attributes
----------
syntax_check : str
One of ``"incomplete"``, ``"syntax-error"``, or ``"complete"`. If the value is
``"incomplete"`` then the future has already been resolved with result equal to
``None``. If the value is ``"syntax-error"``, the ``Future`` has already been
rejected with a ``SyntaxError``. If the value is ``"complete"``, then the input
complete and syntactically correct.
formatted_error : str
If the ``Future`` is rejected, this will be filled with a formatted version of
the code. This is a convenience that simplifies code and helps to avoid large
memory leaks when using from Javascript.
"""
def __init__(
self,
syntax: Union[
Literal["incomplete"], Literal["syntax-error"], Literal["complete"]
],
):
super().__init__()
self.syntax_check: Union[
Literal["incomplete"], Literal["syntax-error"], Literal["complete"]
] = syntax
self.formatted_error: Optional[str] = None
class Console:
"""Interactive Pyodide console
An interactive console based on the Python standard library
`code.InteractiveConsole` that manages stream redirections and asynchronous
execution of the code.
The stream callbacks can be modified directly as long as
`persistent_stream_redirection` isn't in effect.
Parameters
----------
globals : ``dict``
The global namespace in which to evaluate the code. Defaults to a new empty dictionary.
stdout_callback : ``Callable[[str], None]``
Function to call at each write to ``sys.stdout``. Defaults to ``None``.
stderr_callback : ``Callable[[str], None]``
Function to call at each write to ``sys.stderr``. Defaults to ``None``.
stdin_callback : ``Callable[[str], None]``
Function to call at each read from ``sys.stdin``. Defaults to ``None``.
persistent_stream_redirection : ``bool``
Should redirection of standard streams be kept between calls to :any:`runcode <Console.runcode>`?
Defaults to ``False``.
filename : ``str``
The file name to report in error messages. Defaults to ``<console>``.
Attributes
----------
globals : ``Dict[str, Any]``
The namespace used as the global
stdout_callback : ``Callback[[str], None]``
Function to call at each write to ``sys.stdout``.
stderr_callback : ``Callback[[str], None]``
Function to call at each write to ``sys.stderr``.
stdin_callback : ``Callback[[str], None]``
Function to call at each read from ``sys.stdin``.
buffer : ``List[str]``
The list of strings that have been :any:`pushed <Console.push>` to the console.
completer_word_break_characters : ``str``
The set of characters considered by :any:`complete <Console.complete>` to be word breaks.
"""
def __init__(
self,
globals: Optional[dict] = None,
*,
stdout_callback: Optional[Callable[[str], None]] = None,
stderr_callback: Optional[Callable[[str], None]] = None,
stdin_callback: Optional[Callable[[str], None]] = None,
persistent_stream_redirection: bool = False,
filename: str = "<console>",
):
if globals is None:
globals = {"__name__": "__console__", "__doc__": None}
self.globals = globals
self._stdout = None
self._stderr = None
self.stdout_callback = stdout_callback
self.stderr_callback = stderr_callback
self.stdin_callback = stdin_callback
self.filename = filename
self.buffer: List[str] = []
self._lock = asyncio.Lock()
self._streams_redirected = False
self._stream_generator = None # track persistent stream redirection
if persistent_stream_redirection:
self.persistent_redirect_streams()
self._completer = rlcompleter.Completer(self.globals) # type: ignore
# all nonalphanums except '.'
# see https://github.com/python/cpython/blob/a4258e8cd776ba655cc54ba54eaeffeddb0a267c/Modules/readline.c#L1211
self.completer_word_break_characters = (
""" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?"""
)
self._compile = _CommandCompiler(flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) # type: ignore
def persistent_redirect_streams(self):
"""Redirect stdin/stdout/stderr persistently"""
if self._stream_generator:
return
self._stream_generator = self._stdstreams_redirections_inner()
next(self._stream_generator) # trigger stream redirection
# streams will be reverted to normal when self._stream_generator is destroyed.
def persistent_restore_streams(self):
"""Restore stdin/stdout/stderr if they have been persistently redirected"""
# allowing _stream_generator to be garbage collected restores the streams
self._stream_generator = None
@contextmanager
def redirect_streams(self):
"""A context manager to redirect standard streams.
This supports nesting."""
yield from self._stdstreams_redirections_inner()
def _stdstreams_redirections_inner(self):
"""This is the generator which implements redirect_streams and the stdstreams_redirections"""
# already redirected?
if self._streams_redirected:
yield
return
redirects = []
if self.stdout_callback:
redirects.append(
redirect_stdout(
_WriteStream(self.stdout_callback, name=sys.stdout.name)
)
)
if self.stderr_callback:
redirects.append(
redirect_stderr(
_WriteStream(self.stderr_callback, name=sys.stderr.name)
)
)
if self.stdin_callback:
redirects.append(
redirect_stdin(_ReadStream(self.stdin_callback, name=sys.stdin.name))
)
try:
self._streams_redirected = True
with ExitStack() as stack:
for redirect in redirects:
stack.enter_context(redirect)
yield
finally:
self._streams_redirected = False
def runsource(self, source: str, filename: str = "<console>") -> ConsoleFuture:
"""Compile and run source code in the interpreter.
Returns
-------
:any:`ConsoleFuture`
"""
try:
code = self._compile(source, filename, "single")
except (OverflowError, SyntaxError, ValueError) as e:
# Case 1
res = ConsoleFuture(SYNTAX_ERROR)
res.set_exception(e)
res.formatted_error = self.formatsyntaxerror(e)
return res
if code is None:
res = ConsoleFuture(INCOMPLETE)
res.set_result(None)
return res
res = ConsoleFuture(COMPLETE)
def done_cb(fut):
exc = fut.exception()
if exc:
res.formatted_error = self.formattraceback(exc)
res.set_exception(exc)
exc = None
else:
res.set_result(fut.result())
ensure_future(self.runcode(source, code)).add_done_callback(done_cb)
return res
async def runcode(self, source: str, code: CodeRunner) -> Any:
"""Execute a code object and return the result."""
async with self._lock:
with self.redirect_streams():
try:
return await code.run_async(self.globals)
finally:
sys.stdout.flush()
sys.stderr.flush()
def formatsyntaxerror(self, e: Exception) -> str:
"""Format the syntax error that just occurred.
This doesn't include a stack trace because there isn't one. The actual
error object is stored into `sys.last_value`.
"""
sys.last_type = type(e)
sys.last_value = e
sys.last_traceback = None
try:
return "".join(traceback.format_exception_only(type(e), e))
finally:
e = None # type: ignore
def num_frames_to_keep(self, tb):
keep_frames = False
kept_frames = 0
# Try to trim out stack frames inside our code
for (frame, _) in traceback.walk_tb(tb):
keep_frames = keep_frames or frame.f_code.co_filename == "<console>"
keep_frames = keep_frames or frame.f_code.co_filename == "<exec>"
if keep_frames:
kept_frames += 1
return kept_frames
def formattraceback(self, e: Exception) -> str:
"""Format the exception that just occurred.
The actual error object is stored into `sys.last_value`.
"""
print("!!!!!!!!!!!")
print("formattraceback!")
try:
sys.last_type = type(e)
sys.last_value = e
sys.last_traceback = e.__traceback__
nframes = self.num_frames_to_keep(e.__traceback__)
return "".join(
traceback.format_exception(type(e), e, e.__traceback__, -nframes)
)
finally:
e = None # type: ignore
def push(self, line: str) -> ConsoleFuture:
"""Push a line to the interpreter.
The line should not have a trailing newline; it may have internal
newlines. The line is appended to a buffer and the interpreter's
runsource() method is called with the concatenated contents of the
buffer as source. If this indicates that the command was executed or
invalid, the buffer is reset; otherwise, the command is incomplete, and
the buffer is left as it was after the line was appended.
The return value is the result of calling :any:`Console.runsource` on the current buffer
contents.
"""
self.buffer.append(line)
source = "\n".join(self.buffer)
result = self.runsource(source, self.filename)
if result.syntax_check != INCOMPLETE:
self.buffer = []
return result
def complete(self, source: str) -> Tuple[List[str], int]:
"""Use Python's rlcompleter to complete the source string using the :any:`globals <Console.globals>` namespace.
Finds last "word" in the source string and completes it with rlcompleter. Word
breaks are determined by the set of characters in
:any:`completer_word_break_characters <Console.completer_word_break_characters>`.
Parameters
----------
source : str
The source string to complete at the end.
Returns
-------
completions : List[str]
A list of completion strings.
start : int
The index where completion starts.
Examples
--------
>>> shell = Console()
>>> shell.complete("str.isa")
(['str.isalnum(', 'str.isalpha(', 'str.isascii('], 0)
>>> shell.complete("a = 5 ; str.isa")
(['str.isalnum(', 'str.isalpha(', 'str.isascii('], 8)
"""
start = max(map(source.rfind, self.completer_word_break_characters)) + 1
source = source[start:]
if "." in source:
completions = self._completer.attr_matches(source) # type: ignore
else:
completions = self._completer.global_matches(source) # type: ignore
return completions, start
def repr_shorten(
value: Any, limit: int = 1000, split: Optional[int] = None, separator: str = "..."
) -> str:
"""Compute the string representation of ``value`` and shorten it
if necessary.
If it is longer than ``limit`` then return the firsts ``split``
characters and the last ``split`` characters seperated by '...'.
Default value for ``split`` is `limit // 2`.
"""
if split is None:
split = limit // 2
text = repr(value)
if len(text) > limit:
text = f"{text[:split]}{separator}{text[-split:]}"
return text

1
src/py/pyodide/__init__.py
View File

@ -49,6 +49,7 @@ __all__ = [
"unregister_js_module",
"create_once_callable",
"create_proxy",
"console",
"should_quiet",
"ConversionError",
"destroy_proxies",

385
src/py/pyodide/console.py
View File

@ -1,388 +1,39 @@
import ast
import asyncio
import code
from codeop import Compile, CommandCompiler, _features # type: ignore
from contextlib import (
contextmanager,
redirect_stdout,
redirect_stderr,
ExitStack,
)
from contextlib import _RedirectStream # type: ignore
import rlcompleter
import platform
import sys
import traceback
from typing import Optional, Callable, Any, List, Tuple
from _pyodide.console import Console, repr_shorten, ConsoleFuture
import _pyodide.console
BANNER = _pyodide.console.BANNER # type: ignore
from _pyodide._base import CodeRunner
from _pyodide._base import eval_code_async, should_quiet, CodeRunner
from ._core import IN_BROWSER
# this import can fail when we are outside a browser (e.g. for tests)
if IN_BROWSER:
from pyodide_js import loadPackagesFromImports as _load_packages_from_imports
from asyncio import ensure_future
else:
from asyncio import Future
def ensure_future(co): # type: ignore
fut = Future()
try:
co.send(None)
except StopIteration as v:
result = v.args[0] if v.args else None
fut.set_result(result)
except BaseException as e:
fut.set_exception(e)
else:
raise Exception("coroutine didn't finish in one pass")
return fut
async def _load_packages_from_imports(*args):
pass
__all__ = ["repr_shorten", "BANNER"]
__all__ = ["Console", "PyodideConsole", "Banner", "repr_shorten"]
def _banner():
"""A banner similar to the one printed by the real Python interpreter."""
# copied from https://github.com/python/cpython/blob/799f8489d418b7f9207d333eac38214931bd7dcc/Lib/code.py#L214
cprt = 'Type "help", "copyright", "credits" or "license" for more information.'
version = platform.python_version()
build = f"({', '.join(platform.python_build())})"
return f"Python {version} {build} on WebAssembly VM\n{cprt}"
class PyodideConsole(Console):
"""A subclass of :any:`Console` that uses :any:`pyodide.loadPackagesFromImports` before running the code."""
async def runcode(self, source: str, code: CodeRunner) -> ConsoleFuture:
"""Execute a code object.
BANNER = _banner()
del _banner
class redirect_stdin(_RedirectStream):
_stream = "stdin"
class _WriteStream:
"""A utility class so we can specify our own handlers for writes to sdout, stderr"""
def __init__(self, write_handler, name=None):
self.write_handler = write_handler
self.name = name
def write(self, text):
self.write_handler(text)
def flush(self):
pass
class _ReadStream:
"""A utility class so we can specify our own handler for reading from stdin"""
def __init__(self, read_handler, name=None):
self.read_handler = read_handler
self.name = name
def readline(self, n=-1):
return self.read_handler(n)
def flush(self):
pass
class _CodeRunnerCompile(Compile):
"""Compile code with CodeRunner, and remember future imports
Instances of this class behave much like the built-in compile function,
but if one is used to compile text containing a future statement, it
"remembers" and compiles all subsequent program texts with the statement in
force. It uses CodeRunner instead of the built in compile.
"""
def __init__(
self,
*,
return_mode="last_expr",
quiet_trailing_semicolon=True,
flags=0x0,
):
super().__init__()
self.flags |= flags
self.return_mode = return_mode
self.quiet_trailing_semicolon = quiet_trailing_semicolon
def __call__(self, source, filename, symbol) -> CodeRunner: # type: ignore
return_mode = self.return_mode
if self.quiet_trailing_semicolon and should_quiet(source):
return_mode = None
code_runner = CodeRunner(
source,
mode=symbol,
filename=filename,
return_mode=return_mode,
flags=self.flags,
).compile()
for feature in _features:
if code_runner.code.co_flags & feature.compiler_flag:
self.flags |= feature.compiler_flag
return code_runner
class _CodeRunnerCommandCompiler(CommandCompiler):
"""Compile code with CodeRunner, and remember future imports, return None if
code is incomplete.
Instances of this class have __call__ methods identical in signature to
compile; the difference is that if the instance compiles program text
containing a __future__ statement, the instance 'remembers' and compiles all
subsequent program texts with the statement in force.
If the source is determined to be incomplete, will suppress the SyntaxError
and return ``None``.
"""
def __init__(
self,
*,
return_mode="last_expr",
quiet_trailing_semicolon=True,
flags=0x0,
):
self.compiler = _CodeRunnerCompile(
return_mode=return_mode,
quiet_trailing_semicolon=quiet_trailing_semicolon,
flags=flags,
)
def __call__(self, source, filename="<console>", symbol="single") -> CodeRunner: # type: ignore
return super().__call__(source, filename, symbol) # type: ignore
class _InteractiveConsole(code.InteractiveConsole):
"""Interactive Pyodide console
Base implementation for an interactive console that manages
stdout/stderr redirection. Since packages are loaded before running
code, :any:`_InteractiveConsole.runcode` returns a JS promise.
``self.stdout_callback`` and ``self.stderr_callback`` can be overloaded.
Parameters
----------
locals
Namespace to evaluate code.
stdout_callback
Function to call at each ``sys.stdout`` flush.
stderr_callback
Function to call at each ``sys.stderr`` flush.
persistent_stream_redirection
Whether or not the std redirection should be kept between calls to
``runcode``.
"""
def __init__(
self,
locals: Optional[dict] = None,
*,
stdout_callback: Optional[Callable[[str], None]] = None,
stderr_callback: Optional[Callable[[str], None]] = None,
stdin_callback: Optional[Callable[[str], None]] = None,
persistent_stream_redirection: bool = False,
):
super().__init__(locals)
self._stdout = None
self._stderr = None
self.stdout_callback = stdout_callback
self.stderr_callback = stderr_callback
self.stdin_callback = stdin_callback
self._streams_redirected = False
self._stream_generator = None # track persistent stream redirection
if persistent_stream_redirection:
self.persistent_redirect_streams()
self.run_complete: asyncio.Future = asyncio.Future()
self.run_complete.set_result(None)
self._completer = rlcompleter.Completer(self.locals) # type: ignore
# all nonalphanums except '.'
# see https://github.com/python/cpython/blob/a4258e8cd776ba655cc54ba54eaeffeddb0a267c/Modules/readline.c#L1211
self.completer_word_break_characters = (
""" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?"""
)
self.output_truncated_text = "\\n[[;orange;]<long output truncated>]\\n"
self.compile.compiler.flags |= ast.PyCF_ALLOW_TOP_LEVEL_AWAIT # type: ignore
def persistent_redirect_streams(self):
"""Redirect stdin/stdout/stderr persistently"""
if self._stream_generator:
return
self._stream_generator = self._stdstreams_redirections_inner()
next(self._stream_generator) # trigger stream redirection
# streams will be reverted to normal when self._stream_generator is destroyed.
def persistent_restore_streams(self):
"""Restore stdin/stdout/stderr if they have been persistently redirected"""
# allowing _stream_generator to be garbage collected restores the streams
self._stream_generator = None
@contextmanager
def redirect_streams(self):
"""A context manager to redirect standard streams.
This supports nesting."""
yield from self._stdstreams_redirections_inner()
def _stdstreams_redirections_inner(self):
"""This is the generator which implements redirect_streams and the stdstreams_redirections"""
# already redirected?
if self._streams_redirected:
yield
return
redirects = []
if self.stdout_callback:
redirects.append(
redirect_stdout(
_WriteStream(self.stdout_callback, name=sys.stdout.name)
)
)
if self.stderr_callback:
redirects.append(
redirect_stderr(
_WriteStream(self.stderr_callback, name=sys.stderr.name)
)
)
if self.stdin_callback:
redirects.append(
redirect_stdin(_ReadStream(self.stdin_callback, name=sys.stdin.name))
)
try:
self._streams_redirected = True
with ExitStack() as stack:
for redirect in redirects:
stack.enter_context(redirect)
yield
finally:
self._streams_redirected = False
def flush_all(self):
"""Force stdout/stderr flush."""
with self.redirect_streams():
sys.stdout.flush()
sys.stderr.flush()
def runsource(self, *args, **kwargs):
"""Force streams redirection.
Syntax errors are not thrown by :any:`_InteractiveConsole.runcode` but
here in :any:`_InteractiveConsole.runsource`. This is why we force
redirection here since doing twice is not an issue.
"""
with self.redirect_streams():
return super().runsource(*args, **kwargs)
def runcode(self, code):
"""Load imported packages then run code, async.
To achieve nice result representation, the interactive console is fully
implemented in Python. The interactive console api is synchronous, but
we want to implement asynchronous package loading and top level await.
Thus, instead of blocking like it normally would, this this function
sets the future ``self.run_complete``. If you need the result of the
computation, you should await for it.
"""
source = "\n".join(self.buffer)
self.run_complete = ensure_future(
self.load_packages_and_run(self.run_complete, source)
)
def num_frames_to_keep(self, tb):
keep_frames = False
kept_frames = 0
# Try to trim out stack frames inside our code
for (frame, _) in traceback.walk_tb(tb):
keep_frames = keep_frames or frame.f_code.co_filename == "<console>"
keep_frames = keep_frames or frame.f_code.co_filename == "<exec>"
if keep_frames:
kept_frames += 1
return kept_frames
async def load_packages_and_run(self, run_complete, source):
try:
await run_complete
except BaseException:
# Throw away old error
pass
with self.redirect_streams():
await _load_packages_from_imports(source)
try:
result = await eval_code_async(
source, self.locals, filename="<console>"
)
except BaseException as e:
nframes = self.num_frames_to_keep(e.__traceback__)
traceback.print_exception(type(e), e, e.__traceback__, -nframes)
raise e
else:
self.display(result)
# in CPython's REPL, flush is performed
# by input(prompt) at each new prompt ;
# since we are not using input, we force
# flushing here
self.flush_all()
return result
def complete(self, source: str) -> Tuple[List[str], int]:
"""Use CPython's rlcompleter to complete a source from local namespace.
You can use ``completer_word_break_characters`` to get/set the
way ``source`` is splitted to find the last part to be completed.
Parameters
----------
source
The source string to complete at the end.
All exceptions are caught except SystemExit, which is reraised.
Returns
-------
completions
A list of completion strings.
start
The index where completion starts.
Examples
--------
>>> shell = _InteractiveConsole()
>>> shell.complete("str.isa")
(['str.isalnum(', 'str.isalpha(', 'str.isascii('], 0)
>>> shell.complete("a = 5 ; str.isa")
(['str.isalnum(', 'str.isalpha(', 'str.isascii('], 8)
The return value is a dependent sum type with the following possibilities:
* `("success", result : Any)` -- the code executed successfully
* `("exception", message : str)` -- An exception occurred. `message` is the
result of calling :any:`Console.formattraceback`.
"""
start = max(map(source.rfind, self.completer_word_break_characters)) + 1
source = source[start:]
if "." in source:
completions = self._completer.attr_matches(source) # type: ignore
else:
completions = self._completer.global_matches(source) # type: ignore
return completions, start
from pyodide_js import loadPackagesFromImports
def display(self, value):
if value is None:
return
print(repr_shorten(value, separator=self.output_truncated_text))
def repr_shorten(
value: Any, limit: int = 1000, split: Optional[int] = None, separator: str = "..."
):
"""Compute the string representation of ``value`` and shorten it
if necessary.
If it is longer than ``limit`` then return the firsts ``split``
characters and the last ``split`` characters seperated by '...'.
Default value for ``split`` is `limit // 2`.
"""
if split is None:
split = limit // 2
text = repr(value)
if len(text) > limit:
text = f"{text[:split]}{separator}{text[-split:]}"
return text
await loadPackagesFromImports(source)
return await super().runcode(source, code)

3
src/py/pyodide/webloop.py
View File

@ -390,3 +390,6 @@ class WebLoopPolicy(asyncio.DefaultEventLoopPolicy): # type: ignore
def set_event_loop(self, loop: asyncio.AbstractEventLoop):
"""Set the current event loop"""
self._default_loop = loop
__all__ = ["WebLoop", "WebLoopPolicy"]

91
src/templates/console.html
View File

@ -17,9 +17,11 @@
</head>
<body>
<script>
"use strict";
function sleep(s) {
return new Promise((resolve) => setTimeout(resolve, s));
}
async function main() {
globalThis.pyodide = await loadPyodide({
indexURL: "{{ PYODIDE_BASE_URL }}",
@ -29,21 +31,14 @@
`
import sys
import js
from pyodide.console import _InteractiveConsole, BANNER
from pyodide.console import PyodideConsole, repr_shorten, BANNER
import __main__
class PyConsole(_InteractiveConsole):
def __init__(self):
super().__init__(
__main__.__dict__,
persistent_stream_redirection=False,
)
BANNER = "Welcome to the Pyodide terminal emulator 🐍\\n" + BANNER
js.pyconsole = PyConsole()
js.pyconsole = PyodideConsole(__main__.__dict__)
`,
namespace
);
let repr_shorten = namespace.get("repr_shorten");
let banner = namespace.get("BANNER");
namespace.destroy();
@ -60,31 +55,43 @@
async function interpreter(command) {
let unlock = await lock();
try {
term.pause();
// multiline should be splitted (useful when pasting)
for (const c of command.split("\n")) {
let run_complete = pyconsole.run_complete;
try {
const incomplete = pyconsole.push(c);
term.set_prompt(incomplete ? ps2 : ps1);
let r = await run_complete;
if (pyodide.isPyProxy(r)) {
r.destroy();
}
} catch (e) {
if (e.name !== "PythonError") {
term.error(e);
throw e;
}
}
run_complete.destroy();
term.pause();
// multiline should be splitted (useful when pasting)
for (const c of command.split("\n")) {
let fut = pyconsole.push(c);
term.set_prompt(fut.syntax_check === "incomplete" ? ps2 : ps1);
switch (fut.syntax_check) {
case "syntax-error":
term.error(fut.formatted_error.trimEnd());
continue;
case "incomplete":
continue;
case "complete":
break;
default:
throw new Error(`Unexpected type ${ty}`);
}
} finally {
term.resume();
await sleep(10);
unlock();
// complete case, get result / error and print it.
try {
let value = await fut;
if (value !== undefined) {
term.echo(
repr_shorten.callKwargs(value, {
separator: "\\n[[;orange;]<long output truncated>]\\n",
})
);
}
if (value.destroy) {
value.destroy();
}
} catch (e) {
term.error(fut.formatted_error.trimEnd());
}
fut.destroy();
}
term.resume();
await sleep(10);
unlock();
}
let term = $("body").terminal(interpreter, {
@ -97,22 +104,8 @@
});
window.term = term;
pyconsole.stdout_callback = (s) => term.echo(s, { newline: false });
let url_re =
/(\bhttps?:\/\/(?:(?:(?!&[^;]+;)|(?=&amp;))[^\s"'<>\][)])+)/gi;
pyconsole.stderr_callback = function (s) {
if (s.endsWith("\n")) {
term.error(s.slice(0, -1));
return;
}
if (s === "") {
return;
}
s = $.terminal
.escape_brackets(s)
.replace(/\\$/, "&#92;")
.replace(url_re, "]$1[[;;;terminal-error]");
s = `[[;;;terminal-error]${s}]`.replace(/(\n\s*)]/, "]$1");
term.echo(s, { newline: false });
pyconsole.stderr_callback = (s) => {
term.error(s.trimEnd());
};
term.ready = Promise.resolve();
pyodide._module.on_fatal = async (e) => {

457
src/tests/test_console.py
View File

@ -1,18 +1,24 @@
import asyncio
import pytest
from pathlib import Path
import time
import sys
from pyodide_build.testing import run_in_pyodide
from conftest import selenium_common
sys.path.append(str(Path(__file__).resolve().parents[2] / "src" / "py"))
from pyodide import console, CodeRunner # noqa: E402
from pyodide.console import _CodeRunnerCompile, _CodeRunnerCommandCompiler # noqa: E402
from pyodide import CodeRunner # noqa: E402
from _pyodide.console import (
Console,
_Compile,
_CommandCompiler,
) # noqa: E402
from _pyodide import console
def test_command_compiler():
c = _CodeRunnerCompile()
c = _Compile()
with pytest.raises(SyntaxError, match="unexpected EOF while parsing"):
c("def test():\n 1", "<input>", "single")
assert isinstance(c("def test():\n 1\n", "<input>", "single"), CodeRunner)
@ -23,7 +29,7 @@ def test_command_compiler():
)
assert isinstance(c("1<>2", "<input>", "single"), CodeRunner)
c = _CodeRunnerCommandCompiler()
c = _CommandCompiler()
assert c("def test():\n 1", "<input>", "single") is None
assert isinstance(c("def test():\n 1\n", "<input>", "single"), CodeRunner)
with pytest.raises(SyntaxError, match="invalid syntax"):
@ -34,7 +40,7 @@ def test_command_compiler():
assert isinstance(c("1<>2", "<input>", "single"), CodeRunner)
def test_stream_redirection():
def test_write_stream():
my_buffer = ""
def callback(string):
@ -49,114 +55,7 @@ def test_stream_redirection():
assert my_buffer == "foo\nbar\n"
@pytest.fixture
def safe_sys_redirections():
redirected = sys.stdout, sys.stderr, sys.displayhook
try:
yield
finally:
sys.stdout, sys.stderr, sys.displayhook = redirected
def test_interactive_console_streams(safe_sys_redirections):
my_stdout = ""
my_stderr = ""
orig_sys_stdout_name = sys.stdout.name
orig_sys_stderr_name = sys.stderr.name
def stdout_callback(string):
nonlocal my_stdout
my_stdout += string
def stderr_callback(string):
nonlocal my_stderr
my_stderr += string
##########################
# Persistent redirection #
##########################
shell = console._InteractiveConsole(
stdout_callback=stdout_callback,
stderr_callback=stderr_callback,
persistent_stream_redirection=True,
)
# std names
assert sys.stdout.name == orig_sys_stdout_name
assert sys.stderr.name == orig_sys_stderr_name
# std redirections
print("foo")
assert my_stdout == "foo\n"
print("bar", file=sys.stderr)
assert my_stderr == "bar\n"
shell.push("print('foobar')")
assert my_stdout == "foo\nfoobar\n"
shell.push("print('foobar')")
assert my_stdout == "foo\nfoobar\nfoobar\n"
shell.push("1+1")
assert my_stdout == "foo\nfoobar\nfoobar\n2\n"
assert shell.run_complete.result() == 2
my_stderr = ""
shell.push("raise Exception('hi')")
assert (
my_stderr
== 'Traceback (most recent call last):\n File "<console>", line 1, in <module>\nException: hi\n'
)
assert shell.run_complete.exception() is not None
my_stderr = ""
shell.push("1+1")
assert my_stderr == ""
assert shell.run_complete.result() == 2
del shell
import gc
gc.collect()
my_stdout = ""
my_stderr = ""
print("bar")
assert my_stdout == ""
print("foo", file=sys.stdout)
assert my_stderr == ""
##############################
# Non persistent redirection #
##############################
shell = console._InteractiveConsole(
stdout_callback=stdout_callback,
stderr_callback=stderr_callback,
persistent_stream_redirection=False,
)
print("foo")
assert my_stdout == ""
shell.push("print('foobar')")
assert my_stdout == "foobar\n"
print("bar")
assert my_stdout == "foobar\n"
shell.push("print('foobar')")
assert my_stdout == "foobar\nfoobar\n"
shell.push("import sys")
shell.push("print('foobar', file=sys.stderr)")
assert my_stderr == "foobar\n"
shell.push("1+1")
assert my_stdout == "foobar\nfoobar\n2\n"
def test_repr(safe_sys_redirections):
def test_repr():
sep = "..."
for string in ("x" * 10 ** 5, "x" * (10 ** 5 + 1)):
for limit in (9, 10, 100, 101):
@ -165,89 +64,9 @@ def test_repr(safe_sys_redirections):
) == 2 * (limit // 2) + len(sep)
@pytest.fixture
def safe_selenium_sys_redirections(selenium):
# Import console early since it makes three global hiwire allocations, and we don't want to anger
# the memory leak checker
selenium.run_js("pyodide._module.runPythonSimple(`from pyodide import console`)")
selenium.run_js(
"pyodide._module.runPythonSimple(`import sys; _redirected = sys.stdout, sys.stderr, sys.displayhook`)"
)
try:
yield
finally:
selenium.run_js(
"pyodide._module.runPythonSimple(`sys.stdout, sys.stderr, sys.displayhook = _redirected`)"
)
def test_interactive_console(selenium, safe_selenium_sys_redirections):
selenium.run(
"""
import sys
from pyodide.console import _InteractiveConsole
result = None
def display(value):
global result
result = value
shell = _InteractiveConsole()
shell.display = display
"""
)
selenium.run("shell.push('x = 5')")
selenium.run("shell.push('x')")
selenium.run_js("await pyodide.runPythonAsync('await shell.run_complete');")
assert selenium.run("result") == 5
selenium.run("shell.push('x ** 2')")
selenium.run_js("await pyodide.runPythonAsync('await shell.run_complete');")
assert selenium.run("result") == 25
selenium.run("shell.push('def f(x):')")
selenium.run("shell.push(' return x*x + 1')")
selenium.run("shell.push('')")
selenium.run("shell.push('str([f(x) for x in range(5)])')")
selenium.run_js("await pyodide.runPythonAsync('await shell.run_complete');")
assert selenium.run("result") == str([1, 2, 5, 10, 17])
selenium.run("shell.push('def factorial(n):')")
selenium.run("shell.push(' if n < 2:')")
selenium.run("shell.push(' return 1')")
selenium.run("shell.push(' else:')")
selenium.run("shell.push(' return n * factorial(n - 1)')")
selenium.run("shell.push('')")
selenium.run("shell.push('factorial(10)')")
selenium.run_js("await pyodide.runPythonAsync('await shell.run_complete');")
assert selenium.run("result") == 3628800
# with package load
selenium.run("shell.push('import pytz')")
selenium.run("shell.push('pytz.utc.zone')")
selenium.run_js("await pyodide.runPythonAsync('await shell.run_complete');")
assert selenium.run("result") == "UTC"
def test_completion(selenium, safe_selenium_sys_redirections):
selenium.run(
"""
from pyodide import console
shell = console._InteractiveConsole()
"""
)
assert selenium.run(
"""
[completions, start] = shell.complete('a')
[tuple(completions), start]
"""
) == [
def test_completion():
shell = Console({"a_variable": 7})
shell.complete("a") == (
[
"and ",
"as ",
@ -258,48 +77,219 @@ def test_completion(selenium, safe_selenium_sys_redirections):
"all(",
"any(",
"ascii(",
"a_variable",
],
0,
]
)
assert selenium.run(
"""
[completions, start] = shell.complete('a = 0 ; print.__g')
[tuple(completions), start]
"""
) == [
assert shell.complete("a = 0 ; print.__g") == (
[
"print.__ge__(",
"print.__getattribute__(",
"print.__gt__(",
],
8,
]
def test_interactive_console_top_level_await(selenium, safe_selenium_sys_redirections):
selenium.run(
"""
import sys
from pyodide.console import _InteractiveConsole
result = None
def display(value):
global result
result = value
shell = _InteractiveConsole()
shell.display = display
"""
)
selenium.run("shell.push('from js import fetch')")
time.sleep(0.2)
selenium.run("""shell.push("await (await fetch('packages.json')).json()")""")
time.sleep(0.2)
res = selenium.run("result")
assert isinstance(res, dict)
assert res["dependencies"]["micropip"] == ["pyparsing", "packaging", "distutils"]
def test_interactive_console():
shell = Console()
def assert_incomplete(input):
res = shell.push(input)
assert res.syntax_check == "incomplete"
async def get_result(input):
res = shell.push(input)
assert res.syntax_check == "complete"
return await res
async def test():
assert await get_result("x = 5") == None
assert await get_result("x") == 5
assert await get_result("x ** 2") == 25
assert_incomplete("def f(x):")
assert_incomplete(" return x*x + 1")
assert await get_result("") == None
assert await get_result("[f(x) for x in range(5)]") == [1, 2, 5, 10, 17]
assert_incomplete("def factorial(n):")
assert_incomplete(" if n < 2:")
assert_incomplete(" return 1")
assert_incomplete(" else:")
assert_incomplete(" return n * factorial(n - 1)")
assert await get_result("") == None
assert await get_result("factorial(10)") == 3628800
assert await get_result("import pytz") == None
assert await get_result("pytz.utc.zone") == "UTC"
fut = shell.push("1+")
assert fut.syntax_check == "syntax-error"
assert fut.exception() is not None
assert (
fut.formatted_error
== ' File "<console>", line 1\n 1+\n ^\nSyntaxError: invalid syntax\n'
)
fut = shell.push("raise Exception('hi')")
try:
await fut
except:
assert (
fut.formatted_error
== 'Traceback (most recent call last):\n File "<console>", line 1, in <module>\nException: hi\n'
)
asyncio.get_event_loop().run_until_complete(test())
def test_top_level_await():
from asyncio import Queue, sleep
q = Queue()
shell = Console(locals())
fut = shell.push("await q.get()")
async def test():
await sleep(0.3)
assert not fut.done()
await q.put(5)
assert await fut == 5
asyncio.get_event_loop().run_until_complete(test())
@pytest.fixture
def safe_sys_redirections():
redirected = sys.stdout, sys.stderr, sys.displayhook
try:
yield
finally:
sys.stdout, sys.stderr, sys.displayhook = redirected
def test_persistent_redirection(safe_sys_redirections):
my_stdout = ""
my_stderr = ""
orig_stdout = sys.stdout
orig_stderr = sys.stderr
def stdout_callback(string):
nonlocal my_stdout
my_stdout += string
def stderr_callback(string):
nonlocal my_stderr
my_stderr += string
shell = Console(
stdout_callback=stdout_callback,
stderr_callback=stderr_callback,
persistent_stream_redirection=True,
)
# std names
assert sys.stdout.name == orig_stdout.name
assert sys.stderr.name == orig_stderr.name
# std redirections
print("foo")
assert my_stdout == "foo\n"
print("bar", file=sys.stderr)
assert my_stderr == "bar\n"
my_stderr = ""
async def get_result(input):
res = shell.push(input)
assert res.syntax_check == "complete"
return await res
async def test():
assert await get_result("print('foobar')") == None
assert my_stdout == "foo\nfoobar\n"
assert await get_result("print('foobar')") == None
assert my_stdout == "foo\nfoobar\nfoobar\n"
assert await get_result("1+1") == 2
assert my_stdout == "foo\nfoobar\nfoobar\n"
asyncio.get_event_loop().run_until_complete(test())
my_stderr = ""
shell.persistent_restore_streams()
my_stdout = ""
my_stderr = ""
print(sys.stdout, file=orig_stdout)
print("bar")
assert my_stdout == ""
print("foo", file=sys.stdout)
assert my_stderr == ""
def test_nonpersistent_redirection(safe_sys_redirections):
my_stdout = ""
my_stderr = ""
def stdout_callback(string):
nonlocal my_stdout
my_stdout += string
def stderr_callback(string):
nonlocal my_stderr
my_stderr += string
async def get_result(input):
res = shell.push(input)
assert res.syntax_check == "complete"
return await res
shell = Console(
stdout_callback=stdout_callback,
stderr_callback=stderr_callback,
persistent_stream_redirection=False,
)
print("foo")
assert my_stdout == ""
async def test():
assert await get_result("print('foobar')") == None
assert my_stdout == "foobar\n"
print("bar")
assert my_stdout == "foobar\n"
assert await get_result("print('foobar')") == None
assert my_stdout == "foobar\nfoobar\n"
assert await get_result("import sys") == None
assert await get_result("print('foobar', file=sys.stderr)") == None
assert my_stderr == "foobar\n"
assert await get_result("1+1") == 2
asyncio.get_event_loop().run_until_complete(test())
@pytest.mark.skip_refcount_check
@run_in_pyodide
async def test_console_imports():
from pyodide.console import PyodideConsole
shell = PyodideConsole()
async def get_result(input):
res = shell.push(input)
assert res.syntax_check == "complete"
return await res
assert await get_result("import pytz") == None
assert await get_result("pytz.utc.zone") == "UTC"
@pytest.fixture(params=["firefox", "chrome"], scope="function")
@ -349,14 +339,14 @@ SyntaxError: invalid syntax]`
await term.ready;
result.push([term.get_output(),
`>>> raise Exception('hi')
[[;;;terminal-error]Traceback (most recent call last):]
[[;;;terminal-error] File "<console>", line 1, in <module>]
[[;;;terminal-error]Exception: hi]`
[[;;;terminal-error]Traceback (most recent call last):
File "<console>", line 1, in <module>
Exception: hi]`
]);
term.clear();
term.exec("from _pyodide_core import trigger_fatal_error; trigger_fatal_error()");
await term.ready;
await sleep(100);
result.push([term.get_output(),
`>>> from _pyodide_core import trigger_fatal_error; trigger_fatal_error()
[[;;;terminal-error]Pyodide has suffered a fatal error. Please report this to the Pyodide maintainers.]
@ -365,7 +355,6 @@ SyntaxError: invalid syntax]`
[[;;;terminal-error]Look in the browser console for more details.]`
]);
await sleep(30);
assert(() => term.paused());
return result;
"""

7
src/tests/test_testing.py
View File

@ -14,6 +14,13 @@ def test_run_in_pyodide():
pass
@run_in_pyodide
async def test_run_in_pyodide_async():
from js import sleep
await sleep(5)
def dummy_decorator(*args, **kwargs):
def func(f):
return f