diff --git a/.prettierignore b/.prettierignore index 90aaf11d2..81f290b4a 100644 --- a/.prettierignore +++ b/.prettierignore @@ -10,3 +10,4 @@ cpython .pytest_cache .clang-format packages/libf2c/make.inc +src/js/generated_struct_info32.json diff --git a/conftest.py b/conftest.py index 310e98eaf..bed44ab6f 100644 --- a/conftest.py +++ b/conftest.py @@ -5,6 +5,7 @@ import os import pathlib import re import sys +from collections.abc import Sequence import pytest @@ -251,3 +252,18 @@ def extra_checks_test_wrapper(browser, trace_hiwire_refs, trace_pyproxies): def package_is_built(package_name): return _package_is_built(package_name, pytest.pyodide_dist_dir) + + +def strip_assertions_stderr(messages: Sequence[str]) -> list[str]: + """Strip additional messages on stderr included when ASSERTIONS=1""" + res = [] + for msg in messages: + if msg.strip() in [ + "sigaction: signal type not supported: this is a no-op.", + "Calling stub instead of siginterrupt()", + "warning: no blob constructor, cannot create blobs with mimetypes", + "warning: no BlobBuilder", + ]: + continue + res.append(msg) + return res diff --git a/docs/conf.py b/docs/conf.py index a76f637e8..5c1060128 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -61,7 +61,7 @@ autosummary_generate = True autodoc_default_flags = ["members", "inherited-members"] intersphinx_mapping = { - "python": ("https://docs.python.org/3.10", None), + "python": ("https://docs.python.org/3.11", None), "micropip": (f"https://micropip.pyodide.org/en/v{micropip.__version__}/", None), "numpy": ("https://numpy.org/doc/stable/", None), } diff --git a/docs/project/changelog.md b/docs/project/changelog.md index be9755699..37fc0a52f 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -75,6 +75,10 @@ myst: `FetchResponse.raise_for_status` to raise an `OSError` for error status codes. {pr}`3986` {pr}`4053` +- {{ Enhancement }} The `setStdin`, `setStdout` and `setStderr` APIs have been + improved with extra control and better performance. + {pr}`4035` + ### Packages - OpenBLAS has been added and scipy now uses OpenBLAS rather than CLAPACK diff --git a/docs/sphinx_pyodide/sphinx_pyodide/lexers.py b/docs/sphinx_pyodide/sphinx_pyodide/lexers.py index f49dd4e36..af129b9d2 100644 --- a/docs/sphinx_pyodide/sphinx_pyodide/lexers.py +++ b/docs/sphinx_pyodide/sphinx_pyodide/lexers.py @@ -22,9 +22,12 @@ class PyodideLexer(JavascriptLexer): ], "python-code": [ ( - rf"({quotemark})((?:\\\\|\\[^\\]|[^{quotemark}\\])*)({quotemark})", + rf"([A-Za-z.]*)({quotemark})((?:\\\\|\\[^\\]|[^{quotemark}\\])*)({quotemark})", bygroups( - Token.Literal.String, using(PythonLexer), Token.Literal.String + using(JavascriptLexer), + Token.Literal.String, + using(PythonLexer), + Token.Literal.String, ), "#pop", ) diff --git a/docs/usage/streams.md b/docs/usage/streams.md index 29847ca83..2178cfb38 100644 --- a/docs/usage/streams.md +++ b/docs/usage/streams.md @@ -7,36 +7,351 @@ Pyodide has three functions {js:func}`pyodide.setStdin`, behavior of reading from {py:data}`~sys.stdin` and writing to {py:data}`~sys.stdout` and {py:data}`~sys.stderr` respectively. -`setStdin({stdin?, isatty?, error?})` takes a function which should take zero -arguments and return either a string or an ArrayBufferView of information read -from stdin. The `isatty` argument signals whether {py:func}`isatty(stdin) ` should be true -or false. If you pass `error: true` then reading from stdin will return an -error. If `setStdin` is called with no arguments, the default value is restored. -In Node the default behavior is to read from {js:data}`process.stdin` and in the browser -it is to throw an error. +## Standard Input -`setStdout({batched?, raw?, isattty?})` sets the standard out handler and -similarly `setStderr` (same arguments) sets the stdandard error handler. If a -`raw` handler is provided then the handler is called with a `number` for each -byte of the output to stdout. The handler is expected to deal with this in -whatever way it prefers. `isattty` again controls whether {py:func}`isatty(stdout) ` -returns `true` or `false`. +(streams-stdin)= -On the other hand, a `batched` handler is only called with complete lines of -text (or when the output is flushed). A `batched` handler cannot have `isatty` -set to `true` because it is impossible to use such a handler to make something -behave like a tty. +{js:func}`pyodide.setStdin` sets the standard in handler. There are several +different ways to do this depending on the options passed to `setStdin`. -Passing neither `raw` nor `batched` sets the default behavior. In Node the -default behavior is to write directly to {js:data}`process.stdout` and -{js:data}`process.stderr` (in this case `isatty` depends on whether -{js:data}`process.stdout` and {js:data}`process.stderr` are ttys). In browser, -the default behavior is achieved with `pyodide.setStdout({batched: console.log})` -and `pyodide.setStderr({batched: console.warn})`. +### Always raise IO Error -The arguments `stdin`, `stdout`, and `stderr` to `loadPyodide` provide a -diminished amount of control compared to `setStdin`, `setStdout`, and -`setStderr`. They all set `isatty` to `false` and use batched processing for -`setStdout` and `setStderr`. In most cases, nothing is written or read to any of -these streams while Pyodide is starting, so if you need the added flexibility -you can wait until Pyodide is loaded and then use the `pyodide.setStdxxx` apis. +If we pass `{error: true}`, any read from stdin raises an I/O error. + +```js +pyodide.setStdin({ error: true }); +pyodide.runPython(` + with pytest.raises(OSError, match="I/O error"): + input() +`); +``` + +### Set the default behavior + +You can set the default behavior by calling `pyodide.setStdin()` with no +arguments. In Node the default behavior is to read directly from Node's standard +input. In the browser, the default is the same as +`pyodide.setStdin({ stdin: () => prompt() })`. + +### A stdin handler + +We can pass the options `{stdin, isatty}`. `stdin` should be a +zero-argument function which should return one of: + +1. A string which represents a full line of text (it will have a newline + appended if it does not already end in one). +2. An array buffer or Uint8Array containing utf8 encoded characters +3. A number between 0 and 255 which indicates one byte of input +4. `undefined` which indicates EOF. + +`isatty` is a boolean which +indicates whether `sys.stdin.isatty()` should return `true` or `false`. + +For example, the following class plays back a list of results. + +```js +class StdinHandler { + constructor(results, options) { + this.results = results; + this.idx = 0; + Object.assign(this, options); + } + + stdin() { + return this.results[this.idx++]; + } +} +``` + +Here it is in use: + +```pyodide +pyodide.setStdin( + new StdinHandler(["a", "bcd", "efg"]), +); +pyodide.runPython(` + assert input() == "a" + assert input() == "bcd" + assert input() == "efg" + # after this, further attempts to read from stdin will return undefined which + # indicates end of file + with pytest.raises(EOFError, match="EOF when reading a line"): + input() +`); +``` + +Note that the `input()` function automatically reads a line of text and +removes the trailing newline. If we use `sys.stdin.read` we see that newlines +have been appended to strings that don't end in a newline: + +```pyodide +pyodide.setStdin( + new StdinHandler(["a", "bcd\n", "efg", undefined, "h", "i"]), +); +pyodide.runPython(String.raw` + import sys + assert sys.stdin.read() == "a\nbcd\nefg\n" + assert sys.stdin.read() == "h\ni\n" +`); +``` + +Instead of strings we can return the list of utf8 bytes for the input: + +```pyodide +pyodide.setStdin( + new StdinHandler( + [0x61 /* a */, 0x0a /* \n */, 0x62 /* b */, 0x63 /* c */], + true, + ), +); +pyodide.runPython(` + assert input() == "a" + assert input() == "bc" +`); +``` + +Or we can return a `Uint8Array` with the utf8-encoded text that we wish to +render: + +```pyodide +pyodide.setStdin( + new StdinHandler([new Uint8Array([0x61, 0x0a, 0x62, 0x63])]), +); +pyodide.runPython(` + assert input() == "a" + assert input() == "bc" +`); +``` + +### A read handler + +A read handler takes a `Uint8Array` as an argument and is supposed to place +the data into this buffer and return the number of bytes read. This is useful in +Node. For example, the following class can be used to read from a Node file +descriptor: + +```js +const fs = require("fs"); +const tty = require("tty"); +class NodeReader { + constructor(fd) { + this.fd = fd; + this.isatty = tty.isatty(fd); + } + + read(buffer) { + return fs.readSync(this.fd, buffer); + } +} +``` + +For instance to set stdin to read from a file called `input.txt`, we can do the +following: + +```js +const fd = fs.openSync("input.txt", "r"); +py.setStdin(new NodeReader(fd)); +``` + +Or we can read from node's stdin (the default behavior) as follows: + +```js +fd = fs.openSync("/dev/stdin", "r"); +py.setStdin(new NodeReader(fd)); +``` + +### isatty + +It is possible to control whether or not {py:meth}`sys.stdin.isatty() ` +returns true with the `isatty` option: + +```pyodide +pyodide.setStdin(new StdinHandler([], {isatty: true})); +pyodide.runPython(` + import sys + assert sys.stdin.isatty() # returns true as we requested +`); +pyodide.setStdin(new StdinHandler([], {isatty: false})); +pyodide.runPython(` + assert not sys.stdin.isatty() # returns false as we requested +`); +``` + +This will change the behavior of cli applications that behave differently in an +interactive terminal, for example pytest does this. + +### Raising IO errors + +To raise an IO error in either a `stdin` or `read` handler, you should throw an +IO error as follows: + +```js +throw new pyodide.FS.ErrnoError(pyodide.ERRNO_CODES.EIO); +``` + +for instance, saying: + +```js +pyodide.setStdin({ + read(buf) { + throw new pyodide.FS.ErrnoError(pyodide.ERRNO_CODES.EIO); + }, +}); +``` + +is the same as `pyodide.setStdin({error: true})`. + +### Handling Keyboard interrupts + +To handle a keyboard interrupt in an input handler, you should periodically call +{js:func}`pyodide.checkInterrupt`. For example, the following stdin handler +always raises a keyboard interrupt: + +```js +const interruptBuffer = new Int32Array(new SharedArrayBuffer(4)); +pyodide.setInterruptBuffer(interruptBuffer); +pyodide.setStdin({ + read(buf) { + // Put signal into interrupt buffer + interruptBuffer[0] = 2; + // Call checkInterrupt to raise an error + pyodide.checkInterrupt(); + console.log( + "This code won't ever be executed because pyodide.checkInterrupt raises an error!", + ); + }, +}); +``` + +For a more realistic example that handles reading stdin in a worker and also +keyboard interrupts, you might something like the following code: + +```js +pyodide.setStdin({read(buf) { + const timeoutMilliseconds = 100; + while(true) { + switch(Atomics.wait(stdinSharedBuffer, 0, 0, timeoutMilliseconds) { + case "timed-out": + // 100 ms passed but got no data, check for keyboard interrupt then return to waiting on data. + pyodide.checkInterrupt(); + break; + case "ok": + // ... handle the data somehow + break; + } + } +}}); +``` + +See also {ref}`interrupting_execution`. + +## Standard Out / Standard Error + +(streams-stdout)= + +{js:func}`pyodide.setStdout` and {js:func}`pyodide.setStderr` respectively set +the standard output and standard error handlers. These APIs are identical except +in their defaults, so we will only discuss the `pyodide.setStdout` except in +cases where they differ. + +As with {js:func}`pyodide.setStdin`, there are quite a few different ways to set +the standard output handlers. + +### Set the default behavior + +As with stdin, `pyodide.setStdout()` sets the default behavior. In node, this is +to write directly to `process.stdout`. In the browser, the default is as if you +wrote +`setStdout({batched: (str) => console.log(str)})` +see below. + +### A batched handler + +A batched handler is the easiest standard out handler to implement but it is +also the coarsest. It is intended to use with APIs like `console.log` that don't +understand partial lines of text or for quick and dirty code. + +The batched handler receives a string which is either: + +1. a complete line of text with the newline removed or +2. a partial line of text that was flushed. + +For instance after: + +```py +print("hello!") +import sys +print("partial line", end="") +sys.stdout.flush() +``` + +the batched handler is called with `"hello!"` and then with `"partial line"`. +Note that there is no indication that `"hello!"` was a complete line of text and +`"partial line"` was not. + +### A raw handler + +A raw handler receives the output one character code at a time. This is neither +very convenient nor very efficient. It is present primarily for backwards +compatibility reasons. + +For example, the following code: + +```py +print("h") +import sys +print("p ", end="") +print("l", end="") +sys.stdout.flush() +``` + +will call the raw handler with the sequence of bytes: + +```py +0x68 - h +0x0A - newline +0x70 - p +0x20 - space +0x6c - l +``` + +### A write handler + +A write handler takes a `Uint8Array` as an argument and is supposed to write the +data in this buffer to standard output and return the number of bytes written. +For example, the following class can be used to write to a Node file descriptor: + +```js +const fs = require("fs"); +const tty = require("tty"); +class NodeWriter { + constructor(fd) { + this.fd = fd; + this.isatty = tty.isatty(fd); + } + + write(buffer) { + return fs.writeSync(this.fd, buffer); + } +} +``` + +Using it as follows redirects output from Pyodide to `out.txt`: + +```js +const fd = fs.openSync("out.txt", "w"); +py.setStdout(new NodeWriter(fd)); +``` + +Or the following gives the default behavior: + +```js +const fd = fs.openSync("out.txt", "w"); +py.setStdout(new NodeWriter(process.stdout.fd)); +``` + +### isatty + +As with `stdin`, is possible to control whether or not +{py:meth}`sys.stdout.isatty() ` returns true with the `isatty` +option. You cannot combine `isatty: true` with a batched handler. diff --git a/emsdk/Makefile b/emsdk/Makefile index b3f76e20b..22116ca40 100644 --- a/emsdk/Makefile +++ b/emsdk/Makefile @@ -10,7 +10,18 @@ emsdk/.complete: ../Makefile.envs $(wildcard patches/*.patch) cd emsdk/upstream/emscripten/ && cat ../../../patches/*.patch | patch -p1 --verbose cd emsdk && ./emsdk install --build=Release $(PYODIDE_EMSCRIPTEN_VERSION) ccache-git-emscripten-64bit cd emsdk && ./emsdk activate --embedded --build=Release $(PYODIDE_EMSCRIPTEN_VERSION) + + # Check that generated_struct_info is up to date. + cmp --silent emsdk/upstream/emscripten/src/generated_struct_info32.json ../src/js/generated_struct_info32.json || \ + ( \ + echo "" && \ + echo "The vendored copy of generated_struct_info32.json does not match the copy in Emscripten" && \ + exit 1 \ + ) + touch emsdk/.complete + + clean: rm -rf emsdk diff --git a/src/js/api.ts b/src/js/api.ts index 45abd88e1..d3ce98cd0 100644 --- a/src/js/api.ts +++ b/src/js/api.ts @@ -560,8 +560,19 @@ export class PyodideAPI { * during execution of C code. */ static checkInterrupt() { - if (Module.__PyErr_CheckSignals()) { - Module._pythonexc2js(); + if (Module._PyGILState_Check()) { + // GIL held, so it's okay to call __PyErr_CheckSignals. + if (Module.__PyErr_CheckSignals()) { + Module._pythonexc2js(); + } + return; + } else { + // GIL not held. This is very likely because we're in a IO handler. If + // buffer has a 2, throwing EINTR quits out from the IO handler and tells + // the calling context to call `PyErr_CheckSignals`. + if (Module.Py_EmscriptenSignalBuffer[0] === 2) { + throw new Module.FS.ErrnoError(cDefs.EINTR); + } } } diff --git a/src/js/constants.ts b/src/js/constants.ts new file mode 100644 index 000000000..d256a35aa --- /dev/null +++ b/src/js/constants.ts @@ -0,0 +1,9 @@ +import type { Module as OrigModule } from "./module"; +import { defines } from "./generated_struct_info32.json"; + +declare global { + export const cDefs: typeof defines; + export const DEBUG: boolean; + export var Module: OrigModule; + export var API: any; +} diff --git a/src/js/esbuild.config.mjs b/src/js/esbuild.config.mjs index 4e0df1c6d..cefad787f 100644 --- a/src/js/esbuild.config.mjs +++ b/src/js/esbuild.config.mjs @@ -27,6 +27,27 @@ const outputs = [ ]; const dest = (output) => join(__dirname, "..", "..", output); +const DEFINES = { DEBUG: DEBUG.toString() }; + +function toDefines(o, path = "") { + return Object.entries(o).flatMap(([x, v]) => { + // Drop anything that's not a valid identifier + if (!/^[A-Za-z_$]*$/.test(x)) { + return []; + } + // Flatten objects + if (typeof v === "object") { + return toDefines(v, path + x + "."); + } + // Else convert to string + return [[path + x, v.toString()]]; + }); +} + +const cdefsFile = join(__dirname, "generated_struct_info32.json"); +const origConstants = JSON.parse(readFileSync(cdefsFile)); +const constants = { cDefs: origConstants.defines }; +Object.assign(DEFINES, Object.fromEntries(toDefines(constants))); const config = ({ input, output, format, name: globalName }) => ({ entryPoints: [join(__dirname, input + ".ts")], @@ -43,7 +64,7 @@ const config = ({ input, output, format, name: globalName }) => ({ "vm", "ws", ], - define: { DEBUG: DEBUG.toString() }, + define: DEFINES, minify: !DEBUG, keepNames: true, sourcemap: true, diff --git a/src/js/generated_struct_info32.json b/src/js/generated_struct_info32.json new file mode 100644 index 000000000..448b5d6d0 --- /dev/null +++ b/src/js/generated_struct_info32.json @@ -0,0 +1,1576 @@ +{ + "defines": { + "AF_INET": 2, + "AF_INET6": 10, + "AF_UNSPEC": 0, + "AI_ADDRCONFIG": 32, + "AI_ALL": 16, + "AI_CANONNAME": 2, + "AI_NUMERICHOST": 4, + "AI_NUMERICSERV": 1024, + "AI_PASSIVE": 1, + "AI_V4MAPPED": 8, + "ALC_FALSE": 0, + "ALC_INVALID_DEVICE": 40961, + "ALC_INVALID_ENUM": 40963, + "ALC_INVALID_VALUE": 40964, + "ALC_NO_ERROR": 0, + "ALC_TRUE": 1, + "AL_DIRECTION": 4101, + "AL_DISTANCE_MODEL": 53248, + "AL_DOPPLER_FACTOR": 49152, + "AL_FALSE": 0, + "AL_GAIN": 4106, + "AL_INITIAL": 4113, + "AL_INVALID_ENUM": 40962, + "AL_INVALID_NAME": 40961, + "AL_INVALID_OPERATION": 40964, + "AL_INVALID_VALUE": 40963, + "AL_NONE": 0, + "AL_NO_ERROR": 0, + "AL_ORIENTATION": 4111, + "AL_PAUSED": 4115, + "AL_PLAYING": 4114, + "AL_POSITION": 4100, + "AL_SPEED_OF_SOUND": 49155, + "AL_STATIC": 4136, + "AL_STOPPED": 4116, + "AL_TRUE": 1, + "AL_VELOCITY": 4102, + "AT_EMPTY_PATH": 4096, + "AT_FDCWD": -100, + "AT_NO_AUTOMOUNT": 2048, + "AT_REMOVEDIR": 512, + "AT_SYMLINK_NOFOLLOW": 256, + "AUDIO_F32": 33056, + "AUDIO_S16LSB": 32784, + "AUDIO_U8": 8, + "B38400": 15, + "CLOCK_REALTIME": 0, + "CREAD": 128, + "CSIZE": 48, + "E2BIG": 1, + "EACCES": 2, + "EADDRINUSE": 3, + "EADDRNOTAVAIL": 4, + "EADV": 122, + "EAFNOSUPPORT": 5, + "EAGAIN": 6, + "EAI_AGAIN": -3, + "EAI_BADFLAGS": -1, + "EAI_FAIL": -4, + "EAI_FAMILY": -6, + "EAI_MEMORY": -10, + "EAI_NONAME": -2, + "EAI_OVERFLOW": -12, + "EAI_SERVICE": -8, + "EAI_SOCKTYPE": -7, + "EAI_SYSTEM": -11, + "EALREADY": 7, + "EBADE": 113, + "EBADF": 8, + "EBADFD": 127, + "EBADMSG": 9, + "EBADR": 114, + "EBADRQC": 103, + "EBADSLT": 102, + "EBFONT": 101, + "EBUSY": 10, + "ECANCELED": 11, + "ECHILD": 12, + "ECHO": 8, + "ECHOCTL": 512, + "ECHOE": 16, + "ECHOK": 32, + "ECHOKE": 2048, + "ECHRNG": 106, + "ECOMM": 124, + "ECONNABORTED": 13, + "ECONNREFUSED": 14, + "ECONNRESET": 15, + "EDEADLK": 16, + "EDEADLOCK": 16, + "EDESTADDRREQ": 17, + "EDOM": 18, + "EDOTDOT": 125, + "EDQUOT": 19, + "EEXIST": 20, + "EFAULT": 21, + "EFBIG": 22, + "EHOSTDOWN": 142, + "EHOSTUNREACH": 23, + "EIDRM": 24, + "EILSEQ": 25, + "EINPROGRESS": 26, + "EINTR": 27, + "EINVAL": 28, + "EIO": 29, + "EISCONN": 30, + "EISDIR": 31, + "EL2HLT": 112, + "EL2NSYNC": 156, + "EL3HLT": 107, + "EL3RST": 108, + "ELIBACC": 129, + "ELIBBAD": 130, + "ELIBEXEC": 133, + "ELIBMAX": 132, + "ELIBSCN": 131, + "ELNRNG": 109, + "ELOOP": 32, + "EMFILE": 33, + "EMLINK": 34, + "EMSCRIPTEN_DEVICE_MOTION_EVENT_SUPPORTS_ACCELERATION": 1, + "EMSCRIPTEN_DEVICE_MOTION_EVENT_SUPPORTS_ACCELERATION_INCLUDING_GRAVITY": 2, + "EMSCRIPTEN_DEVICE_MOTION_EVENT_SUPPORTS_ROTATION_RATE": 4, + "EMSCRIPTEN_EVENT_BATTERYCHARGINGCHANGE": 29, + "EMSCRIPTEN_EVENT_BATTERYLEVELCHANGE": 30, + "EMSCRIPTEN_EVENT_BEFOREUNLOAD": 28, + "EMSCRIPTEN_EVENT_BLUR": 12, + "EMSCRIPTEN_EVENT_CANVASRESIZED": 37, + "EMSCRIPTEN_EVENT_CLICK": 4, + "EMSCRIPTEN_EVENT_DBLCLICK": 7, + "EMSCRIPTEN_EVENT_DEVICEMOTION": 17, + "EMSCRIPTEN_EVENT_DEVICEORIENTATION": 16, + "EMSCRIPTEN_EVENT_FOCUS": 13, + "EMSCRIPTEN_EVENT_FOCUSIN": 14, + "EMSCRIPTEN_EVENT_FOCUSOUT": 15, + "EMSCRIPTEN_EVENT_FULLSCREENCHANGE": 19, + "EMSCRIPTEN_EVENT_GAMEPADCONNECTED": 26, + "EMSCRIPTEN_EVENT_GAMEPADDISCONNECTED": 27, + "EMSCRIPTEN_EVENT_KEYDOWN": 2, + "EMSCRIPTEN_EVENT_KEYPRESS": 1, + "EMSCRIPTEN_EVENT_KEYUP": 3, + "EMSCRIPTEN_EVENT_MOUSEDOWN": 5, + "EMSCRIPTEN_EVENT_MOUSEENTER": 33, + "EMSCRIPTEN_EVENT_MOUSELEAVE": 34, + "EMSCRIPTEN_EVENT_MOUSEMOVE": 8, + "EMSCRIPTEN_EVENT_MOUSEOUT": 36, + "EMSCRIPTEN_EVENT_MOUSEOVER": 35, + "EMSCRIPTEN_EVENT_MOUSEUP": 6, + "EMSCRIPTEN_EVENT_ORIENTATIONCHANGE": 18, + "EMSCRIPTEN_EVENT_POINTERLOCKCHANGE": 20, + "EMSCRIPTEN_EVENT_POINTERLOCKERROR": 38, + "EMSCRIPTEN_EVENT_RESIZE": 10, + "EMSCRIPTEN_EVENT_SCROLL": 11, + "EMSCRIPTEN_EVENT_TARGET_DOCUMENT": 1, + "EMSCRIPTEN_EVENT_TARGET_INVALID": 0, + "EMSCRIPTEN_EVENT_TARGET_SCREEN": 3, + "EMSCRIPTEN_EVENT_TARGET_WINDOW": 2, + "EMSCRIPTEN_EVENT_TOUCHCANCEL": 25, + "EMSCRIPTEN_EVENT_TOUCHEND": 23, + "EMSCRIPTEN_EVENT_TOUCHMOVE": 24, + "EMSCRIPTEN_EVENT_TOUCHSTART": 22, + "EMSCRIPTEN_EVENT_VISIBILITYCHANGE": 21, + "EMSCRIPTEN_EVENT_WEBGLCONTEXTLOST": 31, + "EMSCRIPTEN_EVENT_WEBGLCONTEXTRESTORED": 32, + "EMSCRIPTEN_EVENT_WHEEL": 9, + "EMSCRIPTEN_FETCH_APPEND": 8, + "EMSCRIPTEN_FETCH_LOAD_TO_MEMORY": 1, + "EMSCRIPTEN_FETCH_NO_DOWNLOAD": 32, + "EMSCRIPTEN_FETCH_PERSIST_FILE": 4, + "EMSCRIPTEN_FETCH_REPLACE": 16, + "EMSCRIPTEN_FETCH_STREAM_DATA": 2, + "EMSCRIPTEN_FETCH_SYNCHRONOUS": 64, + "EMSCRIPTEN_FETCH_WAITABLE": 128, + "EMSCRIPTEN_FULLSCREEN_CANVAS_SCALE_HIDEF": 2, + "EMSCRIPTEN_FULLSCREEN_CANVAS_SCALE_NONE": 0, + "EMSCRIPTEN_FULLSCREEN_CANVAS_SCALE_STDDEF": 1, + "EMSCRIPTEN_FULLSCREEN_FILTERING_BILINEAR": 2, + "EMSCRIPTEN_FULLSCREEN_FILTERING_DEFAULT": 0, + "EMSCRIPTEN_FULLSCREEN_FILTERING_NEAREST": 1, + "EMSCRIPTEN_FULLSCREEN_SCALE_ASPECT": 2, + "EMSCRIPTEN_FULLSCREEN_SCALE_CENTER": 3, + "EMSCRIPTEN_FULLSCREEN_SCALE_DEFAULT": 0, + "EMSCRIPTEN_FULLSCREEN_SCALE_STRETCH": 1, + "EMSCRIPTEN_RESULT_DEFERRED": 1, + "EMSCRIPTEN_RESULT_FAILED": -6, + "EMSCRIPTEN_RESULT_FAILED_NOT_DEFERRED": -2, + "EMSCRIPTEN_RESULT_INVALID_PARAM": -5, + "EMSCRIPTEN_RESULT_INVALID_TARGET": -3, + "EMSCRIPTEN_RESULT_NOT_SUPPORTED": -1, + "EMSCRIPTEN_RESULT_NO_DATA": -7, + "EMSCRIPTEN_RESULT_SUCCESS": 0, + "EMSCRIPTEN_RESULT_TIMED_OUT": -8, + "EMSCRIPTEN_RESULT_UNKNOWN_TARGET": -4, + "EMSCRIPTEN_WEBGL_CONTEXT_PROXY_ALWAYS": 2, + "EMSCRIPTEN_WEBGL_CONTEXT_PROXY_DISALLOW": 0, + "EMSCRIPTEN_WEBGL_CONTEXT_PROXY_FALLBACK": 1, + "EMSGSIZE": 35, + "EMULTIHOP": 36, + "EM_CALLBACK_THREAD_CONTEXT_CALLING_THREAD": 2, + "EM_CALLBACK_THREAD_CONTEXT_MAIN_RUNTIME_THREAD": 1, + "EM_FUNC_SIG_I": 536870912, + "EM_FUNC_SIG_II": 570425344, + "EM_FUNC_SIG_III": 603979776, + "EM_FUNC_SIG_IIII": 637534208, + "EM_FUNC_SIG_PARAM_B": 4, + "EM_FUNC_SIG_PARAM_D": 3, + "EM_FUNC_SIG_PARAM_F": 2, + "EM_FUNC_SIG_PARAM_F2I": 5, + "EM_FUNC_SIG_PARAM_I": 0, + "EM_FUNC_SIG_PARAM_I64": 1, + "EM_FUNC_SIG_V": 0, + "EM_FUNC_SIG_VI": 33554432, + "EM_FUNC_SIG_VII": 67108864, + "EM_FUNC_SIG_VIII": 100663296, + "EM_HTML5_LONG_STRING_LEN_BYTES": 128, + "EM_HTML5_MEDIUM_STRING_LEN_BYTES": 64, + "EM_HTML5_SHORT_STRING_LEN_BYTES": 32, + "EM_LOG_CONSOLE": 1, + "EM_LOG_C_STACK": 8, + "EM_LOG_DEBUG": 256, + "EM_LOG_DEMANGLE": 32, + "EM_LOG_ERROR": 4, + "EM_LOG_FUNC_PARAMS": 128, + "EM_LOG_INFO": 512, + "EM_LOG_JS_STACK": 16, + "EM_LOG_NO_PATHS": 64, + "EM_LOG_WARN": 2, + "EM_PROMISE_FULFILL": 0, + "EM_PROMISE_MATCH": 1, + "EM_PROMISE_MATCH_RELEASE": 2, + "EM_PROMISE_REJECT": 3, + "EM_PROXIED_RESIZE_OFFSCREENCANVAS": 654311424, + "EM_QUEUED_JS_CALL_MAX_ARGS": 20, + "EM_TIMING_RAF": 1, + "EM_TIMING_SETIMMEDIATE": 2, + "EM_TIMING_SETTIMEOUT": 0, + "ENAMETOOLONG": 37, + "ENETDOWN": 38, + "ENETRESET": 39, + "ENETUNREACH": 40, + "ENFILE": 41, + "ENOANO": 104, + "ENOBUFS": 42, + "ENOCSI": 111, + "ENODATA": 116, + "ENODEV": 43, + "ENOENT": 44, + "ENOEXEC": 45, + "ENOLCK": 46, + "ENOLINK": 47, + "ENOMEDIUM": 148, + "ENOMEM": 48, + "ENOMSG": 49, + "ENONET": 119, + "ENOPKG": 120, + "ENOPROTOOPT": 50, + "ENOSPC": 51, + "ENOSR": 118, + "ENOSTR": 100, + "ENOSYS": 52, + "ENOTBLK": 105, + "ENOTCONN": 53, + "ENOTDIR": 54, + "ENOTEMPTY": 55, + "ENOTRECOVERABLE": 56, + "ENOTSOCK": 57, + "ENOTSUP": 138, + "ENOTTY": 59, + "ENOTUNIQ": 126, + "ENXIO": 60, + "EOPNOTSUPP": 138, + "EOVERFLOW": 61, + "EOWNERDEAD": 62, + "EPERM": 63, + "EPFNOSUPPORT": 139, + "EPIPE": 64, + "EPROTO": 65, + "EPROTONOSUPPORT": 66, + "EPROTOTYPE": 67, + "ERANGE": 68, + "EREMCHG": 128, + "EREMOTE": 121, + "EROFS": 69, + "ESHUTDOWN": 140, + "ESOCKTNOSUPPORT": 137, + "ESPIPE": 70, + "ESRCH": 71, + "ESRMNT": 123, + "ESTALE": 72, + "ESTRPIPE": 135, + "ETIME": 117, + "ETIMEDOUT": 73, + "ETOOMANYREFS": 141, + "ETXTBSY": 74, + "EUNATCH": 110, + "EUSERS": 136, + "EWOULDBLOCK": 6, + "EXDEV": 75, + "EXFULL": 115, + "FIONREAD": 21531, + "F_DUPFD": 0, + "F_GETFD": 1, + "F_GETFL": 3, + "F_GETLK": 5, + "F_GETLK64": 5, + "F_GETOWN": 9, + "F_GETOWN_EX": 16, + "F_SETFD": 2, + "F_SETFL": 4, + "F_SETLK": 6, + "F_SETLK64": 6, + "F_SETLKW": 7, + "F_SETLKW64": 7, + "F_SETOWN": 8, + "F_UNLCK": 2, + "File::DataFileKind": 1, + "File::DirectoryKind": 2, + "File::SymlinkKind": 3, + "File::UnknownKind": 0, + "ICANON": 2, + "ICRNL": 256, + "IEXTEN": 32768, + "IMAXBEL": 8192, + "INADDR_LOOPBACK": 2130706433, + "IPPROTO_TCP": 6, + "IPPROTO_UDP": 17, + "ISIG": 1, + "IUTF8": 16384, + "IXON": 1024, + "MAP_ANONYMOUS": 32, + "MAP_FIXED": 16, + "MAP_PRIVATE": 2, + "NCCS": 32, + "NI_NAMEREQD": 8, + "NI_NUMERICHOST": 1, + "NOTIFICATION_NONE": 0, + "NOTIFICATION_PENDING": 2, + "NOTIFICATION_RECEIVED": 1, + "ONLCR": 4, + "OPOST": 1, + "O_ACCMODE": 2097155, + "O_APPEND": 1024, + "O_CLOEXEC": 524288, + "O_CREAT": 64, + "O_DIRECTORY": 65536, + "O_DSYNC": 4096, + "O_EXCL": 128, + "O_LARGEFILE": 32768, + "O_NOCTTY": 256, + "O_NOFOLLOW": 131072, + "O_NONBLOCK": 2048, + "O_PATH": 2097152, + "O_RDONLY": 0, + "O_RDWR": 2, + "O_TRUNC": 512, + "O_WRONLY": 1, + "POLLERR": 8, + "POLLHUP": 16, + "POLLIN": 1, + "POLLNVAL": 32, + "POLLOUT": 4, + "POLLPRI": 2, + "POLLRDNORM": 64, + "POLLWRNORM": 256, + "PROT_WRITE": 2, + "PTHREAD_CANCELED": -1, + "RTLD_DEFAULT": 0, + "RTLD_GLOBAL": 256, + "RTLD_LAZY": 1, + "RTLD_NODELETE": 4096, + "RTLD_NOW": 2, + "R_OK": 4, + "SDL_PIXELFORMAT_RGBA8888": -2042224636, + "SDL_TOUCH_MOUSEID": -1, + "SEEK_CUR": 1, + "SEEK_END": 2, + "SEEK_SET": 0, + "SIGALRM": 14, + "SIGCANCEL": 33, + "SOCK_CLOEXEC": 524288, + "SOCK_DGRAM": 2, + "SOCK_NONBLOCK": 2048, + "SOCK_STREAM": 1, + "SOL_SOCKET": 1, + "SO_ERROR": 4, + "S_IALLUGO": 4095, + "S_IFBLK": 24576, + "S_IFCHR": 8192, + "S_IFDIR": 16384, + "S_IFIFO": 4096, + "S_IFLNK": 40960, + "S_IFMT": 61440, + "S_IFREG": 32768, + "S_IFSOCK": 49152, + "S_IRUGO": 292, + "S_IRWXO": 7, + "S_IRWXUGO": 511, + "S_ISVTX": 512, + "S_IWUGO": 146, + "S_IXUGO": 73, + "TCFLSH": 21515, + "TCGETA": 21509, + "TCGETS": 21505, + "TCSETA": 21510, + "TCSETAF": 21512, + "TCSETAW": 21511, + "TCSETS": 21506, + "TCSETSF": 21508, + "TCSETSW": 21507, + "TIOCGPGRP": 21519, + "TIOCGWINSZ": 21523, + "TIOCSPGRP": 21520, + "TIOCSWINSZ": 21524, + "UUID_TYPE_DCE_RANDOM": 4, + "UUID_VARIANT_DCE": 1, + "W_OK": 2, + "X_OK": 1, + "__WASI_CLOCKID_MONOTONIC": 1, + "__WASI_CLOCKID_PROCESS_CPUTIME_ID": 2, + "__WASI_CLOCKID_REALTIME": 0, + "__WASI_CLOCKID_THREAD_CPUTIME_ID": 3, + "__WASI_FDFLAGS_APPEND": 1, + "__WASI_FDFLAGS_DSYNC": 2, + "__WASI_FDFLAGS_NONBLOCK": 4, + "__WASI_FDFLAGS_RSYNC": 8, + "__WASI_FDFLAGS_SYNC": 16, + "__WASI_FILETYPE_CHARACTER_DEVICE": 2, + "__WASI_FILETYPE_DIRECTORY": 3, + "__WASI_FILETYPE_REGULAR_FILE": 4, + "__WASI_FILETYPE_SYMBOLIC_LINK": 7, + "__WASI_OFLAGS_CREAT": 1, + "__WASI_OFLAGS_DIRECTORY": 2, + "__WASI_OFLAGS_EXCL": 4, + "__WASI_OFLAGS_TRUNC": 8, + "__WASI_PREOPENTYPE_DIR": 0, + "__WASI_RIGHTS_FD_ADVISE": 128, + "__WASI_RIGHTS_FD_ALLOCATE": 256, + "__WASI_RIGHTS_FD_DATASYNC": 1, + "__WASI_RIGHTS_FD_FDSTAT_SET_FLAGS": 8, + "__WASI_RIGHTS_FD_FILESTAT_GET": 2097152, + "__WASI_RIGHTS_FD_FILESTAT_SET_SIZE": 4194304, + "__WASI_RIGHTS_FD_FILESTAT_SET_TIMES": 8388608, + "__WASI_RIGHTS_FD_READ": 2, + "__WASI_RIGHTS_FD_READDIR": 16384, + "__WASI_RIGHTS_FD_SEEK": 4, + "__WASI_RIGHTS_FD_SYNC": 16, + "__WASI_RIGHTS_FD_TELL": 32, + "__WASI_RIGHTS_FD_WRITE": 64, + "__WASI_RIGHTS_PATH_CREATE_DIRECTORY": 512, + "__WASI_RIGHTS_PATH_CREATE_FILE": 1024, + "__WASI_RIGHTS_PATH_FILESTAT_GET": 262144, + "__WASI_RIGHTS_PATH_FILESTAT_SET_SIZE": 524288, + "__WASI_RIGHTS_PATH_FILESTAT_SET_TIMES": 1048576, + "__WASI_RIGHTS_PATH_LINK_SOURCE": 2048, + "__WASI_RIGHTS_PATH_LINK_TARGET": 4096, + "__WASI_RIGHTS_PATH_OPEN": 8192, + "__WASI_RIGHTS_PATH_READLINK": 32768, + "__WASI_RIGHTS_PATH_REMOVE_DIRECTORY": 33554432, + "__WASI_RIGHTS_PATH_RENAME_SOURCE": 65536, + "__WASI_RIGHTS_PATH_RENAME_TARGET": 131072, + "__WASI_RIGHTS_PATH_SYMLINK": 16777216, + "__WASI_RIGHTS_PATH_UNLINK_FILE": 67108864, + "__WASI_RIGHTS_POLL_FD_READWRITE": 134217728, + "__WASI_RIGHTS_SOCK_SHUTDOWN": 268435456 + }, + "structs": { + "EmscriptenBatteryEvent": { + "__size__": 32, + "charging": 24, + "chargingTime": 0, + "dischargingTime": 8, + "level": 16 + }, + "EmscriptenDeviceMotionEvent": { + "__size__": 80, + "accelerationIncludingGravityX": 24, + "accelerationIncludingGravityY": 32, + "accelerationIncludingGravityZ": 40, + "accelerationX": 0, + "accelerationY": 8, + "accelerationZ": 16, + "rotationRateAlpha": 48, + "rotationRateBeta": 56, + "rotationRateGamma": 64 + }, + "EmscriptenDeviceOrientationEvent": { + "__size__": 32, + "absolute": 24, + "alpha": 0, + "beta": 8, + "gamma": 16 + }, + "EmscriptenFocusEvent": { + "__size__": 256, + "id": 128, + "nodeName": 0 + }, + "EmscriptenFullscreenChangeEvent": { + "__size__": 280, + "elementHeight": 268, + "elementWidth": 264, + "fullscreenEnabled": 4, + "id": 136, + "isFullscreen": 0, + "nodeName": 8, + "screenHeight": 276, + "screenWidth": 272 + }, + "EmscriptenFullscreenStrategy": { + "__size__": 24, + "canvasResizedCallback": 12, + "canvasResizedCallbackTargetThread": 20, + "canvasResizedCallbackUserData": 16, + "canvasResolutionScaleMode": 4, + "filteringMode": 8, + "scaleMode": 0 + }, + "EmscriptenGamepadEvent": { + "__size__": 1432, + "analogButton": 528, + "axis": 16, + "connected": 1296, + "digitalButton": 1040, + "id": 1304, + "index": 1300, + "mapping": 1368, + "numAxes": 8, + "numButtons": 12, + "timestamp": 0 + }, + "EmscriptenKeyboardEvent": { + "__size__": 176, + "altKey": 20, + "charCode": 32, + "charValue": 108, + "code": 76, + "ctrlKey": 12, + "key": 44, + "keyCode": 36, + "locale": 140, + "location": 8, + "metaKey": 24, + "repeat": 28, + "shiftKey": 16, + "timestamp": 0, + "which": 40 + }, + "EmscriptenMouseEvent": { + "__size__": 72, + "altKey": 32, + "button": 40, + "buttons": 42, + "canvasX": 60, + "canvasY": 64, + "clientX": 16, + "clientY": 20, + "ctrlKey": 24, + "metaKey": 36, + "movementX": 44, + "movementY": 48, + "screenX": 8, + "screenY": 12, + "shiftKey": 28, + "targetX": 52, + "targetY": 56, + "timestamp": 0 + }, + "EmscriptenOrientationChangeEvent": { + "__size__": 8, + "orientationAngle": 4, + "orientationIndex": 0 + }, + "EmscriptenPointerlockChangeEvent": { + "__size__": 260, + "id": 132, + "isActive": 0, + "nodeName": 4 + }, + "EmscriptenTouchEvent": { + "__size__": 1696, + "altKey": 20, + "ctrlKey": 12, + "metaKey": 24, + "numTouches": 8, + "shiftKey": 16, + "timestamp": 0, + "touches": 28 + }, + "EmscriptenTouchPoint": { + "__size__": 52, + "canvasX": 44, + "canvasY": 48, + "clientX": 12, + "clientY": 16, + "identifier": 0, + "isChanged": 28, + "onTarget": 32, + "pageX": 20, + "pageY": 24, + "screenX": 4, + "screenY": 8, + "targetX": 36, + "targetY": 40 + }, + "EmscriptenUiEvent": { + "__size__": 36, + "detail": 0, + "documentBodyClientHeight": 8, + "documentBodyClientWidth": 4, + "scrollLeft": 32, + "scrollTop": 28, + "windowInnerHeight": 16, + "windowInnerWidth": 12, + "windowOuterHeight": 24, + "windowOuterWidth": 20 + }, + "EmscriptenVisibilityChangeEvent": { + "__size__": 8, + "hidden": 0, + "visibilityState": 4 + }, + "EmscriptenWebGLContextAttributes": { + "__size__": 56, + "alpha": 0, + "antialias": 12, + "depth": 4, + "enableExtensionsByDefault": 40, + "explicitSwapControl": 44, + "failIfMajorPerformanceCaveat": 28, + "majorVersion": 32, + "minorVersion": 36, + "powerPreference": 24, + "premultipliedAlpha": 16, + "preserveDrawingBuffer": 20, + "proxyContextToMainThread": 48, + "renderViaOffscreenBackBuffer": 52, + "stencil": 8 + }, + "EmscriptenWheelEvent": { + "__size__": 104, + "deltaMode": 96, + "deltaX": 72, + "deltaY": 80, + "deltaZ": 88, + "mouse": 0 + }, + "SDL_AudioSpec": { + "__size__": 24, + "callback": 16, + "channels": 6, + "format": 4, + "freq": 0, + "samples": 8, + "silence": 7, + "userdata": 20 + }, + "SDL_JoyAxisEvent": { + "__size__": 12, + "axis": 5, + "type": 0, + "value": 8, + "which": 4 + }, + "SDL_JoyButtonEvent": { + "__size__": 8, + "button": 5, + "state": 6, + "type": 0, + "which": 4 + }, + "SDL_KeyboardEvent": { + "__size__": 28, + "keysym": 12, + "repeat": 9, + "state": 8, + "type": 0 + }, + "SDL_Keysym": { + "__size__": 16, + "mod": 8, + "scancode": 0, + "sym": 4, + "unicode": 12 + }, + "SDL_MouseButtonEvent": { + "__size__": 28, + "button": 16, + "state": 17, + "timestamp": 4, + "type": 0, + "which": 12, + "windowID": 8, + "x": 20, + "y": 24 + }, + "SDL_MouseMotionEvent": { + "__size__": 36, + "state": 16, + "timestamp": 4, + "type": 0, + "which": 12, + "windowID": 8, + "x": 20, + "xrel": 28, + "y": 24, + "yrel": 32 + }, + "SDL_MouseWheelEvent": { + "__size__": 24, + "type": 0, + "x": 16, + "y": 20 + }, + "SDL_PixelFormat": { + "Amask": 24, + "BitsPerPixel": 8, + "Bmask": 20, + "BytesPerPixel": 9, + "Gmask": 16, + "Rmask": 12, + "__size__": 44, + "format": 0, + "palette": 4 + }, + "SDL_Rect": { + "__size__": 16, + "h": 12, + "w": 8, + "x": 0, + "y": 4 + }, + "SDL_ResizeEvent": { + "__size__": 12, + "h": 8, + "w": 4 + }, + "SDL_Surface": { + "__size__": 60, + "clip_rect": 36, + "flags": 0, + "format": 4, + "h": 12, + "pitch": 16, + "pixels": 20, + "refcount": 56, + "w": 8 + }, + "SDL_TextInputEvent": { + "__size__": 40, + "text": 8, + "type": 0 + }, + "SDL_TouchFingerEvent": { + "__size__": 48, + "dx": 32, + "dy": 36, + "fingerId": 16, + "pressure": 40, + "timestamp": 4, + "touchId": 8, + "type": 0, + "x": 24, + "y": 28 + }, + "SDL_VideoInfo": { + "__size__": 20, + "current_h": 16, + "current_w": 12 + }, + "SDL_WindowEvent": { + "__size__": 20, + "event": 8, + "type": 0, + "windowID": 4 + }, + "SDL_version": { + "__size__": 3, + "major": 0, + "minor": 1, + "patch": 2 + }, + "WGPUAdapterProperties": { + "__size__": 40, + "adapterType": 28, + "architecture": 12, + "backendType": 32, + "compatibilityMode": 36, + "deviceID": 16, + "driverDescription": 24, + "name": 20, + "nextInChain": 0, + "vendorID": 4, + "vendorName": 8 + }, + "WGPUBindGroupDescriptor": { + "__size__": 20, + "entries": 16, + "entryCount": 12, + "label": 4, + "layout": 8, + "nextInChain": 0 + }, + "WGPUBindGroupEntry": { + "__size__": 40, + "binding": 4, + "buffer": 8, + "nextInChain": 0, + "offset": 16, + "sampler": 32, + "size": 24, + "textureView": 36 + }, + "WGPUBindGroupLayoutDescriptor": { + "__size__": 16, + "entries": 12, + "entryCount": 8, + "label": 4, + "nextInChain": 0 + }, + "WGPUBindGroupLayoutEntry": { + "__size__": 80, + "binding": 4, + "buffer": 16, + "nextInChain": 0, + "sampler": 40, + "storageTexture": 64, + "texture": 48, + "visibility": 8 + }, + "WGPUBlendComponent": { + "__size__": 12, + "dstFactor": 8, + "operation": 0, + "srcFactor": 4 + }, + "WGPUBlendState": { + "__size__": 24, + "alpha": 12, + "color": 0 + }, + "WGPUBufferBindingLayout": { + "__size__": 24, + "hasDynamicOffset": 8, + "minBindingSize": 16, + "nextInChain": 0, + "type": 4 + }, + "WGPUBufferDescriptor": { + "__size__": 32, + "label": 4, + "mappedAtCreation": 24, + "nextInChain": 0, + "size": 16, + "usage": 8 + }, + "WGPUChainedStruct": { + "__size__": 8, + "next": 0, + "sType": 4 + }, + "WGPUColor": { + "__size__": 32, + "a": 24, + "b": 16, + "g": 8, + "r": 0 + }, + "WGPUColorTargetState": { + "__size__": 16, + "blend": 8, + "format": 4, + "nextInChain": 0, + "writeMask": 12 + }, + "WGPUCommandBufferDescriptor": { + "__size__": 8, + "label": 4, + "nextInChain": 0 + }, + "WGPUCommandEncoderDescriptor": { + "__size__": 8, + "label": 4, + "nextInChain": 0 + }, + "WGPUCompilationInfo": { + "__size__": 12, + "messageCount": 4, + "messages": 8, + "nextInChain": 0 + }, + "WGPUCompilationMessage": { + "__size__": 72, + "length": 40, + "lineNum": 16, + "linePos": 24, + "message": 4, + "nextInChain": 0, + "offset": 32, + "type": 8, + "utf16Length": 64, + "utf16LinePos": 48, + "utf16Offset": 56 + }, + "WGPUComputePassDescriptor": { + "__size__": 16, + "label": 4, + "nextInChain": 0, + "timestampWriteCount": 8, + "timestampWrites": 12 + }, + "WGPUComputePassTimestampWrite": { + "__size__": 12, + "location": 8, + "queryIndex": 4, + "querySet": 0 + }, + "WGPUComputePipelineDescriptor": { + "__size__": 32, + "compute": 12, + "label": 4, + "layout": 8, + "nextInChain": 0 + }, + "WGPUConstantEntry": { + "__size__": 16, + "key": 4, + "nextInChain": 0, + "value": 8 + }, + "WGPUDepthStencilState": { + "__size__": 68, + "depthBias": 56, + "depthBiasClamp": 64, + "depthBiasSlopeScale": 60, + "depthCompare": 12, + "depthWriteEnabled": 8, + "format": 4, + "nextInChain": 0, + "stencilBack": 32, + "stencilFront": 16, + "stencilReadMask": 48, + "stencilWriteMask": 52 + }, + "WGPUDeviceDescriptor": { + "__size__": 36, + "defaultQueue": 20, + "deviceLostCallback": 28, + "deviceLostUserdata": 32, + "label": 4, + "nextInChain": 0, + "requiredFeatures": 12, + "requiredFeaturesCount": 8, + "requiredLimits": 16 + }, + "WGPUExtent3D": { + "__size__": 12, + "depthOrArrayLayers": 8, + "height": 4, + "width": 0 + }, + "WGPUFragmentState": { + "__size__": 28, + "constantCount": 12, + "constants": 16, + "entryPoint": 8, + "module": 4, + "nextInChain": 0, + "targetCount": 20, + "targets": 24 + }, + "WGPUImageCopyBuffer": { + "__size__": 40, + "buffer": 32, + "layout": 8, + "nextInChain": 0 + }, + "WGPUImageCopyTexture": { + "__size__": 28, + "aspect": 24, + "mipLevel": 8, + "nextInChain": 0, + "origin": 12, + "texture": 4 + }, + "WGPUInstanceDescriptor": { + "__size__": 4, + "nextInChain": 0 + }, + "WGPULimits": { + "__size__": 144, + "maxBindGroups": 16, + "maxBindGroupsPlusVertexBuffers": 20, + "maxBindingsPerBindGroup": 24, + "maxBufferSize": 88, + "maxColorAttachmentBytesPerSample": 116, + "maxColorAttachments": 112, + "maxComputeInvocationsPerWorkgroup": 124, + "maxComputeWorkgroupSizeX": 128, + "maxComputeWorkgroupSizeY": 132, + "maxComputeWorkgroupSizeZ": 136, + "maxComputeWorkgroupStorageSize": 120, + "maxComputeWorkgroupsPerDimension": 140, + "maxDynamicStorageBuffersPerPipelineLayout": 32, + "maxDynamicUniformBuffersPerPipelineLayout": 28, + "maxInterStageShaderComponents": 104, + "maxInterStageShaderVariables": 108, + "maxSampledTexturesPerShaderStage": 36, + "maxSamplersPerShaderStage": 40, + "maxStorageBufferBindingSize": 64, + "maxStorageBuffersPerShaderStage": 44, + "maxStorageTexturesPerShaderStage": 48, + "maxTextureArrayLayers": 12, + "maxTextureDimension1D": 0, + "maxTextureDimension2D": 4, + "maxTextureDimension3D": 8, + "maxUniformBufferBindingSize": 56, + "maxUniformBuffersPerShaderStage": 52, + "maxVertexAttributes": 96, + "maxVertexBufferArrayStride": 100, + "maxVertexBuffers": 80, + "minStorageBufferOffsetAlignment": 76, + "minUniformBufferOffsetAlignment": 72 + }, + "WGPUMultisampleState": { + "__size__": 16, + "alphaToCoverageEnabled": 12, + "count": 4, + "mask": 8, + "nextInChain": 0 + }, + "WGPUOrigin3D": { + "__size__": 12, + "x": 0, + "y": 4, + "z": 8 + }, + "WGPUPipelineLayoutDescriptor": { + "__size__": 16, + "bindGroupLayoutCount": 8, + "bindGroupLayouts": 12, + "label": 4, + "nextInChain": 0 + }, + "WGPUPrimitiveDepthClipControl": { + "__size__": 12, + "chain": 0, + "unclippedDepth": 8 + }, + "WGPUPrimitiveState": { + "__size__": 20, + "cullMode": 16, + "frontFace": 12, + "nextInChain": 0, + "stripIndexFormat": 8, + "topology": 4 + }, + "WGPUProgrammableStageDescriptor": { + "__size__": 20, + "constantCount": 12, + "constants": 16, + "entryPoint": 8, + "module": 4, + "nextInChain": 0 + }, + "WGPUQuerySetDescriptor": { + "__size__": 24, + "count": 12, + "label": 4, + "nextInChain": 0, + "pipelineStatistics": 16, + "pipelineStatisticsCount": 20, + "type": 8 + }, + "WGPUQueueDescriptor": { + "__size__": 8, + "label": 4, + "nextInChain": 0 + }, + "WGPURenderBundleDescriptor": { + "__size__": 8, + "label": 4, + "nextInChain": 0 + }, + "WGPURenderBundleEncoderDescriptor": { + "__size__": 28, + "colorFormats": 12, + "colorFormatsCount": 8, + "depthReadOnly": 24, + "depthStencilFormat": 16, + "label": 4, + "nextInChain": 0, + "sampleCount": 20, + "stencilReadOnly": 25 + }, + "WGPURenderPassColorAttachment": { + "__size__": 56, + "clearValue": 24, + "loadOp": 12, + "nextInChain": 0, + "resolveTarget": 8, + "storeOp": 16, + "view": 4 + }, + "WGPURenderPassDepthStencilAttachment": { + "__size__": 36, + "depthClearValue": 12, + "depthLoadOp": 4, + "depthReadOnly": 16, + "depthStoreOp": 8, + "stencilClearValue": 28, + "stencilLoadOp": 20, + "stencilReadOnly": 32, + "stencilStoreOp": 24, + "view": 0 + }, + "WGPURenderPassDescriptor": { + "__size__": 32, + "colorAttachmentCount": 8, + "colorAttachments": 12, + "depthStencilAttachment": 16, + "label": 4, + "nextInChain": 0, + "occlusionQuerySet": 20, + "timestampWriteCount": 24, + "timestampWrites": 28 + }, + "WGPURenderPassDescriptorMaxDrawCount": { + "__size__": 16, + "chain": 0, + "maxDrawCount": 8 + }, + "WGPURenderPassTimestampWrite": { + "__size__": 12, + "location": 8, + "queryIndex": 4, + "querySet": 0 + }, + "WGPURenderPipelineDescriptor": { + "__size__": 84, + "depthStencil": 60, + "fragment": 80, + "label": 4, + "layout": 8, + "multisample": 64, + "nextInChain": 0, + "primitive": 40, + "vertex": 12 + }, + "WGPURequestAdapterOptions": { + "__size__": 20, + "backendType": 12, + "compatibilityMode": 17, + "compatibleSurface": 4, + "forceFallbackAdapter": 16, + "nextInChain": 0, + "powerPreference": 8 + }, + "WGPURequiredLimits": { + "__size__": 152, + "limits": 8, + "nextInChain": 0 + }, + "WGPUSamplerBindingLayout": { + "__size__": 8, + "nextInChain": 0, + "type": 4 + }, + "WGPUSamplerDescriptor": { + "__size__": 48, + "addressModeU": 8, + "addressModeV": 12, + "addressModeW": 16, + "compare": 40, + "label": 4, + "lodMaxClamp": 36, + "lodMinClamp": 32, + "magFilter": 20, + "maxAnisotropy": 44, + "minFilter": 24, + "mipmapFilter": 28, + "nextInChain": 0 + }, + "WGPUShaderModuleDescriptor": { + "__size__": 8, + "label": 4, + "nextInChain": 0 + }, + "WGPUShaderModuleSPIRVDescriptor": { + "__size__": 16, + "chain": 0, + "code": 12, + "codeSize": 8 + }, + "WGPUShaderModuleWGSLDescriptor": { + "__size__": 12, + "chain": 0, + "code": 8 + }, + "WGPUStencilFaceState": { + "__size__": 16, + "compare": 0, + "depthFailOp": 8, + "failOp": 4, + "passOp": 12 + }, + "WGPUStorageTextureBindingLayout": { + "__size__": 16, + "access": 4, + "format": 8, + "nextInChain": 0, + "viewDimension": 12 + }, + "WGPUSupportedLimits": { + "__size__": 152, + "limits": 8, + "nextInChain": 0 + }, + "WGPUSurfaceDescriptor": { + "__size__": 8, + "label": 4, + "nextInChain": 0 + }, + "WGPUSurfaceDescriptorFromCanvasHTMLSelector": { + "__size__": 12, + "chain": 0, + "selector": 8 + }, + "WGPUSwapChainDescriptor": { + "__size__": 28, + "format": 12, + "height": 20, + "label": 4, + "nextInChain": 0, + "presentMode": 24, + "usage": 8, + "width": 16 + }, + "WGPUTextureBindingLayout": { + "__size__": 16, + "multisampled": 12, + "nextInChain": 0, + "sampleType": 4, + "viewDimension": 8 + }, + "WGPUTextureDataLayout": { + "__size__": 24, + "bytesPerRow": 16, + "nextInChain": 0, + "offset": 8, + "rowsPerImage": 20 + }, + "WGPUTextureDescriptor": { + "__size__": 48, + "dimension": 12, + "format": 28, + "label": 4, + "mipLevelCount": 32, + "nextInChain": 0, + "sampleCount": 36, + "size": 16, + "usage": 8, + "viewFormatCount": 40, + "viewFormats": 44 + }, + "WGPUTextureViewDescriptor": { + "__size__": 36, + "arrayLayerCount": 28, + "aspect": 32, + "baseArrayLayer": 24, + "baseMipLevel": 16, + "dimension": 12, + "format": 8, + "label": 4, + "mipLevelCount": 20, + "nextInChain": 0 + }, + "WGPUVertexAttribute": { + "__size__": 24, + "format": 0, + "offset": 8, + "shaderLocation": 16 + }, + "WGPUVertexBufferLayout": { + "__size__": 24, + "arrayStride": 0, + "attributeCount": 12, + "attributes": 16, + "stepMode": 8 + }, + "WGPUVertexState": { + "__size__": 28, + "bufferCount": 20, + "buffers": 24, + "constantCount": 12, + "constants": 16, + "entryPoint": 8, + "module": 4, + "nextInChain": 0 + }, + "__cxa_exception": { + "__size__": 24, + "adjustedPtr": 16, + "caught": 12, + "exceptionDestructor": 8, + "exceptionType": 4, + "referenceCount": 0, + "rethrown": 13 + }, + "__wasi_fdstat_t": { + "__size__": 24, + "fs_filetype": 0, + "fs_flags": 2, + "fs_rights_base": 8, + "fs_rights_inheriting": 16 + }, + "__wasi_filestat_t": { + "__size__": 64, + "atim": 40, + "ctim": 56, + "dev": 0, + "filetype": 16, + "ino": 8, + "mtim": 48, + "nlink": 24, + "size": 32 + }, + "__wasi_prestat_dir_t": { + "__size__": 4, + "pr_name_len": 0 + }, + "__wasi_prestat_t": { + "__size__": 8, + "pr_type": 0, + "u": 4 + }, + "addrinfo": { + "__size__": 32, + "ai_addr": 20, + "ai_addrlen": 16, + "ai_canonname": 24, + "ai_family": 4, + "ai_flags": 0, + "ai_next": 28, + "ai_protocol": 12, + "ai_socktype": 8 + }, + "asyncify_data_s": { + "__size__": 12, + "rewind_id": 8, + "stack_limit": 4, + "stack_ptr": 0 + }, + "dirent": { + "__size__": 280, + "d_ino": 0, + "d_name": 19, + "d_off": 8, + "d_reclen": 16, + "d_type": 18 + }, + "dso": { + "__size__": 36, + "file_data": 28, + "file_data_size": 32, + "flags": 4, + "mem_addr": 12, + "mem_allocated": 8, + "mem_size": 16, + "name": 36, + "table_addr": 20, + "table_size": 24 + }, + "em_settled_result_t": { + "__size__": 8, + "result": 0, + "value": 4 + }, + "emscripten_fetch_attr_t": { + "__size__": 92, + "attributes": 52, + "destinationPath": 64, + "onerror": 40, + "onprogress": 44, + "onreadystatechange": 48, + "onsuccess": 36, + "overriddenMimeType": 80, + "password": 72, + "requestData": 84, + "requestDataSize": 88, + "requestHeaders": 76, + "requestMethod": 0, + "timeoutMSecs": 56, + "userData": 32, + "userName": 68, + "withCredentials": 60 + }, + "emscripten_fetch_t": { + "__attributes": 112, + "__proxyState": 108, + "__size__": 208, + "data": 12, + "dataOffset": 24, + "id": 0, + "numBytes": 16, + "readyState": 40, + "status": 42, + "statusText": 44, + "totalBytes": 32, + "url": 8, + "userData": 4 + }, + "emscripten_fiber_s": { + "__size__": 32, + "asyncify_data": 20, + "entry": 12, + "stack_base": 0, + "stack_limit": 4, + "stack_ptr": 8, + "user_data": 16 + }, + "flock": { + "__size__": 32, + "l_type": 0 + }, + "hostent": { + "__size__": 20, + "h_addr_list": 16, + "h_addrtype": 8, + "h_aliases": 4, + "h_length": 12, + "h_name": 0 + }, + "iovec": { + "__size__": 8, + "iov_base": 0, + "iov_len": 4 + }, + "msghdr": { + "__size__": 28, + "msg_iov": 8, + "msg_iovlen": 12, + "msg_name": 0, + "msg_namelen": 4 + }, + "pollfd": { + "__size__": 8, + "events": 4, + "fd": 0, + "revents": 6 + }, + "protoent": { + "__size__": 12, + "p_aliases": 4, + "p_name": 0, + "p_proto": 8 + }, + "pthread": { + "__size__": 132, + "profilerBlock": 112, + "stack": 52, + "stack_size": 56, + "waiting_async": 128 + }, + "pthread_attr_t": { + "__size__": 44, + "_a_transferredcanvases": 40 + }, + "sockaddr_in": { + "__size__": 16, + "sin_addr": { + "__size__": 4, + "s_addr": 4 + }, + "sin_family": 0, + "sin_port": 2 + }, + "sockaddr_in6": { + "__size__": 28, + "sin6_addr": { + "__in6_union": { + "__s6_addr": 8, + "__s6_addr16": 8, + "__s6_addr32": 8, + "__size__": 16 + }, + "__size__": 16 + }, + "sin6_family": 0, + "sin6_port": 2 + }, + "stat": { + "__size__": 96, + "st_atim": { + "__size__": 16, + "tv_nsec": 48, + "tv_sec": 40 + }, + "st_blksize": 32, + "st_blocks": 36, + "st_ctim": { + "__size__": 16, + "tv_nsec": 80, + "tv_sec": 72 + }, + "st_dev": 0, + "st_gid": 16, + "st_ino": 88, + "st_mode": 4, + "st_mtim": { + "__size__": 16, + "tv_nsec": 64, + "tv_sec": 56 + }, + "st_nlink": 8, + "st_rdev": 20, + "st_size": 24, + "st_uid": 12 + }, + "statfs": { + "__size__": 64, + "f_bavail": 16, + "f_bfree": 12, + "f_blocks": 8, + "f_bsize": 4, + "f_ffree": 24, + "f_files": 20, + "f_flags": 44, + "f_frsize": 40, + "f_fsid": 28, + "f_namelen": 36 + }, + "termios": { + "__c_ispeed": 52, + "__c_ospeed": 56, + "__size__": 60, + "c_cc": 17, + "c_cflag": 8, + "c_iflag": 0, + "c_lflag": 12, + "c_line": 16, + "c_oflag": 4 + }, + "thread_profiler_block": { + "__size__": 104, + "name": 72, + "threadStatus": 0, + "timeSpentInStatus": 16 + }, + "timespec": { + "__size__": 16, + "tv_nsec": 8, + "tv_sec": 0 + }, + "timeval": { + "__size__": 16, + "tv_sec": 0, + "tv_usec": 8 + }, + "tm": { + "__size__": 44, + "tm_gmtoff": 36, + "tm_hour": 8, + "tm_isdst": 32, + "tm_mday": 12, + "tm_min": 4, + "tm_mon": 16, + "tm_sec": 0, + "tm_wday": 24, + "tm_yday": 28, + "tm_year": 20, + "tm_zone": 40 + } + } +} diff --git a/src/js/generated_struct_info32.json.d.ts b/src/js/generated_struct_info32.json.d.ts new file mode 100644 index 000000000..6a2845996 --- /dev/null +++ b/src/js/generated_struct_info32.json.d.ts @@ -0,0 +1,2 @@ +// Trick dts-bundle-generator into behaving correctly +export declare const defines: { [k: string]: number }; diff --git a/src/js/load-package.ts b/src/js/load-package.ts index 38f3b0521..b2b15d581 100644 --- a/src/js/load-package.ts +++ b/src/js/load-package.ts @@ -1,7 +1,4 @@ -declare var Module: any; -declare var Tests: any; -declare var API: any; -declare var DEBUG: boolean; +import "./constants"; import { IN_NODE, diff --git a/src/js/module.ts b/src/js/module.ts index 6c77ce955..4bda84f23 100644 --- a/src/js/module.ts +++ b/src/js/module.ts @@ -4,20 +4,54 @@ import { ConfigType } from "./pyodide"; import { initializeNativeFS } from "./nativefs"; import { loadBinaryFile } from "./compat"; -type FSNode = any; -type FSStream = any; +export type FSNode = { + timestamp: number; + rdev: number; + contents: Uint8Array; +}; + +export type FSStream = { + tty?: boolean; + seekable?: boolean; + stream_ops: FSStreamOps; + node: FSNode; +}; + +export type FSStreamOps = FSStreamOpsGen; + +export type FSStreamOpsGen = { + open: (a: T) => void; + close: (a: T) => void; + fsync: (a: T) => void; + read: ( + a: T, + b: Uint8Array, + offset: number, + length: number, + pos: number, + ) => number; + write: ( + a: T, + b: Uint8Array, + offset: number, + length: number, + pos: number, + ) => number; +}; export interface FS { unlink: (path: string) => void; mkdirTree: (path: string, mode?: number) => void; chdir: (path: string) => void; symlink: (target: string, src: string) => FSNode; - createDevice: ( + createDevice: (( parent: string, name: string, input?: (() => number | null) | null, output?: ((code: number) => void) | null, - ) => FSNode; + ) => FSNode) & { + major: number; + }; closeStream: (fd: number) => void; open: (path: string, flags: string | number, mode?: number) => FSStream; makedev: (major: number, minor: number) => number; @@ -26,7 +60,7 @@ export interface FS { stat: (path: string, dontFollow?: boolean) => any; readdir: (node: FSNode) => string[]; isDir: (mode: number) => boolean; - lookupPath: (path: string) => FSNode; + lookupPath: (path: string) => { node: FSNode }; isFile: (mode: number) => boolean; writeFile: (path: string, contents: any, o?: { canOwn?: boolean }) => void; chmod: (path: string, mode: number) => void; @@ -41,6 +75,8 @@ export interface FS { position?: number, ) => number; close: (stream: FSStream) => void; + ErrnoError: { new (errno: number): Error }; + registerDevice(dev: number, ops: FSStreamOpsGen): void; } export interface Module { @@ -58,6 +94,8 @@ export interface Module { canvas?: HTMLCanvasElement; addRunDependency: (id: string) => void; removeRunDependency: (id: string) => void; + reportUndefinedSymbols: () => void; + ERRNO_CODES: { [k: string]: number }; } /** diff --git a/src/js/streams.ts b/src/js/streams.ts index 93a24ced6..5bf9613da 100644 --- a/src/js/streams.ts +++ b/src/js/streams.ts @@ -1,16 +1,40 @@ import { IN_NODE } from "./compat.js"; -import type { Module } from "./module"; +import "./constants"; -declare var API: any; -declare var Module: Module; +import type { FSStream, FSStreamOpsGen } from "./module"; +const fs: any = IN_NODE ? require("fs") : undefined; +const tty: any = IN_NODE ? require("tty") : undefined; + +function nodeFsync(fd: number): void { + try { + fs.fsyncSync(fd); + } catch (e: any) { + if (e && e.code === "EINVAL") { + return; + } + throw e; + } +} + +type Reader = { + isatty?: boolean; + fsync?: () => void; + read(buffer: Uint8Array): number; +}; +type Writer = { + isatty?: boolean; + fsync?: () => void; + write(buffer: Uint8Array): number; +}; + +type Stream = FSStream & { + stream_ops: StreamOps; + devops: Reader & Writer; +}; + +type StreamOps = FSStreamOpsGen; declare var FS: typeof Module.FS; -declare var TTY: any; - -// The type of the function we need to produce to read from stdin -// (Either directly from a device when isatty is false or as part of a tty when -// isatty is true) -type GetCharType = () => null | number; // The type of the function we expect the user to give us. make_get_char takes // one of these and turns it into a GetCharType function for us. @@ -23,21 +47,6 @@ export type InFuncType = () => | Uint8Array | number; -// To define the output behavior of a tty we need to define put_char and fsync. -// fsync flushes the stream. -// -// If isatty is false, we ignore fsync and use put_char.bind to fill in a dummy -// value for the tty argument. We don't ever use the tty argument. -type PutCharType = { - put_char: (tty: void, val: number) => void; - fsync: (tty: void) => void; -}; - -// A tty needs both a GetChar function and a PutChar pair. -type TtyOps = { - get_char: GetCharType; -} & PutCharType; - /** * We call refreshStreams at the end of every update method, but refreshStreams * won't work until initializeStreams is called. So when INITIALIZED is false, @@ -45,56 +54,158 @@ type TtyOps = { * @private */ let INITIALIZED = false; +const DEVOPS: { [k: number]: Reader & Writer } = {}; +// DEVS is initialized in initializeStreams +const DEVS = {} as { + stdin: number; + stdout: number; + stderr: number; +}; -// These can't be used until they are initialized by initializeStreams. -const ttyout_ops = {} as TtyOps; -const ttyerr_ops = {} as TtyOps; -const isattys = {} as { - stdin: boolean; - stdout: boolean; - stderr: boolean; +function _setStdinOps(ops: Reader) { + DEVOPS[DEVS.stdin] = ops as Reader & Writer; +} + +function _setStdoutOps(ops: Writer) { + DEVOPS[DEVS.stdout] = ops as Reader & Writer; +} + +function _setStderrOps(ops: Writer) { + DEVOPS[DEVS.stderr] = ops as Reader & Writer; +} + +function isErrnoError(e: any) { + return e && typeof e === "object" && "errno" in e; +} + +const waitBuffer = new Int32Array( + new WebAssembly.Memory({ shared: true, initial: 1, maximum: 1 }).buffer, +); +function syncSleep(timeout: number): boolean { + try { + Atomics.wait(waitBuffer, 0, 0, timeout); + return true; + } catch (_) { + return false; + } +} + +function readHelper(devops: Reader, buffer: Uint8Array): number { + while (true) { + try { + return devops.read(buffer); + } catch (e: any) { + if (e && e.code === "EAGAIN") { + // Presumably this means we're in node and tried to read from an + // O_NONBLOCK file descriptor. Synchronously sleep for 100ms as + // requested by EAGAIN and try again. In case for some reason we fail to + // sleep, propagate the error (it will turn into an EOFError). + if (syncSleep(100)) { + continue; + } + } + throw e; + } + } +} + +const stream_ops: StreamOps = { + open: function (stream) { + const devops = DEVOPS[stream.node.rdev]; + if (!devops) { + throw new FS.ErrnoError(cDefs.ENODEV); + } + stream.devops = devops; + stream.tty = stream.devops.isatty; + stream.seekable = false; + }, + close: function (stream) { + // flush any pending line data + stream.stream_ops.fsync(stream); + }, + fsync: function (stream) { + const ops = stream.devops; + if (ops.fsync) { + ops.fsync(); + } + }, + read: function (stream, buffer, offset, length, pos /* ignored */) { + buffer = API.typedArrayAsUint8Array(buffer).subarray( + offset, + offset + length, + ); + let bytesRead; + try { + bytesRead = readHelper(stream.devops, buffer); + } catch (e: any) { + if (e && e.code && Module.ERRNO_CODES[e.code]) { + throw new FS.ErrnoError(Module.ERRNO_CODES[e.code]); + } + if (isErrnoError(e)) { + // the handler set an errno, propagate it + throw e; + } + console.error("Error thrown in read:"); + console.error(e); + throw new FS.ErrnoError(cDefs.EIO); + } + if (bytesRead === undefined) { + // Prevent an infinite loop caused by incorrect code that doesn't return a + // value + // + // Maybe we should set bytesWritten = buffer.length here instead? + console.warn( + "read returned undefined; a correct implementation must return a number", + ); + throw new FS.ErrnoError(cDefs.EIO); + } + if (bytesRead !== 0) { + stream.node.timestamp = Date.now(); + } + return bytesRead; + }, + write: function (stream, buffer, offset, length, pos /* ignored */): number { + buffer = API.typedArrayAsUint8Array(buffer); + let bytesWritten; + try { + bytesWritten = stream.devops.write( + buffer.subarray(offset, offset + length), + ); + } catch (e) { + if (isErrnoError(e)) { + throw e; + } + console.error("Error thrown in write:"); + console.error(e); + throw new FS.ErrnoError(cDefs.EIO); + } + if (bytesWritten === undefined) { + // Prevent an infinite loop caused by incorrect code that doesn't return a + // value + // + // Maybe we should set bytesWritten = buffer.length here instead? + console.warn( + "write returned undefined; a correct implementation must return a number", + ); + throw new FS.ErrnoError(cDefs.EIO); + } + if (length) { + stream.node.timestamp = Date.now(); + } + return bytesWritten; + }, }; function refreshStreams() { if (!INITIALIZED) { return; } - FS.unlink("/dev/stdin"); - FS.unlink("/dev/stdout"); - FS.unlink("/dev/stderr"); - if (isattys.stdin) { - FS.symlink("/dev/tty", "/dev/stdin"); - } else { - FS.createDevice("/dev", "stdin", ttyout_ops.get_char); - } - if (isattys.stdout) { - FS.symlink("/dev/tty", "/dev/stdout"); - } else { - FS.createDevice( - "/dev", - "stdout", - null, - ttyout_ops.put_char.bind(undefined, undefined), - ); - } - if (isattys.stderr) { - FS.symlink("/dev/tty", "/dev/stderr"); - } else { - FS.createDevice( - "/dev", - "stderr", - null, - ttyerr_ops.put_char.bind(undefined, undefined), - ); - } - - // Refresh std streams so they use our new versions FS.closeStream(0 /* stdin */); FS.closeStream(1 /* stdout */); FS.closeStream(2 /* stderr */); - FS.open("/dev/stdin", 0 /* write only */); - FS.open("/dev/stdout", 1 /* read only */); - FS.open("/dev/stderr", 1 /* read only */); + FS.open("/dev/stdin", cDefs.O_RDONLY); + FS.open("/dev/stdout", cDefs.O_WRONLY); + FS.open("/dev/stderr", cDefs.O_WRONLY); } /** @@ -112,25 +223,27 @@ API.initializeStreams = function ( stdout?: (a: string) => void, stderr?: (a: string) => void, ) { - setStdin({ stdin }); - if (stdout) { - setStdout({ batched: stdout }); - } else { - setDefaultStdout(); - } + const major = FS.createDevice.major++; + DEVS.stdin = FS.makedev(major, 0); + DEVS.stdout = FS.makedev(major, 1); + DEVS.stderr = FS.makedev(major, 2); + + FS.registerDevice(DEVS.stdin, stream_ops); + FS.registerDevice(DEVS.stdout, stream_ops); + FS.registerDevice(DEVS.stderr, stream_ops); + + FS.unlink("/dev/stdin"); + FS.unlink("/dev/stdout"); + FS.unlink("/dev/stderr"); + + FS.mkdev("/dev/stdin", DEVS.stdin); + FS.mkdev("/dev/stdout", DEVS.stdout); + FS.mkdev("/dev/stderr", DEVS.stderr); + + setStdin({ stdin }); + setStdout({ batched: stdout }); + setStderr({ batched: stderr }); - if (stderr) { - setStderr({ batched: stderr }); - } else { - setDefaultStderr(); - } - // 5.0 and 6.0 are the device numbers that Emscripten uses (see library_fs.js). - // These haven't changed in ~10 years. If we used different ones nothing would - // break. - const ttyout_dev = FS.makedev(5, 0); - const ttyerr_dev = FS.makedev(6, 0); - TTY.register(ttyout_dev, ttyout_ops); - TTY.register(ttyerr_dev, ttyerr_ops); INITIALIZED = true; refreshStreams(); }; @@ -142,33 +255,9 @@ API.initializeStreams = function ( */ function setDefaultStdin() { if (IN_NODE) { - const BUFSIZE = 256; - const buf = Buffer.alloc(BUFSIZE); - const fs = require("fs"); - const tty = require("tty"); - const stdin = function () { - let bytesRead; - try { - bytesRead = fs.readSync(process.stdin.fd, buf, 0, BUFSIZE, -1); - } catch (e) { - // Platform differences: on Windows, reading EOF throws an exception, - // but on other OSes, reading EOF returns 0. Uniformize behavior by - // catching the EOF exception and returning 0. - if ((e as Error).toString().includes("EOF")) { - bytesRead = 0; - } else { - throw e; - } - } - if (bytesRead === 0) { - return null; - } - return buf.subarray(0, bytesRead); - }; - const isatty: boolean = tty.isatty(process.stdin.fd); - setStdin({ stdin, isatty }); + setStdin(new NodeReader(process.stdin.fd)); } else { - setStdinError(); + setStdin({ stdin: () => prompt() }); } } @@ -177,73 +266,149 @@ function setDefaultStdin() { * error. */ function setStdinError() { - isattys.stdin = false; - const get_char = () => { - throw 0; - }; - ttyout_ops.get_char = get_char; - ttyerr_ops.get_char = get_char; + _setStdinOps(new ErrorReader()); refreshStreams(); } +type StdinOptions = { + stdin?: InFuncType; + error?: boolean; + isatty?: boolean; + autoEOF?: boolean; +}; + /** - * Set a stdin handler. + * Set a stdin handler. See :ref:`redirecting standard streams ` + * for a more detailed explanation. There are two different possible interfaces + * to implement a handler. It's also possible to select either the default + * handler or an error handler that always returns an IO error. * - * The stdin handler is called with zero arguments whenever stdin is read and - * the current input buffer is exhausted. It should return one of: + * 1. passing a ``read`` function (see below), + * 2. passing a ``stdin`` function (see below), + * 3. passing ``error: true`` indicates that attempting to read from stdin + * should always raise an IO error. + * 4. passing none of these sets the default behavior. In node, the default is + * to read from stdin. In the browser, the default is to raise an error. * - * - :js:data:`null` or :js:data:`undefined`: these are interpreted as end of file. - * - a number - * - a string - * - an :js:class:`ArrayBuffer` or :js:class:`TypedArray` with - * :js:data:`~TypedArray.BYTES_PER_ELEMENT` equal to 1. + * The functions on the ``options`` argument will be called with ``options`` + * bound to ``this`` so passing an instance of a class as the ``options`` object + * works as expected. * - * If a number is returned, it is interpreted as a single character code. The - * number should be between 0 and 255. + * The interfaces that the handlers implement are as follows: * - * If a string is returned, a new line is appended if one is not present and the - * resulting string is turned into a :js:class:`Uint8Array` using - * :js:class:`TextEncoder`. + * 1. The ``read`` function is called with a ``Uint8Array`` argument. The + * function should place the utf8-encoded input into this buffer and return + * the number of bytes written. For instance, if the buffer was completely + * filled with input, then return `buffer.length`. If a ``read`` function is + * passed you may optionally also pass an ``fsync`` function which is called + * when stdin is flushed. * - * Returning a buffer is more efficient and allows returning partial lines of - * text. + * 2. The ``stdin`` function is called with zero arguments. It should return one + * of: * - * @param options.stdin The stdin handler. + * - :js:data:`null` or :js:data:`undefined`: these are interpreted as end of + * file. + * - a number + * - a string + * - an :js:class:`ArrayBuffer` or :js:class:`TypedArray` with + * :js:data:`~TypedArray.BYTES_PER_ELEMENT` equal to 1. The buffer should + * contain utf8 encoded text. + * + * If a number is returned, it is interpreted as a single character code. The + * number should be between 0 and 255. + * + * If a string is returned, it is encoded into a buffer using + * :js:class:`TextEncoder`. By default, an EOF is appended after each string + * or buffer returned. If this behavior is not desired, pass `autoEOF: + * false`. + * + * @param options.stdin A stdin handler + * @param options.read A read handler * @param options.error If this is set to ``true``, attempts to read from stdin * will always set an IO error. * @param options.isatty Should :py:func:`isatty(stdin) ` be ``true`` * or ``false`` (default ``false``). * @param options.autoEOF Insert an EOF automatically after each string or - * buffer? (default ``true``). + * buffer? (default ``true``). This option can only be used with the stdin + * handler. */ export function setStdin( options: { stdin?: InFuncType; + read?: (buffer: Uint8Array) => number; error?: boolean; isatty?: boolean; autoEOF?: boolean; } = {}, ) { - if (options.stdin && options.error) { + let { stdin, error, isatty, autoEOF, read } = options as StdinOptions & + Partial; + const numset = +!!stdin + +!!error + +!!read; + if (numset > 1) { throw new TypeError( - "Both a stdin handler provided and the error argument was set", + "At most one of stdin, read, and error must be provided.", ); } - if (options.error) { + if (!stdin && autoEOF !== undefined) { + throw new TypeError( + "The 'autoEOF' option can only be used with the 'stdin' option", + ); + } + if (numset === 0) { + setDefaultStdin(); + return; + } + if (error) { setStdinError(); - return; } - if (options.stdin) { - let autoEOF = options.autoEOF; + if (stdin) { autoEOF = autoEOF === undefined ? true : autoEOF; - isattys.stdin = !!options.isatty; - const get_char = make_get_char(options.stdin, autoEOF); - ttyout_ops.get_char = get_char; - ttyerr_ops.get_char = get_char; - refreshStreams(); - return; + _setStdinOps(new LegacyReader(stdin.bind(options), !!isatty, autoEOF)); } - setDefaultStdin(); + if (read) { + _setStdinOps(options as Reader); + } + refreshStreams(); +} + +type StdwriteOpts = { + batched?: (a: string) => void; + raw?: (a: number) => void; + isatty?: boolean; +}; + +function _setStdwrite( + options: StdwriteOpts & Partial, + setOps: (ops: Writer) => void, + getDefaults: () => StdwriteOpts & Partial, +) { + let { raw, isatty, batched, write } = options as StdwriteOpts & + Partial; + let nset = +!!raw + +!!batched + +!!write; + if (nset === 0) { + options = getDefaults(); + ({ raw, isatty, batched, write } = options); + } + if (nset > 1) { + throw new TypeError( + "At most one of 'raw', 'batched', and 'write' must be passed", + ); + } + if (!raw && !write && isatty) { + throw new TypeError( + "Cannot set 'isatty' to true unless 'raw' or 'write' is provided", + ); + } + if (raw) { + setOps(new CharacterCodeWriter(raw.bind(options), !!isatty)); + } + if (batched) { + setOps(new StringWriter(batched.bind(options))); + } + if (write) { + setOps(options as Writer); + } + refreshStreams(); } /** @@ -251,31 +416,48 @@ export function setStdin( * to tty.isatty(process.stdout.fd). * If in a browser, sets stdout to write to console.log and sets isatty(stdout) to false. */ -function setDefaultStdout() { +function _getStdoutDefaults(): StdwriteOpts & Partial { if (IN_NODE) { - const tty = require("tty"); - const raw = (x: number) => process.stdout.write(Buffer.from([x])); - const isatty: boolean = tty.isatty(process.stdout.fd); - setStdout({ raw, isatty }); + return new NodeWriter(process.stdout.fd); } else { - setStdout({ batched: (x) => console.log(x) }); + return { batched: (x) => console.log(x) }; } } /** - * Sets the standard out handler. A batched handler or a raw handler can be - * provided (but not both). If neither is provided, we restore the default + * If in node, sets stdout to write directly to process.stdout and sets isatty(stdout) + * to tty.isatty(process.stdout.fd). + * If in a browser, sets stdout to write to console.log and sets isatty(stdout) to false. + */ +function _getStderrDefaults(): StdwriteOpts & Partial { + if (IN_NODE) { + return new NodeWriter(process.stderr.fd); + } else { + return { batched: (x) => console.warn(x) }; + } +} + +/** + * Sets the standard out handler. A batched handler, a raw handler, or a write + * function can be provided. If no handler is provided, we restore the default * handler. * + * The functions on the ``options`` argument will be called with ``options`` + * bound to ``this`` so passing an instance of a class as the ``options`` object + * works as expected. + * * @param options.batched A batched handler is called with a string whenever a * newline character is written is written or stdout is flushed. In the former * case, the received line will end with a newline, in the latter case it will * not. * @param options.raw A raw handler is called with the handler is called with a * `number` for each byte of the output to stdout. + * @param options.write A write handler is called with a buffer that contains + * the utf8 encoded binary data * @param options.isatty Should :py:func:`isatty(stdout) ` return - * ``true`` or ``false``. Can only be set to ``true`` if a raw handler is - * provided (default ``false``). + * ``true`` or ``false``. Must be ``false`` if a batched handler is used. + * (default ``false``). + * * @example * async function main(){ * const pyodide = await loadPyodide(); @@ -293,203 +475,246 @@ function setDefaultStdout() { */ export function setStdout( options: { - batched?: (a: string) => void; - raw?: (a: number) => void; + batched?: (output: string) => void; + raw?: (charCode: number) => void; + write?: (buffer: Uint8Array) => number; isatty?: boolean; } = {}, ) { - if (options.raw && options.batched) { - throw new TypeError("Both a batched handler and a raw handler provided"); - } - if (!options.raw && options.isatty) { - throw new TypeError( - "Cannot set isatty to true unless a raw handler is provided", - ); - } - if (options.raw) { - isattys.stdout = !!options.isatty; - Object.assign(ttyout_ops, make_unbatched_put_char(options.raw)); - refreshStreams(); - return; - } - if (options.batched) { - isattys.stdout = false; - Object.assign(ttyout_ops, make_batched_put_char(options.batched)); - refreshStreams(); - return; - } - setDefaultStdout(); + _setStdwrite(options, _setStdoutOps, _getStdoutDefaults); } /** - * If in node, sets stderr to write directly to process.stderr and sets isatty(stderr) - * to tty.isatty(process.stderr.fd). - * If in a browser, sets stderr to write to console.warn and sets isatty(stderr) to false. - */ -function setDefaultStderr() { - if (IN_NODE) { - const tty = require("tty"); - const raw = (x: number) => process.stderr.write(Buffer.from([x])); - const isatty: boolean = tty.isatty(process.stderr.fd); - setStderr({ raw, isatty }); - } else { - setStderr({ batched: (x) => console.warn(x) }); - } -} - -/** - * Sets the standard error handler. A batched handler or a raw handler can be - * provided (but not both). If neither is provided, we restore the default - * handler. - * - * @param options.batched A batched handler is called with a string whenever a - * newline character is written is written or stderr is flushed. In the former - * case, the received line will end with a newline, in the latter case it will - * not. isatty(stderr) is set to false (when using a batched handler, stderr is - * buffered so it is impossible to make a tty with it). - * @param options.raw A raw handler is called with the handler is called with a - * `number` for each byte of the output to stderr. - * @param options.isatty Should :py:func:`isatty(stderr) ` return - * ``true`` or ``false``. Can only be set to ``true`` if a raw handler is - * provided (default ``false``). - * @example - * async function main(){ - * const pyodide = await loadPyodide(); - * pyodide.setStderr({ batched: (msg) => console.warn(msg) }); - * pyodide.runPython("import sys; print('ABC', file=sys.stderr)"); - * // 'ABC' - * pyodide.setStderr({ raw: (byte) => console.warn(byte) }); - * pyodide.runPython("import sys; print('ABC', file=sys.stderr)"); - * // 65 - * // 66 - * // 67 - * // 10 (the ascii values for 'ABC' including a new line character) - * } - * main(); + * Sets the standard error handler. See the documentation for + * :js:func:`pyodide.setStdout`. */ export function setStderr( options: { - batched?: (a: string) => void; - raw?: (a: number) => void; + batched?: (output: string) => void; + raw?: (charCode: number) => void; + write?: (buffer: Uint8Array) => number; isatty?: boolean; } = {}, ) { - if (options.raw && options.batched) { - throw new TypeError("Both a batched handler and a raw handler provided"); - } - if (!options.raw && options.isatty) { - throw new TypeError( - "Cannot set isatty to true unless a raw handler is provided", - ); - } - if (options.raw) { - isattys.stderr = !!options.isatty; - Object.assign(ttyerr_ops, make_unbatched_put_char(options.raw)); - refreshStreams(); - return; - } - if (options.batched) { - isattys.stderr = false; - Object.assign(ttyerr_ops, make_batched_put_char(options.batched)); - refreshStreams(); - return; - } - setDefaultStderr(); + _setStdwrite(options, _setStderrOps, _getStderrDefaults); } const textencoder = new TextEncoder(); const textdecoder = new TextDecoder(); -function make_get_char(infunc: InFuncType, autoEOF: boolean): GetCharType { - let index = 0; - let buf: Uint8Array = new Uint8Array(0); - let insertEOF = false; - // get_char has 3 particular return values: - // a.) the next character represented as an integer - // b.) undefined to signal that no data is currently available - // c.) null to signal an EOF - return function get_char() { +// Reader implementations + +class ErrorReader { + read(buffer: Uint8Array): number { + // always set an IO error. + throw new FS.ErrnoError(cDefs.EIO); + } +} + +class NodeReader { + fd: number; + isatty: boolean; + + constructor(fd: number) { + this.fd = fd; + this.isatty = tty.isatty(fd); + } + + read(buffer: Uint8Array): number { try { - if (index >= buf.length) { - if (insertEOF) { - insertEOF = false; - return null; - } - let input = infunc(); - if (input === undefined || input === null) { - return null; - } - if (typeof input === "number") { - return input; - } else if (typeof input === "string") { - if (!input.endsWith("\n")) { - input += "\n"; - } - buf = textencoder.encode(input); - } else if (ArrayBuffer.isView(input)) { - if ((input as any).BYTES_PER_ELEMENT !== 1) { - throw new Error("Expected BYTES_PER_ELEMENT to be 1"); - } - buf = input as Uint8Array; - } else if ( - Object.prototype.toString.call(input) === "[object ArrayBuffer]" - ) { - buf = new Uint8Array(input); - } else { - throw new Error( - "Expected result to be undefined, null, string, array buffer, or array buffer view", - ); - } - if (buf.length === 0) { - return null; - } - if (autoEOF) { - insertEOF = true; - } - index = 0; - } - return buf[index++]; + return fs.readSync(this.fd, buffer); } catch (e) { - // emscripten will catch this and set an IOError which is unhelpful for - // debugging. - console.error("Error thrown in stdin:"); - console.error(e); + // Platform differences: on Windows, reading EOF throws an exception, + // but on other OSes, reading EOF returns 0. Uniformize behavior by + // catching the EOF exception and returning 0. + if ((e as Error).toString().includes("EOF")) { + return 0; + } throw e; } - }; + } + + fsync() { + nodeFsync(this.fd); + } } -function make_unbatched_put_char(out: (a: number) => void): PutCharType { - return { - put_char(tty: any, val: number) { - out(val); - }, - fsync() {}, - }; -} +class LegacyReader { + infunc: InFuncType; + autoEOF: boolean; + index: number; + saved: Uint8Array | string | undefined; + insertEOF: boolean; + isatty: boolean; -function make_batched_put_char(out: (a: string) => void): PutCharType { - let output: number[] = []; - return { - // get_char has 3 particular return values: - // a.) the next character represented as an integer - // b.) undefined to signal that no data is currently available - // c.) null to signal an EOF, - put_char(tty: any, val: number) { - if (val === null || val === 10 /* charCode('\n') */) { - out(textdecoder.decode(new Uint8Array(output))); - output = []; - } else { - if (val !== 0) { - output.push(val); // val == 0 would cut text output off in the middle. + constructor(infunc: InFuncType, isatty: boolean, autoEOF: boolean) { + this.infunc = infunc; + this.isatty = isatty; + this.autoEOF = autoEOF; + this.index = 0; + this.saved = undefined; + this.insertEOF = false; + } + + _getInput(): Uint8Array | string | number | undefined { + if (this.saved) { + return this.saved; + } + let val; + try { + val = this.infunc(); + } catch (e) { + if (isErrnoError(e)) { + // Allow infunc to set other errno + throw e; + } + // Since we're throwing a new error without the traceback, let people know + // what the original cause was. + console.error("Error thrown in stdin:"); + console.error(e); + throw new FS.ErrnoError(cDefs.EIO); + } + if (typeof val === "number") { + return val; + } + if (!val) { + return undefined; + } + if (ArrayBuffer.isView(val)) { + if ((val as any).BYTES_PER_ELEMENT !== 1) { + console.warn( + `Expected BYTES_PER_ELEMENT to be 1, infunc gave ${val.constructor}`, + ); + throw new FS.ErrnoError(cDefs.EIO); + } + return val; + } + if (typeof val === "string") { + if (!val.endsWith("\n")) { + val += "\n"; + } + return val; + } + if (Object.prototype.toString.call(val) === "[object ArrayBuffer]") { + return new Uint8Array(val as ArrayBuffer); + } + console.warn( + "Expected result to be undefined, null, string, array buffer, or array buffer view", + ); + throw new FS.ErrnoError(cDefs.EIO); + } + + read(buffer: Uint8Array): number { + if (this.insertEOF) { + this.insertEOF = false; + return 0; + } + let bytesRead = 0; + while (true) { + let val = this._getInput(); + if (typeof val === "number") { + buffer[0] = val; + buffer = buffer.subarray(1); + bytesRead++; + continue; + } + let lastwritten; + if (val && val.length > 0) { + if (typeof val === "string") { + let { read, written } = textencoder.encodeInto(val, buffer); + this.saved = val.slice(read); + bytesRead += written!; + lastwritten = buffer[written! - 1]; + buffer = buffer.subarray(written); + } else { + let written; + if (val.length > buffer.length) { + buffer.set(val.subarray(0, buffer.length)); + this.saved = val.subarray(buffer.length); + written = buffer.length; + } else { + buffer.set(val); + this.saved = undefined; + written = val.length; + } + bytesRead += written; + lastwritten = buffer[written - 1]; + buffer = buffer.subarray(written); } } - }, - fsync(tty: any) { - if (output && output.length > 0) { - out(textdecoder.decode(new Uint8Array(output))); - output = []; + if (!(val && val.length > 0) || this.autoEOF || buffer.length === 0) { + this.insertEOF = bytesRead > 0 && this.autoEOF && lastwritten !== 10; + return bytesRead; } - }, - }; + } + } + + fsync() {} +} + +// Writer implementations + +class CharacterCodeWriter { + out: (a: number) => void; + isatty: boolean; + + constructor(out: (a: number) => void, isatty: boolean) { + this.out = out; + this.isatty = isatty; + } + + write(buffer: Uint8Array) { + for (let i of buffer) { + this.out(i); + } + return buffer.length; + } +} + +class StringWriter { + out: (a: string) => void; + isatty: boolean = false; + output: number[]; + + constructor(out: (a: string) => void) { + this.out = out; + this.output = []; + } + + write(buffer: Uint8Array) { + for (let val of buffer) { + if (val === 10 /* charCode('\n') */) { + this.out(textdecoder.decode(new Uint8Array(this.output))); + this.output = []; + } else if (val !== 0) { + // val == 0 would cut text output off in the middle. + this.output.push(val); + } + } + return buffer.length; + } + + fsync() { + if (this.output && this.output.length > 0) { + this.out(textdecoder.decode(new Uint8Array(this.output))); + this.output = []; + } + } +} + +class NodeWriter { + fd: number; + isatty: boolean; + constructor(fd: number) { + this.fd = fd; + this.isatty = tty.isatty(fd); + } + + write(buffer: Uint8Array): number { + return fs.writeSync(this.fd, buffer); + } + + fsync() { + nodeFsync(this.fd); + } } diff --git a/src/js/tsconfig.json b/src/js/tsconfig.json index 30e23e21c..e6d4797ec 100644 --- a/src/js/tsconfig.json +++ b/src/js/tsconfig.json @@ -1,5 +1,6 @@ { "$schema": "http://json.schemastore.org/tsconfig", + "rootDir": "../../", "compilerOptions": { "outDir": "../../dist", "esModuleInterop": true, @@ -8,11 +9,12 @@ "moduleResolution": "node", "composite": true, "strict": true, - "noUnusedLocals": true, + "noUnusedLocals": false, "types": ["node"], "experimentalDecorators": true, - "lib": ["ES2018", "DOM"] + "lib": ["ES2018", "DOM"], + "resolveJsonModule": true }, - "include": ["*.ts"], + "include": ["*.ts", "*.json"], "exclude": ["test/**/*"] } diff --git a/src/tests/test_pyodide.py b/src/tests/test_pyodide.py index 3098e5c76..dbab5805d 100644 --- a/src/tests/test_pyodide.py +++ b/src/tests/test_pyodide.py @@ -1,7 +1,6 @@ import re import shutil import subprocess -from collections.abc import Sequence from pathlib import Path from textwrap import dedent from typing import Any @@ -11,26 +10,11 @@ from pytest_pyodide import run_in_pyodide from pytest_pyodide.fixture import selenium_standalone_noload_common from pytest_pyodide.server import spawn_web_server -from conftest import DIST_PATH, ROOT_PATH +from conftest import DIST_PATH, ROOT_PATH, strip_assertions_stderr from pyodide.code import CodeRunner, eval_code, find_imports, should_quiet # noqa: E402 from pyodide_build.build_env import get_pyodide_root -def _strip_assertions_stderr(messages: Sequence[str]) -> list[str]: - """Strip additional messages on stderr included when ASSERTIONS=1""" - res = [] - for msg in messages: - if msg.strip() in [ - "sigaction: signal type not supported: this is a no-op.", - "Calling stub instead of siginterrupt()", - "warning: no blob constructor, cannot create blobs with mimetypes", - "warning: no BlobBuilder", - ]: - continue - res.append(msg) - return res - - def test_find_imports(): res = find_imports( """ @@ -863,7 +847,7 @@ def test_fatal_error(selenium_standalone): return x err_msg = strip_stack_trace(selenium_standalone.logs) - err_msg = "".join(_strip_assertions_stderr(err_msg.splitlines(keepends=True))) + err_msg = "".join(strip_assertions_stderr(err_msg.splitlines(keepends=True))) assert ( err_msg == dedent( @@ -1134,177 +1118,6 @@ def test_restore_error(selenium): ) -@pytest.mark.skip_refcount_check -@pytest.mark.skip_pyproxy_check -def test_custom_stdin_stdout(selenium_standalone_noload, runtime): - selenium = selenium_standalone_noload - strings = [ - "hello world", - "hello world\n", - "This has a \x00 null byte in the middle...", - "several\nlines\noftext", - "pyodidé", - "碘化物", - "🐍", - ] - selenium.run_js( - """ - function* stdinStrings(){ - for(let x of %s){ - yield x; - } - } - let stdinStringsGen = stdinStrings(); - function stdin(){ - return stdinStringsGen.next().value; - } - self.stdin = stdin; - """ - % strings - ) - selenium.run_js( - """ - self.stdoutStrings = []; - self.stderrStrings = []; - function stdout(s){ - stdoutStrings.push(s); - } - function stderr(s){ - stderrStrings.push(s); - } - let pyodide = await loadPyodide({ - fullStdLib: false, - jsglobals : self, - stdin, - stdout, - stderr, - }); - self.pyodide = pyodide; - globalThis.pyodide = pyodide; - """ - ) - outstrings: list[str] = sum((s.removesuffix("\n").split("\n") for s in strings), []) - print(outstrings) - assert ( - selenium.run_js( - f""" - return pyodide.runPython(` - [input() for x in range({len(outstrings)})] - # ... test more stuff - `).toJs(); - """ - ) - == outstrings - ) - - [stdoutstrings, stderrstrings] = selenium.run_js( - """ - pyodide.runPython(` - import sys - print("something to stdout") - print("something to stderr",file=sys.stderr) - `); - return [self.stdoutStrings, self.stderrStrings]; - """ - ) - assert stdoutstrings[-2:] == [ - "something to stdout", - ] - stderrstrings = _strip_assertions_stderr(stderrstrings) - assert stderrstrings == ["something to stderr"] - IN_NODE = runtime == "node" - selenium.run_js( - f""" - pyodide.runPython(` - import sys - assert not sys.stdin.isatty() - assert not sys.stdout.isatty() - assert not sys.stderr.isatty() - `); - pyodide.setStdin(); - pyodide.setStdout(); - pyodide.setStderr(); - pyodide.runPython(` - import sys - assert sys.stdin.isatty() is {IN_NODE} - assert sys.stdout.isatty() is {IN_NODE} - assert sys.stderr.isatty() is {IN_NODE} - `); - """ - ) - - -def test_custom_stdin_stdout2(selenium): - result = selenium.run_js( - """ - function stdin(){ - return "hello there!\\nThis is a several\\nline\\nstring"; - } - pyodide.setStdin({stdin}); - pyodide.runPython(` - import sys - assert sys.stdin.read(1) == "h" - assert not sys.stdin.isatty() - `); - pyodide.setStdin({stdin, isatty: false}); - pyodide.runPython(` - import sys - assert sys.stdin.read(1) == "e" - `); - pyodide.setStdout(); - pyodide.runPython(` - assert sys.stdin.read(1) == "l" - assert not sys.stdin.isatty() - `); - pyodide.setStdin({stdin, isatty: true}); - pyodide.runPython(` - assert sys.stdin.read(1) == "l" - assert sys.stdin.isatty() - `); - - let stdout_codes = []; - function rawstdout(code) { - stdout_codes.push(code); - } - pyodide.setStdout({raw: rawstdout}); - pyodide.runPython(` - print("hello") - assert sys.stdin.read(1) == "o" - assert not sys.stdout.isatty() - assert sys.stdin.isatty() - `); - pyodide.setStdout({raw: rawstdout, isatty: false}); - pyodide.runPython(` - print("2hello again") - assert sys.stdin.read(1) == " " - assert not sys.stdout.isatty() - assert sys.stdin.isatty() - `); - pyodide.setStdout({raw: rawstdout, isatty: true}); - pyodide.runPython(` - print("3hello") - assert sys.stdin.read(1) == "t" - assert sys.stdout.isatty() - assert sys.stdin.isatty() - `); - pyodide.runPython(` - print("partial line", end="") - `); - let result1 = new TextDecoder().decode(new Uint8Array(stdout_codes)); - pyodide.runPython(` - sys.stdout.flush() - `); - let result2 = new TextDecoder().decode(new Uint8Array(stdout_codes)); - pyodide.setStdin(); - pyodide.setStdout(); - pyodide.setStderr(); - return [result1, result2]; - """ - ) - assert result[0] == "hello\n2hello again\n3hello\n" - assert result[1] == "hello\n2hello again\n3hello\npartial line" - - def test_home_directory(selenium_standalone_noload): selenium = selenium_standalone_noload selenium.run_js( diff --git a/src/tests/test_streams.py b/src/tests/test_streams.py new file mode 100644 index 000000000..6638819d8 --- /dev/null +++ b/src/tests/test_streams.py @@ -0,0 +1,578 @@ +import pytest +from pytest_pyodide import run_in_pyodide + +from conftest import strip_assertions_stderr + + +@pytest.mark.skip_refcount_check +@pytest.mark.skip_pyproxy_check +def test_custom_stdin1(selenium_standalone_noload): + selenium = selenium_standalone_noload + strings = [ + "hello world", + "hello world\n", + "This has a \x00 null byte in the middle...", + "several\nlines\noftext", + "pyodidé", + "碘化物", + "🐍", + ] + outstrings: list[str] = sum( + ((s.removesuffix("\n") + "\n").splitlines(keepends=True) for s in strings), [] + ) + result = selenium.run_js( + f"const strings = {strings};" + f"const numOutlines = {len(outstrings)};" + """ + let stdinStringsGen = strings[Symbol.iterator](); + function stdin(){ + return stdinStringsGen.next().value; + } + const pyodide = await loadPyodide({ + fullStdLib: false, + jsglobals : self, + stdin, + }); + self.pyodide = pyodide; + globalThis.pyodide = pyodide; + return pyodide.runPython(` + import sys + from js import console + [sys.stdin.readline() for _ in range(${numOutlines})] + `).toJs(); + """ + ) + assert result == outstrings + + +@pytest.mark.skip_refcount_check +@pytest.mark.skip_pyproxy_check +def test_custom_stdout1(selenium_standalone_noload, runtime): + selenium = selenium_standalone_noload + [stdoutstrings, stderrstrings] = selenium.run_js( + """ + self.stdoutStrings = []; + self.stderrStrings = []; + function stdout(s){ + stdoutStrings.push(s); + } + function stderr(s){ + stderrStrings.push(s); + } + const pyodide = await loadPyodide({ + fullStdLib: false, + jsglobals : self, + stdout, + stderr, + }); + self.pyodide = pyodide; + globalThis.pyodide = pyodide; + pyodide.runPython(` + import sys + print("something to stdout") + print("something to stderr",file=sys.stderr) + `); + return [stdoutStrings, stderrStrings]; + """ + ) + assert stdoutstrings[-2:] == [ + "something to stdout", + ] + stderrstrings = strip_assertions_stderr(stderrstrings) + assert stderrstrings == ["something to stderr"] + IN_NODE = runtime == "node" + selenium.run_js( + f""" + pyodide.runPython(` + import sys + assert sys.stdin.isatty() is {IN_NODE} + assert not sys.stdout.isatty() + assert not sys.stderr.isatty() + `); + pyodide.setStdin(); + pyodide.setStdout(); + pyodide.setStderr(); + pyodide.runPython(` + import sys + assert sys.stdin.isatty() is {IN_NODE} + assert sys.stdout.isatty() is {IN_NODE} + assert sys.stderr.isatty() is {IN_NODE} + `); + """ + ) + + +def test_custom_stdin_stdout2(selenium): + result = selenium.run_js( + """ + const stdinStringsGen = [ + "hello there!\\nThis is a several\\nline\\nstring" + ][Symbol.iterator](); + function stdin(){ + return stdinStringsGen.next().value; + } + pyodide.setStdin({stdin}); + try { + pyodide.runPython(` + import sys + assert sys.stdin.read(1) == "h" + assert not sys.stdin.isatty() + `); + pyodide.setStdin({stdin, isatty: false}); + pyodide.runPython(` + import sys + assert sys.stdin.read(1) == "e" + `); + pyodide.setStdout(); + pyodide.runPython(` + assert sys.stdin.read(1) == "l" + assert not sys.stdin.isatty() + `); + pyodide.setStdin({stdin, isatty: true}); + pyodide.runPython(` + assert sys.stdin.read(1) == "l" + assert sys.stdin.isatty() + `); + + let stdout_codes = []; + function rawstdout(code) { + stdout_codes.push(code); + } + pyodide.setStdout({raw: rawstdout}); + pyodide.runPython(` + print("hello") + assert sys.stdin.read(1) == "o" + assert not sys.stdout.isatty() + assert sys.stdin.isatty() + `); + pyodide.setStdout({raw: rawstdout, isatty: false}); + pyodide.runPython(` + print("2hello again") + assert sys.stdin.read(1) == " " + assert not sys.stdout.isatty() + assert sys.stdin.isatty() + `); + pyodide.setStdout({raw: rawstdout, isatty: true}); + pyodide.runPython(` + print("3hello") + assert sys.stdin.read(1) == "t" + assert sys.stdout.isatty() + assert sys.stdin.isatty() + `); + pyodide.runPython(` + print("partial line", end="") + `); + let result1 = new TextDecoder().decode(new Uint8Array(stdout_codes)); + pyodide.runPython(` + sys.stdout.flush() + `); + let result2 = new TextDecoder().decode(new Uint8Array(stdout_codes)); + return [result1, result2]; + } finally { + // Flush stdin so other tests don't get messed up. + pyodide.runPython(`sys.stdin.read()`); + pyodide.runPython(`sys.stdin.read()`); + pyodide.runPython(`sys.stdin.read()`); + pyodide.setStdin(); + pyodide.setStdout(); + pyodide.setStderr(); + } + """ + ) + assert result[0] == "hello\n2hello again\n3hello\n" + assert result[1] == "hello\n2hello again\n3hello\npartial line" + + +@run_in_pyodide +def test_stdin_undefined(selenium): + from pyodide.code import run_js + + run_js("pyodide.setStdin({stdin: () => undefined})") + import sys + + try: + print(sys.stdin.read()) + finally: + run_js("pyodide.setStdin()") + + +@run_in_pyodide +def test_custom_stdin_bytes(selenium): + from pyodide.code import run_js + from pyodide_js import setStdin + + run_js( + """ + const sg = [ + 0x61, + 0x62, + 0x00, + null, + 0x63, + 0x64, + null, + 0x65, + 0x66, + 0x67, + ][Symbol.iterator](); + function stdin() { + return sg.next().value; + } + pyodide.setStdin({stdin}); + """ + ) + try: + import sys + + assert sys.stdin.read(5) == "ab\x00" + assert sys.stdin.read(5) == "cd" + assert sys.stdin.read(2) == "ef" + assert sys.stdin.read(2) == "g" + assert sys.stdin.read(2) == "" + finally: + setStdin() + + +@run_in_pyodide +def test_custom_stdin_buffer_autoeof(selenium): + import sys + + from pyodide.code import run_js + from pyodide_js import setStdin + + stdin = run_js( + """ + const sg = [ + new Uint8Array([0x61, 0x62, 0x00]), + new Uint8Array([0x63, 0x64]), + null, + new Uint8Array([0x65, 0x66, 0x67]), + ][Symbol.iterator](); + function stdin() { + return sg.next().value; + } + stdin + """ + ) + try: + setStdin(stdin=stdin) + assert sys.stdin.read(5) == "ab\x00" + assert sys.stdin.read(5) == "cd" + assert sys.stdin.read(2) == "" + assert sys.stdin.read(2) == "ef" + assert sys.stdin.read(2) == "g" + assert sys.stdin.read(2) == "" + finally: + setStdin() + + +@run_in_pyodide +def test_custom_stdin_buffer_noautoeof(selenium): + import sys + + from pyodide.code import run_js + from pyodide_js import setStdin + + stdin = run_js( + """ + const sg = [ + new Uint8Array([0x61, 0x62, 0x00]), + new Uint8Array([0x63, 0x64]), + null, + new Uint8Array([0x65, 0x66, 0x67]), + ][Symbol.iterator](); + function stdin() { + return sg.next().value; + } + stdin; + """ + ) + try: + setStdin(stdin=stdin, autoEOF=False) + assert sys.stdin.read(5) == "ab\x00cd" + assert sys.stdin.read(2) == "ef" + assert sys.stdin.read(2) == "g" + assert sys.stdin.read(2) == "" + finally: + setStdin() + + +@run_in_pyodide +def test_custom_stdin_string_autoeof(selenium): + import sys + + from pyodide.code import run_js + from pyodide_js import setStdin + + stdin = run_js( + r""" + const sg = [ + "ab\x00", + "cd", + null, + "efg", + ][Symbol.iterator](); + function stdin() { + return sg.next().value; + } + stdin + """ + ) + try: + setStdin(stdin=stdin) + assert sys.stdin.read(5) == "ab\x00\nc" + assert sys.stdin.read(5) == "d\n" + assert sys.stdin.read(2) == "ef" + assert sys.stdin.read(2) == "g\n" + assert sys.stdin.read(2) == "" + finally: + sys.stdin.read() + setStdin() + + +@run_in_pyodide +def test_custom_stdin_string_noautoeof(selenium): + import sys + + from pyodide.code import run_js + from pyodide_js import setStdin + + stdin = run_js( + """ + const sg = [ + "ab\x00", + "cd", + null, + "efg", + ][Symbol.iterator](); + function stdin() { + return sg.next().value; + } + stdin; + """ + ) + try: + setStdin(stdin=stdin, autoEOF=False) + assert sys.stdin.read(7) == "ab\x00\ncd\n" + assert sys.stdin.read(2) == "ef" + assert sys.stdin.read(3) == "g\n" + assert sys.stdin.read(2) == "" + finally: + sys.stdin.read() + setStdin() + + +@run_in_pyodide +def test_stdin_error(selenium): + import pytest + + from pyodide_js import setStdin + + try: + setStdin(error=True) + with pytest.raises(OSError, match=r".Errno 29. I/O error"): + input() + finally: + setStdin() + + +def test_custom_stdio_read_buggy(selenium): + @run_in_pyodide + def main(selenium): + import pytest + + from pyodide.code import run_js + from pyodide_js import setStdin + + setStdin(run_js("({read(buffer) {}})")) + try: + with pytest.raises(OSError, match=r"\[Errno 29\] I/O error"): + input() + finally: + setStdin() + + main(selenium) + # Test that we warned about the buggy write implementation + assert selenium.logs.endswith( + "read returned undefined; a correct implementation must return a number" + ) + + +def test_custom_stdio_write_buggy(selenium): + @run_in_pyodide + def main(selenium): + import pytest + + from pyodide.code import run_js + from pyodide_js import setStdout + + setStdout(run_js("({write(buffer) {}})")) + try: + with pytest.raises(OSError, match=r"\[Errno 29\] I/O error"): + print("hi\\nthere!!") + finally: + setStdout() + # flush stdout + print("\n") + + main(selenium) + # Test that we warned about the buggy write implementation + expected = "write returned undefined; a correct implementation must return a number" + assert expected in selenium.logs.splitlines() + + +def test_custom_stdio_write(selenium): + result = selenium.run_js( + r""" + class MyWriter { + constructor() { + this.writtenBuffers = []; + } + write(buffer) { + this.writtenBuffers.push(buffer.slice()); + return buffer.length; + } + } + const o = new MyWriter(); + pyodide.setStdout(o); + pyodide.runPython(String.raw` + print('hi\nthere!!') + print("This\nis a \tmessage!!\n") + `); + pyodide.setStdout(); + return o.writtenBuffers.map((b) => Array.from(b)); + """ + ) + assert [bytes(a).decode() for a in result] == [ + "hi\nthere!!", + "\n", + "This\nis a \tmessage!!\n", + "\n", + ] + + +@run_in_pyodide +def test_custom_stdin_read1(selenium): + from pyodide.code import run_js + from pyodide_js import setStdin + + Reader = run_js( + r""" + function* genFunc(){ + const encoder = new TextEncoder(); + let buffer = yield; + for(const a of [ + "mystring", + "", + "a", + "b", + "c\n", + "def\nghi", + "jkl", + "" + ]) { + encoder.encodeInto(a, buffer); + buffer = yield a.length; + } + } + class Reader { + constructor() { + this.g = genFunc(); + this.g.next(); + } + read(buffer) { + return this.g.next(buffer).value; + } + } + Reader + """ + ) + setStdin(Reader.new()) + try: + assert input() == "mystring" + assert input() == "abc" + assert input() == "def" + assert input() == "ghijkl" + finally: + setStdin() + + setStdin(Reader.new()) + import sys + + try: + assert sys.stdin.readline() == "mystring" + assert sys.stdin.readline() == "abc\n" + assert sys.stdin.readline() == "def\n" + assert sys.stdin.readline() == "ghijkl" + finally: + setStdin() + setStdin(Reader.new()) + try: + assert sys.stdin.read() == "mystring" + assert sys.stdin.read() == "abc\ndef\nghijkl" + finally: + setStdin() + + +@pytest.mark.parametrize("method", ["read", "stdin"]) +@run_in_pyodide +def test_custom_stdin_interrupts(selenium, method): + import pytest + + from pyodide.code import run_js + + run_js( + """ + ib = new Int32Array(1); + pyodide.setInterruptBuffer(ib); + pyodide.setStdin({ + %s () { + ib[0] = 2; + pyodide.checkInterrupt(); + } + }); + """ + % method + ) + try: + with pytest.raises(KeyboardInterrupt): + input() + finally: + run_js( + """ + pyodide.setInterruptBuffer(); + pyodide.setStdin(); + """ + ) + + +@pytest.mark.parametrize("method", ["batched", "raw", "write"]) +@run_in_pyodide +def test_custom_stdout_interrupts(selenium, method): + import pytest + + from pyodide.code import run_js + + run_js( + """ + ib = new Int32Array(1); + pyodide.setInterruptBuffer(ib); + pyodide.setStdout({ + %s () { + ib[0] = 2; + pyodide.checkInterrupt(); + } + }); + """ + % method + ) + try: + with pytest.raises(KeyboardInterrupt): + print() + finally: + run_js( + """ + pyodide.setInterruptBuffer(); + pyodide.setStdout(); + """ + )