diff --git a/docs/index.rst b/docs/index.rst index dfb991b83..e28b53b16 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -47,6 +47,7 @@ Using Pyodide usage/type-conversions.md usage/wasm-constraints.md usage/keyboard-interrupts.md + usage/streams.md usage/api-reference.md usage/faq.md diff --git a/docs/project/changelog.md b/docs/project/changelog.md index 8c2a3ea7b..3a4581502 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -119,6 +119,13 @@ substitutions: a new option `checkIntegrity`. If set to False, integrity check for Python Packages will be disabled. +- {{ Enhancement }} Added APIs {any}`setStdin `, + {any}`setStdout `, {any}`setStderr ` for + changing the stream handlers after loading Pyodide. Also added more careful + control over whether `isatty` returns true or false on stdin, stdout, and + stderr. + {pr}`3268` + - {{ Fix }} Fix undefined symbol error when loading shared library {pr}`3193` diff --git a/docs/usage/streams.md b/docs/usage/streams.md new file mode 100644 index 000000000..e170066dc --- /dev/null +++ b/docs/usage/streams.md @@ -0,0 +1,40 @@ +(streams)= + +# Redirecting standard streams + +Pyodide has three functions `pyodide.setStdin`, `pyodide.setStdout`, and +`pyodide.setStderr` that change the behavior of reading from `stdin` and writing +to `stdout` and `stderr` respectively. + +`setStdin({stdin?, isatty?, error?})` takes a function which should take zero +arguments and return either a string or an ArrayBufferView of information read +from stdin. The `isatty` argument signals whether `isatty(stdin)` should be true +or false. If you pass `error: true` then reading from stdin will return an +error. If `setStdin` is called with no arguments, the default value is restored. +In Node the default behavior is to read from `process.stdin` and in the browser +it is to throw an error. + +`setStdout({batched?, raw?, isattty?})` sets the standard out handler and +similarly `setStderr` (same arguments) sets the stdandard error handler. If a +`raw` handler is provided then the handler is called with a `number` for each +byte of the output to stdout. The handler is expected to deal with this in +whatever way it prefers. `isattty` again controls whether `isatty(stdout)` +returns `true` or `false`. + +On the other hand, a `batched` handler is only called with complete lines of +text (or when the output is flushed). A `batched` handler cannot have `isatty` +set to `true` because it is impossible to use such a handler to make something +behave like a tty. + +Passing neither `raw` nor `batched` sets the default behavior. In Node the +default behavior is to write directly to `process.stdout` and `process.stderr` +(in this case `isatty` depends on whether `process.stdout` and `process.stderr` +are ttys). In browser, the default behavior is achieved with +`pyodide.setStdout({batched: console.log})` and `pyodide.setStderr({batched: console.warn})`. + +The arguments `stdin`, `stdout`, and `stderr` to `loadPyodide` provide a +diminished amount of control compared to `setStdin`, `setStdout`, and +`setStderr`. They all set `isatty` to `false` and use batched processing for +`setStdout` and `setStderr`. In most cases, nothing is written or read to any of +these streams while Pyodide is starting, so if you need the added flexibility +you can wait until Pyodide is loaded and then use the `pyodide.setStdxxx` apis. diff --git a/src/js/api.ts b/src/js/api.ts index 54a1ed158..80798f1a3 100644 --- a/src/js/api.ts +++ b/src/js/api.ts @@ -10,6 +10,7 @@ import { loadBinaryFile } from "./compat"; import version from "./version"; export { loadPackage, loadedPackages, isPyProxy }; import "./error_handling.gen.js"; +import { setStdin, setStdout, setStderr } from "./streams"; API.loadBinaryFile = loadBinaryFile; @@ -518,6 +519,9 @@ export type PyodideInterface = { registerComlink: typeof registerComlink; PythonError: typeof PythonError; PyBuffer: typeof PyBuffer; + setStdin: typeof setStdin; + setStdout: typeof setStdout; + setStderr: typeof setStderr; }; /** @@ -584,6 +588,9 @@ API.makePublicAPI = function (): PyodideInterface { PyBuffer, _module: Module, _api: API, + setStdin, + setStdout, + setStderr, }; API.public_api = namespace; diff --git a/src/js/interface.ts b/src/js/interface.ts new file mode 100644 index 000000000..1f3349e15 --- /dev/null +++ b/src/js/interface.ts @@ -0,0 +1,38 @@ +declare var API: any; +declare var Module: any; +/** + * @private + */ +API.makePublicAPI = function (): PyodideInterface { + FS = Module.FS; + PATH = Module.PATH; + ERRNO_CODES = Module.ERRNO_CODES; + let namespace = { + globals, + FS, + PATH, + ERRNO_CODES, + pyodide_py, + version, + loadPackage, + isPyProxy, + runPython, + runPythonAsync, + registerJsModule, + unregisterJsModule, + setInterruptBuffer, + checkInterrupt, + toPy, + pyimport, + unpackArchive, + mountNativeFS, + registerComlink, + PythonError, + PyBuffer, + _module: Module, + _api: API, + }; + + API.public_api = namespace; + return namespace; +}; diff --git a/src/js/module.ts b/src/js/module.ts index 5bc272365..21b5158fa 100644 --- a/src/js/module.ts +++ b/src/js/module.ts @@ -1,4 +1,35 @@ /** @private */ + +type FSNode = any; +type FSStream = any; + +export interface FS { + unlink: (path: string) => void; + mkdirTree: (path: string, mode?: number) => void; + chdir: (path: string) => void; + symlink: (target: string, src: string) => FSNode; + createDevice: ( + parent: string, + name: string, + input?: (() => number | null) | null, + output?: ((code: number) => void) | null, + ) => FSNode; + closeStream: (fd: number) => void; + open: (path: string, flags: string | number, mode?: number) => FSStream; + makedev: (major: number, minor: number) => number; + mkdev: (path: string, dev: number) => FSNode; + filesystems: any; + stat: (path: string, dontFollow?: boolean) => any; + readdir: (node: FSNode) => string[]; + isDir: (mode: number) => boolean; + lookupPath: (path: string) => FSNode; + isFile: (mode: number) => boolean; + writeFile: (path: string, contents: any, o?: { canOwn?: boolean }) => void; + chmod: (path: string, mode: number) => void; + utime: (path: string, atime: number, mtime: number) => void; + rmdir: (path: string) => void; +} + export interface Module { noImageDecoding: boolean; noAudioDecoding: boolean; @@ -8,8 +39,9 @@ export interface Module { print: (a: string) => void; printErr: (a: string) => void; ENV: { [key: string]: string }; - FS: any; PATH: any; + TTY: any; + FS: FS; } /** @@ -30,81 +62,6 @@ export function createModule(): any { return Module; } -/** - * - * @param stdin - * @param stdout - * @param stderr - * @private - */ -export function setStandardStreams( - Module: Module, - stdin?: () => string, - stdout?: (a: string) => void, - stderr?: (a: string) => void, -) { - // For stdout and stderr, emscripten provides convenient wrappers that save us the trouble of converting the bytes into a string - if (stdout) { - Module.print = stdout; - } - - if (stderr) { - Module.printErr = stderr; - } - - // For stdin, we have to deal with the low level API ourselves - if (stdin) { - Module.preRun.push(function () { - Module.FS.init(createStdinWrapper(stdin), null, null); - }); - } -} - -function createStdinWrapper(stdin: () => string) { - // When called, it asks the user for one whole line of input (stdin) - // Then, it passes the individual bytes of the input to emscripten, one after another. - // And finally, it terminates it with null. - const encoder = new TextEncoder(); - let input = new Uint8Array(0); - let inputIndex = -1; // -1 means that we just returned null - function stdinWrapper() { - try { - if (inputIndex === -1) { - let text = stdin(); - if (text === undefined || text === null) { - return null; - } - if (typeof text !== "string") { - throw new TypeError( - `Expected stdin to return string, null, or undefined, got type ${typeof text}.`, - ); - } - if (!text.endsWith("\n")) { - text += "\n"; - } - input = encoder.encode(text); - inputIndex = 0; - } - - if (inputIndex < input.length) { - let character = input[inputIndex]; - inputIndex++; - return character; - } else { - inputIndex = -1; - return null; - } - } catch (e) { - // emscripten will catch this and set an IOError which is unhelpful for - // debugging. - console.error("Error thrown in stdin:"); - console.error(e); - throw e; - } - } - return stdinWrapper; -} - /** * Make the home directory inside the virtual file system, * then change the working directory to it. diff --git a/src/js/pyodide.ts b/src/js/pyodide.ts index a218dc7e1..9b17da9f3 100644 --- a/src/js/pyodide.ts +++ b/src/js/pyodide.ts @@ -10,7 +10,7 @@ import { resolvePath, } from "./compat"; -import { createModule, setStandardStreams, setHomeDirectory } from "./module"; +import { createModule, setHomeDirectory } from "./module"; import { initializeNativeFS } from "./nativefs"; import version from "./version"; @@ -247,8 +247,8 @@ export async function loadPyodide( * The home directory which Pyodide will use inside virtual file system. Default: "/home/pyodide" */ homedir?: string; - - /** Load the full Python standard library. + /** + * Load the full Python standard library. * Setting this to false excludes unvendored modules from the standard library. * Default: false */ @@ -295,6 +295,8 @@ export async function loadPyodide( ); const Module = createModule(); + Module.print = config.stdout; + Module.printErr = config.stderr; Module.preRun.push(() => { for (const mount of config._node_mounts) { Module.FS.mkdirTree(mount); @@ -306,7 +308,6 @@ export async function loadPyodide( const API: any = { config }; Module.API = API; - setStandardStreams(Module, config.stdin, config.stdout, config.stderr); setHomeDirectory(Module, config.homedir); const moduleLoaded = new Promise((r) => (Module.postRun = r)); @@ -343,14 +344,13 @@ If you updated the Pyodide version, make sure you also updated the 'indexURL' pa `, ); } - - initializeNativeFS(Module); - // Disable further loading of Emscripten file_packager stuff. Module.locateFile = (path: string) => { throw new Error("Didn't expect to load any more file_packager files!"); }; + initializeNativeFS(Module); + const pyodide_py_tar = await pyodide_py_tar_promise; unpackPyodidePy(Module, pyodide_py_tar); API.rawRun("import _pyodide_core"); @@ -375,5 +375,6 @@ If you updated the Pyodide version, make sure you also updated the 'indexURL' pa if (config.fullStdLib) { await pyodide.loadPackage(API._pyodide._importhook.UNVENDORED_STDLIBS); } + API.initializeStreams(config.stdin, config.stdout, config.stderr); return pyodide; } diff --git a/src/js/streams.ts b/src/js/streams.ts new file mode 100644 index 000000000..78a0c7795 --- /dev/null +++ b/src/js/streams.ts @@ -0,0 +1,404 @@ +import { IN_NODE } from "./compat.js"; +import type { Module } from "./module"; + +declare var API: any; +declare var Module: Module; + +declare var FS: typeof Module.FS; +declare var TTY: any; + +// The type of the function we need to produce to read from stdin +// (Either directly from a device when isatty is false or as part of a tty when +// isatty is true) +type GetCharType = () => null | number; + +// The type of the function we expect the user to give us. make_get_char takes +// one of these and turns it into a GetCharType function for us. +type InFuncType = () => + | null + | undefined + | string + | ArrayBuffer + | ArrayBufferView; + +// To define the output behavior of a tty we need to define put_char and fsync. +// fsync flushes the stream. +// +// If isatty is false, we ignore fsync and use put_char.bind to fill in a dummy +// value for the tty argument. We don't ever use the tty argument. +type PutCharType = { + put_char: (tty: void, val: number) => void; + fsync: (tty: void) => void; +}; + +// A tty needs both a GetChar function and a PutChar pair. +type TtyOps = { + get_char: GetCharType; +} & PutCharType; + +/** + * We call refreshStreams at the end of every update method, but refreshStreams + * won't work until initializeStreams is called. So when INITIALIZED is false, + * refreshStreams is a no-op. + * @private + */ +let INITIALIZED = false; + +// These can't be used until they are initialized by initializeStreams. +const ttyout_ops = {} as TtyOps; +const ttyerr_ops = {} as TtyOps; +const isattys = {} as { + stdin: boolean; + stdout: boolean; + stderr: boolean; +}; + +function refreshStreams() { + if (!INITIALIZED) { + return; + } + FS.unlink("/dev/stdin"); + FS.unlink("/dev/stdout"); + FS.unlink("/dev/stderr"); + if (isattys.stdin) { + FS.symlink("/dev/tty", "/dev/stdin"); + } else { + FS.createDevice("/dev", "stdin", ttyout_ops.get_char); + } + if (isattys.stdout) { + FS.symlink("/dev/tty", "/dev/stdout"); + } else { + FS.createDevice( + "/dev", + "stdout", + null, + ttyout_ops.put_char.bind(undefined, undefined), + ); + } + if (isattys.stderr) { + FS.symlink("/dev/tty", "/dev/stderr"); + } else { + FS.createDevice( + "/dev", + "stderr", + null, + ttyerr_ops.put_char.bind(undefined, undefined), + ); + } + + // Refresh std streams so they use our new versions + FS.closeStream(0 /* stdin */); + FS.closeStream(1 /* stdout */); + FS.closeStream(2 /* stderr */); + FS.open("/dev/stdin", 0 /* write only */); + FS.open("/dev/stdout", 1 /* read only */); + FS.open("/dev/stderr", 1 /* read only */); +} + +/** + * This is called at the end of loadPyodide to set up the streams. If + * loadPyodide has been given stdin, stdout, stderr arguments they are provided + * here. Otherwise, we set the default behaviors. This also fills in the global + * state in this file. + * @param stdin + * @param stdout + * @param stderr + * @private + */ +API.initializeStreams = function ( + stdin?: InFuncType, + stdout?: (a: string) => void, + stderr?: (a: string) => void, +) { + setStdin({ stdin }); + if (stdout) { + setStdout({ batched: stdout }); + } else { + setDefaultStdout(); + } + + if (stderr) { + setStderr({ batched: stderr }); + } else { + setDefaultStderr(); + } + // 5.0 and 6.0 are the device numbers that Emscripten uses (see library_fs.js). + // These haven't changed in ~10 years. If we used different ones nothing would + // break. + const ttyout_dev = FS.makedev(5, 0); + const ttyerr_dev = FS.makedev(6, 0); + TTY.register(ttyout_dev, ttyout_ops); + TTY.register(ttyerr_dev, ttyerr_ops); + INITIALIZED = true; + refreshStreams(); +}; + +/** + * Sets the default stdin. If in node, stdin will read from `process.stdin` + * and isatty(stdin) will be set to tty.isatty(process.stdin.fd). + * If in a browser, this calls setStdinError. + */ +function setDefaultStdin() { + if (IN_NODE) { + const BUFSIZE = 256; + const buf = Buffer.alloc(BUFSIZE); + const fs = require("fs"); + const tty = require("tty"); + const stdin = function () { + let bytesRead; + try { + bytesRead = fs.readSync(process.stdin.fd, buf, 0, BUFSIZE, -1); + } catch (e) { + // Platform differences: on Windows, reading EOF throws an exception, + // but on other OSes, reading EOF returns 0. Uniformize behavior by + // catching the EOF exception and returning 0. + if ((e as Error).toString().includes("EOF")) { + bytesRead = 0; + } else { + throw e; + } + } + if (bytesRead === 0) { + return null; + } + return buf.subarray(0, bytesRead); + }; + const isatty: boolean = tty.isatty(process.stdin.fd); + setStdin({ stdin, isatty }); + } else { + setStdinError(); + } +} + +/** + * Sets isatty(stdin) to false and makes reading from stdin always set an EIO + * error. + */ +function setStdinError() { + isattys.stdin = false; + const get_char = () => { + throw 0; + }; + ttyout_ops.get_char = get_char; + ttyerr_ops.get_char = get_char; + refreshStreams(); +} + +/** + * Sets a stdin function. This function will be called whenever stdin is read. + * Also sets isatty(stdin) to the value of the isatty argument (default false). + * + * The stdin function is called with zero arguments. It should return one of: + * - `null` or `undefined`: these are interpreted as EOF + * - a string + * - an ArrayBuffer or an ArrayBufferView with BYTES_PER_ELEMENT === 1 + * + * If a string is returned, a new line is appended if one is not present and the + * resulting string is turned into a Uint8Array using TextEncoder. + * + * Returning a buffer is more efficient and allows returning partial lines of + * text. + * + */ +export function setStdin({ + stdin, + isatty, + error, +}: { stdin?: InFuncType; error?: boolean; isatty?: boolean } = {}) { + if (error) { + setStdinError(); + return; + } + if (stdin) { + isattys.stdin = !!isatty; + const get_char = make_get_char(stdin); + ttyout_ops.get_char = get_char; + ttyerr_ops.get_char = get_char; + refreshStreams(); + return; + } + setDefaultStdin(); +} + +/** + * If in node, sets stdout to write directly to process.stdout and sets isatty(stdout) + * to tty.isatty(process.stdout.fd). + * If in a browser, sets stdout to write to console.log and sets isatty(stdout) to false. + */ +export function setDefaultStdout() { + if (IN_NODE) { + const tty = require("tty"); + const raw = (x: number) => process.stdout.write(Buffer.from([x])); + const isatty: boolean = tty.isatty(process.stdout.fd); + setStdout({ raw, isatty }); + } else { + setStdout({ batched: (x) => console.log(x) }); + } +} + +/** + * Sets writes to stdout to call `stdout(line)` whenever a complete line is + * written or stdout is flushed. In the former case, the received line will end + * with a newline, in the latter case it will not. + * + * isatty(stdout) is set to false (this API buffers stdout so it is impossible + * to make a tty with it). + */ +export function setStdout({ + batched, + raw, + isatty, +}: { + batched?: (a: string) => void; + raw?: (a: number) => void; + isatty?: boolean; +} = {}) { + if (raw) { + isattys.stdout = !!isatty; + Object.assign(ttyout_ops, make_unbatched_put_char(raw)); + refreshStreams(); + return; + } + if (batched) { + isattys.stdout = false; + Object.assign(ttyout_ops, make_batched_put_char(batched)); + refreshStreams(); + return; + } + setDefaultStdout(); +} + +/** + * If in node, sets stderr to write directly to process.stderr and sets isatty(stderr) + * to tty.isatty(process.stderr.fd). + * If in a browser, sets stderr to write to console.warn and sets isatty(stderr) to false. + */ +function setDefaultStderr() { + if (IN_NODE) { + const tty = require("tty"); + const raw = (x: number) => process.stderr.write(Buffer.from([x])); + const isatty: boolean = tty.isatty(process.stderr.fd); + setStderr({ raw, isatty }); + } else { + setStderr({ batched: (x) => console.warn(x) }); + } +} + +/** + * Sets writes to stderr to call `stderr(line)` whenever a complete line is + * written or stderr is flushed. In the former case, the received line will end + * with a newline, in the latter case it will not. + * + * isatty(stderr) is set to false (this API buffers stderr so it is impossible + * to make a tty with it). + */ +export function setStderr({ + batched, + raw, + isatty, +}: { + batched?: (a: string) => void; + raw?: (a: number) => void; + isatty?: boolean; +} = {}) { + if (raw) { + isattys.stderr = !!isatty; + Object.assign(ttyerr_ops, make_unbatched_put_char(raw)); + refreshStreams(); + return; + } + if (batched) { + isattys.stderr = false; + Object.assign(ttyerr_ops, make_batched_put_char(batched)); + refreshStreams(); + return; + } + setDefaultStderr(); +} + +const textencoder = new TextEncoder(); +const textdecoder = new TextDecoder(); + +function make_get_char(infunc: InFuncType): GetCharType { + let index = 0; + let buf: Uint8Array = new Uint8Array(0); + // get_char has 3 particular return values: + // a.) the next character represented as an integer + // b.) undefined to signal that no data is currently available + // c.) null to signal an EOF + return function get_char() { + try { + if (index >= buf.length) { + let input = infunc(); + if (input === undefined || input === null) { + return null; + } + if (typeof input === "string") { + if (!input.endsWith("\n")) { + input += "\n"; + } + buf = textencoder.encode(input); + } else if (ArrayBuffer.isView(input)) { + if ((input as any).BYTES_PER_ELEMENT !== 1) { + throw new Error("Expected BYTES_PER_ELEMENT to be 1"); + } + buf = input as Uint8Array; + } else if ( + Object.prototype.toString.call(input) === "[object ArrayBuffer]" + ) { + buf = new Uint8Array(input); + } else { + throw new Error( + "Expected result to be undefined, null, string, array buffer, or array buffer view", + ); + } + if (buf.length === 0) { + return null; + } + index = 0; + } + return buf[index++]; + } catch (e) { + // emscripten will catch this and set an IOError which is unhelpful for + // debugging. + console.error("Error thrown in stdin:"); + console.error(e); + throw e; + } + }; +} + +function make_unbatched_put_char(out: (a: number) => void): PutCharType { + return { + put_char(tty: any, val: number) { + out(val); + }, + fsync() {}, + }; +} + +function make_batched_put_char(out: (a: string) => void): PutCharType { + let output: number[] = []; + return { + // get_char has 3 particular return values: + // a.) the next character represented as an integer + // b.) undefined to signal that no data is currently available + // c.) null to signal an EOF, + put_char(tty: any, val: number) { + if (val === null || val === 10 /* charCode('\n') */) { + out(textdecoder.decode(new Uint8Array(output))); + output = []; + } else { + if (val !== 0) { + output.push(val); // val == 0 would cut text output off in the middle. + } + } + }, + fsync(tty: any) { + if (output && output.length > 0) { + out(textdecoder.decode(new Uint8Array(output))); + output = []; + } + }, + }; +} diff --git a/src/tests/test_pyodide.py b/src/tests/test_pyodide.py index 76d0d59b9..2ade2dcfd 100644 --- a/src/tests/test_pyodide.py +++ b/src/tests/test_pyodide.py @@ -1061,7 +1061,7 @@ def test_restore_error(selenium): @pytest.mark.skip_refcount_check @pytest.mark.skip_pyproxy_check -def test_custom_stdin_stdout(selenium_standalone_noload): +def test_custom_stdin_stdout(selenium_standalone_noload, runtime): selenium = selenium_standalone_noload strings = [ "hello world", @@ -1113,11 +1113,11 @@ def test_custom_stdin_stdout(selenium_standalone_noload): assert ( selenium.run_js( f""" - return pyodide.runPython(` - [input() for x in range({len(outstrings)})] - # ... test more stuff - `).toJs(); - """ + return pyodide.runPython(` + [input() for x in range({len(outstrings)})] + # ... test more stuff + `).toJs(); + """ ) == outstrings ) @@ -1137,6 +1137,97 @@ def test_custom_stdin_stdout(selenium_standalone_noload): ] stderrstrings = _strip_assertions_stderr(stderrstrings) assert stderrstrings == ["something to stderr"] + IN_NODE = runtime == "node" + selenium.run_js( + f""" + pyodide.runPython(` + import sys + assert not sys.stdin.isatty() + assert not sys.stdout.isatty() + assert not sys.stderr.isatty() + `); + pyodide.setStdin(); + pyodide.setStdout(); + pyodide.setStderr(); + pyodide.runPython(` + import sys + assert sys.stdin.isatty() is {IN_NODE} + assert sys.stdout.isatty() is {IN_NODE} + assert sys.stderr.isatty() is {IN_NODE} + `); + """ + ) + + +def test_custom_stdin_stdout2(selenium): + result = selenium.run_js( + """ + function stdin(){ + return "hello there!\\nThis is a several\\nline\\nstring"; + } + pyodide.setStdin({stdin}); + pyodide.runPython(` + import sys + assert sys.stdin.read(1) == "h" + assert not sys.stdin.isatty() + `); + pyodide.setStdin({stdin, isatty: false}); + pyodide.runPython(` + import sys + assert sys.stdin.read(1) == "e" + `); + pyodide.setStdout(); + pyodide.runPython(` + assert sys.stdin.read(1) == "l" + assert not sys.stdin.isatty() + `); + pyodide.setStdin({stdin, isatty: true}); + pyodide.runPython(` + assert sys.stdin.read(1) == "l" + assert sys.stdin.isatty() + `); + + let stdout_codes = []; + function rawstdout(code) { + stdout_codes.push(code); + } + pyodide.setStdout({raw: rawstdout}); + pyodide.runPython(` + print("hello") + assert sys.stdin.read(1) == "o" + assert not sys.stdout.isatty() + assert sys.stdin.isatty() + `); + pyodide.setStdout({raw: rawstdout, isatty: false}); + pyodide.runPython(` + print("2hello again") + assert sys.stdin.read(1) == " " + assert not sys.stdout.isatty() + assert sys.stdin.isatty() + `); + pyodide.setStdout({raw: rawstdout, isatty: true}); + pyodide.runPython(` + print("3hello") + assert sys.stdin.read(1) == "t" + assert sys.stdout.isatty() + assert sys.stdin.isatty() + `); + pyodide.runPython(` + print("partial line", end="") + `); + let result1 = new TextDecoder().decode(new Uint8Array(stdout_codes)); + pyodide.runPython(` + sys.stdout.flush() + `); + let result2 = new TextDecoder().decode(new Uint8Array(stdout_codes)); + pyodide.setStdin(); + pyodide.setStdout(); + pyodide.setStderr(); + return [result1, result2]; + """ + ) + assert result[0] == "hello\n2hello again\n3hello\n" + assert result[1] == "hello\n2hello again\n3hello\npartial line" def test_home_directory(selenium_standalone_noload): diff --git a/tools/python b/tools/python index 15132b149..8b14e1f8e 100755 --- a/tools/python +++ b/tools/python @@ -49,74 +49,6 @@ exec node "$ARGS" "$0" "$@" const { loadPyodide } = require("../dist/pyodide"); const fs = require("fs"); -/** - * The default stderr/stdout do not handle newline or flush correctly, and stdin - * is also strange. Make a tty that connects Emscripten stdstreams to node - * stdstreams. We will make one tty for stdout and one for stderr, the - * `outstream` argument controls which one we make. - * - * Note that setting Module.stdin / Module.stdout / Module.stderr does not work - * because these cause `isatty(stdstream)` to return false. We want `isatty` to - * be true. (IMO this is an Emscripten issue, maybe someday we can fix it.) - */ -function make_tty_ops(outstream) { - const BUFSIZE = 256; - const buf = Buffer.alloc(BUFSIZE); - let index = 0; - let numbytes = 0; - return { - // get_char has 3 particular return values: - // a.) the next character represented as an integer - // b.) undefined to signal that no data is currently available - // c.) null to signal an EOF - get_char(tty) { - if (index >= numbytes) { - // read synchronously at most BUFSIZE bytes from file descriptor 0 (stdin) - const bytesRead = fs.readSync(0, buf, 0, BUFSIZE, -1); - if (bytesRead === 0) { - return null; - } - index = 0; - numbytes = bytesRead; - } - return buf[index++]; - }, - put_char(tty, val) { - outstream.write(Buffer.from([val])); - }, - flush(tty) {}, - fsync() {}, - }; -} - -/** - * Fix standard streams by replacing them with ttys that work better via make_tty_ops. - */ -function setupStreams(FS, TTY) { - // Create and register devices - let mytty = FS.makedev(FS.createDevice.major++, 0); - let myttyerr = FS.makedev(FS.createDevice.major++, 0); - TTY.register(mytty, make_tty_ops(process.stdout)); - TTY.register(myttyerr, make_tty_ops(process.stderr)); - // Attach devices to files - FS.mkdev("/dev/mytty", mytty); - FS.mkdev("/dev/myttyerr", myttyerr); - // Replace /dev/stdxx with our custom devices - FS.unlink("/dev/stdin"); - FS.unlink("/dev/stdout"); - FS.unlink("/dev/stderr"); - FS.symlink("/dev/mytty", "/dev/stdin"); - FS.symlink("/dev/mytty", "/dev/stdout"); - FS.symlink("/dev/myttyerr", "/dev/stderr"); - // Refresh std streams so they use our new versions - FS.closeStream(0 /* stdin */); - FS.closeStream(1 /* stdout */); - FS.closeStream(2 /* stderr */); - FS.open("/dev/stdin", 0 /* write only */); - FS.open("/dev/stdout", 1 /* read only */); - FS.open("/dev/stderr", 1 /* read only */); -} - /** * Determine which native top level directories to mount into the Emscripten * file system. @@ -142,7 +74,7 @@ async function main() { fullStdLib: false, _node_mounts, homedir: process.cwd(), - // Strip out standard messages written to stdout and stderr while loading + // Strip out messages written to stderr while loading // After Pyodide is loaded we will replace stdstreams with setupStreams. stderr(e) { if ( @@ -154,7 +86,7 @@ async function main() { return; } console.warn(e); - }, + } }); } catch (e) { if (e.constructor.name !== "ExitStatus") { @@ -164,10 +96,10 @@ async function main() { // arguments that is invalid in some way, we will exit here. process.exit(e.status); } - const FS = py.FS; - setupStreams(FS, py._module.TTY); + py.setStdout(); + py.setStderr(); let sideGlobals = py.runPython("{}"); - globalThis.handleExit = function handleExit(code) { + function handleExit(code) { if (code === undefined) { code = 0; } @@ -179,6 +111,7 @@ async function main() { // scheduled will segfault. process.exit(code); }; + sideGlobals.set("handleExit", handleExit); py.runPython( ` @@ -210,7 +143,6 @@ async function main() { loop = asyncio.get_event_loop() # Make sure we don't run _no_in_progress_handler before we finish _run_main. loop._in_progress += 1 - from js import handleExit loop._no_in_progress_handler = handleExit loop._system_exit_handler = handleExit loop._keyboard_interrupt_handler = lambda: handleExit(130)