From c8f39711206435f1cc74f59edcc2fcae0abe9f1b Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Fri, 2 Apr 2021 14:03:10 -0400 Subject: [PATCH] Add buffer format string function and tests (#1411) --- src/core/hiwire.c | 109 ++++++++++++++++++++++++++++++ src/tests/test_typeconversions.py | 69 +++++++++++++++++++ 2 files changed, 178 insertions(+) diff --git a/src/core/hiwire.c b/src/core/hiwire.c index d9e645b5d..749811ce0 100644 --- a/src/core/hiwire.c +++ b/src/core/hiwire.c @@ -143,6 +143,115 @@ EM_JS_NUM(int, hiwire_init, (), { return (!!obj) && typeof obj.then === 'function'; // clang-format on }; + + /** + * Determine type and endianness of data from format. This is a helper + * function for converting buffers from Python to Javascript, used in + * PyProxyBufferMethods and in `toJs` on a buffer. + * + * To understand this function it will be helpful to look at the tables here: + * https://docs.python.org/3/library/struct.html#format-strings + * + * @arg format {String} A Python format string (caller must convert it to a + * Javascript string). + * @arg errorMessage {String} Extra stuff to append to an error message if + * thrown. Should be a complete sentence. + * @returns A pair, an appropriate TypedArray constructor and a boolean which + * is true if the format suggests a big endian array. + * @private + */ + Module.processBufferFormatString = function(formatStr, errorMessage = "") + { + if (formatStr.length > 2) { + throw new Error( + "Expected format string to have length <= 2, " + + `got '${formatStr}'.` + errorMessage); + } + let formatChar = formatStr.slice(-1); + let alignChar = formatStr.slice(0, -1); + let bigEndian; + switch (alignChar) { + case "!": + case ">": + bigEndian = true; + break; + case "<": + case "@": + case "=": + case "": + bigEndian = false; + break; + default: + throw new Error(`Unrecognized alignment character ${ alignChar }.` + + errorMessage); + } + let arrayType; + switch (formatChar) { + case 'b': + arrayType = Int8Array; + break; + case 's': + case 'p': + case 'c': + case 'B': + case '?': + arrayType = Uint8Array; + break; + case 'h': + arrayType = Int16Array; + break; + case 'H': + arrayType = Uint16Array; + break; + case 'i': + case 'l': + case 'n': + arrayType = Int32Array; + break; + case 'I': + case 'L': + case 'N': + case 'P': + arrayType = Uint32Array; + break; + case 'q': + // clang-format off + if (globalThis.BigInt64Array === undefined) { + // clang-format on + throw new Error("BigInt64Array is not supported on this browser." + + errorMessage); + } + arrayType = BigInt64Array; + break; + case 'Q': + // clang-format off + if (globalThis.BigUint64Array === undefined) { + // clang-format on + throw new Error("BigUint64Array is not supported on this browser." + + errorMessage); + } + arrayType = BigUint64Array; + break; + case 'f': + arrayType = Float32Array; + break; + case 'd': + arrayType = Float64Array; + break; + case "e": + // clang-format off + throw new Error( + "Javascript has no Float16 support. Consider converting the data to " + + "float32 before using it from JavaScript. If you are using a webgl " + + "float16 texture then just use `getBuffer('u8')`."); + // clang-format on + default: + throw new Error(`Unrecognized format character '${formatChar}'.` + + errorMessage); + } + return [ arrayType, bigEndian ]; + }; + return 0; }); diff --git a/src/tests/test_typeconversions.py b/src/tests/test_typeconversions.py index b2f361fa7..1ff0e0e71 100644 --- a/src/tests/test_typeconversions.py +++ b/src/tests/test_typeconversions.py @@ -675,3 +675,72 @@ def test_pyimport_deprecation(selenium): selenium.run_js("pyodide.runPython('x = 1')") assert selenium.run_js("return pyodide.pyimport('x') === 1") assert "pyodide.pyimport is deprecated and will be removed" in selenium.logs + + +def test_buffer_format_string(selenium): + errors = [ + ["aaa", "Expected format string to have length <= 2, got 'aaa'"], + ["II", "Unrecognized alignment character I."], + ["x", "Unrecognized format character 'x'."], + ["x", "Unrecognized format character 'x'."], + ["e", "Javascript has no Float16 support."], + ] + for fmt, msg in errors: + with pytest.raises(selenium.JavascriptException, match=msg): + selenium.run_js( + f""" + pyodide._module.processBufferFormatString({fmt!r}); + """ + ) + + format_tests = [ + ["c", "Uint8"], + ["b", "Int8"], + ["B", "Uint8"], + ["?", "Uint8"], + ["h", "Int16"], + ["H", "Uint16"], + ["i", "Int32"], + ["I", "Uint32"], + ["l", "Int32"], + ["L", "Uint32"], + ["n", "Int32"], + ["N", "Uint32"], + ["q", "BigInt64"], + ["Q", "BigUint64"], + ["f", "Float32"], + ["d", "Float64"], + ["s", "Uint8"], + ["p", "Uint8"], + ["P", "Uint32"], + ] + + def process_fmt_string(fmt): + return selenium.run_js( + f""" + let [array, is_big_endian] = pyodide._module.processBufferFormatString({fmt!r}); + if(!array || typeof array.name !== "string" || !array.name.endsWith("Array")){{ + throw new Error("Unexpected output on input {fmt}: " + array); + }} + let arrayName = array.name.slice(0, -"Array".length); + return [arrayName, is_big_endian]; + """ + ) + + for fmt, expected_array_name in format_tests: + [array_name, is_big_endian] = process_fmt_string(fmt) + assert not is_big_endian + assert array_name == expected_array_name + + endian_tests = [ + ["@h", "Int16", False], + ["=H", "Uint16", False], + ["I", "Uint32", True], + ["!l", "Int32", True], + ] + + for fmt, expected_array_name, expected_is_big_endian in endian_tests: + [array_name, is_big_endian] = process_fmt_string(fmt) + assert is_big_endian == expected_is_big_endian + assert array_name == expected_array_name