Add buffer format string function and tests (#1411)

src/core/hiwire.c

@@ -143,6 +143,115 @@ EM_JS_NUM(int, hiwire_init, (), {
     return (!!obj) && typeof obj.then === 'function';
     // clang-format on
   };
+
+  /**
+   * Determine type and endianness of data from format. This is a helper
+   * function for converting buffers from Python to Javascript, used in
+   * PyProxyBufferMethods and in `toJs` on a buffer.
+   *
+   * To understand this function it will be helpful to look at the tables here:
+   * https://docs.python.org/3/library/struct.html#format-strings
+   *
+   * @arg format {String} A Python format string (caller must convert it to a
+   *      Javascript string).
+   * @arg errorMessage {String} Extra stuff to append to an error message if
+   *      thrown. Should be a complete sentence.
+   * @returns A pair, an appropriate TypedArray constructor and a boolean which
+   *      is true if the format suggests a big endian array.
+   * @private
+   */
+  Module.processBufferFormatString = function(formatStr, errorMessage = "")
+  {
+    if (formatStr.length > 2) {
+      throw new Error(
+        "Expected format string to have length <= 2, " +
+         `got '${formatStr}'.` + errorMessage);
+    }
+    let formatChar = formatStr.slice(-1);
+    let alignChar = formatStr.slice(0, -1);
+    let bigEndian;
+    switch (alignChar) {
+      case "!":
+      case ">":
+        bigEndian = true;
+        break;
+      case "<":
+      case "@":
+      case "=":
+      case "":
+        bigEndian = false;
+        break;
+      default:
+        throw new Error(`Unrecognized alignment character ${ alignChar }.` +
+                        errorMessage);
+    }
+    let arrayType;
+    switch (formatChar) {
+      case 'b':
+        arrayType = Int8Array;
+        break;
+      case 's':
+      case 'p':
+      case 'c':
+      case 'B':
+      case '?':
+        arrayType = Uint8Array;
+        break;
+      case 'h':
+        arrayType = Int16Array;
+        break;
+      case 'H':
+        arrayType = Uint16Array;
+        break;
+      case 'i':
+      case 'l':
+      case 'n':
+        arrayType = Int32Array;
+        break;
+      case 'I':
+      case 'L':
+      case 'N':
+      case 'P':
+        arrayType = Uint32Array;
+        break;
+      case 'q':
+        // clang-format off
+        if (globalThis.BigInt64Array === undefined) {
+          // clang-format on
+          throw new Error("BigInt64Array is not supported on this browser." +
+                          errorMessage);
+        }
+        arrayType = BigInt64Array;
+        break;
+      case 'Q':
+        // clang-format off
+        if (globalThis.BigUint64Array === undefined) {
+          // clang-format on
+          throw new Error("BigUint64Array is not supported on this browser." +
+                          errorMessage);
+        }
+        arrayType = BigUint64Array;
+        break;
+      case 'f':
+        arrayType = Float32Array;
+        break;
+      case 'd':
+        arrayType = Float64Array;
+        break;
+      case "e":
+        // clang-format off
+        throw new Error(
+          "Javascript has no Float16 support. Consider converting the data to " +
+          "float32 before using it from JavaScript. If you are using a webgl " +
+          "float16 texture then just use `getBuffer('u8')`.");
+        // clang-format on
+      default:
+        throw new Error(`Unrecognized format character '${formatChar}'.` +
+                        errorMessage);
+    }
+    return [ arrayType, bigEndian ];
+  };
+
   return 0;
 });
 

src/tests/test_typeconversions.py

@@ -675,3 +675,72 @@ def test_pyimport_deprecation(selenium):
     selenium.run_js("pyodide.runPython('x = 1')")
     assert selenium.run_js("return pyodide.pyimport('x') === 1")
     assert "pyodide.pyimport is deprecated and will be removed" in selenium.logs
+
+
+def test_buffer_format_string(selenium):
+    errors = [
+        ["aaa", "Expected format string to have length <= 2, got 'aaa'"],
+        ["II", "Unrecognized alignment character I."],
+        ["x", "Unrecognized format character 'x'."],
+        ["x", "Unrecognized format character 'x'."],
+        ["e", "Javascript has no Float16 support."],
+    ]
+    for fmt, msg in errors:
+        with pytest.raises(selenium.JavascriptException, match=msg):
+            selenium.run_js(
+                f"""
+                pyodide._module.processBufferFormatString({fmt!r});
+                """
+            )
+
+    format_tests = [
+        ["c", "Uint8"],
+        ["b", "Int8"],
+        ["B", "Uint8"],
+        ["?", "Uint8"],
+        ["h", "Int16"],
+        ["H", "Uint16"],
+        ["i", "Int32"],
+        ["I", "Uint32"],
+        ["l", "Int32"],
+        ["L", "Uint32"],
+        ["n", "Int32"],
+        ["N", "Uint32"],
+        ["q", "BigInt64"],
+        ["Q", "BigUint64"],
+        ["f", "Float32"],
+        ["d", "Float64"],
+        ["s", "Uint8"],
+        ["p", "Uint8"],
+        ["P", "Uint32"],
+    ]
+
+    def process_fmt_string(fmt):
+        return selenium.run_js(
+            f"""
+            let [array, is_big_endian] = pyodide._module.processBufferFormatString({fmt!r});
+            if(!array || typeof array.name !== "string" || !array.name.endsWith("Array")){{
+                throw new Error("Unexpected output on input {fmt}: " + array);
+            }}
+            let arrayName = array.name.slice(0, -"Array".length);
+            return [arrayName, is_big_endian];
+            """
+        )
+
+    for fmt, expected_array_name in format_tests:
+        [array_name, is_big_endian] = process_fmt_string(fmt)
+        assert not is_big_endian
+        assert array_name == expected_array_name
+
+    endian_tests = [
+        ["@h", "Int16", False],
+        ["=H", "Uint16", False],
+        ["<i", "Int32", False],
+        [">I", "Uint32", True],
+        ["!l", "Int32", True],
+    ]
+
+    for fmt, expected_array_name, expected_is_big_endian in endian_tests:
+        [array_name, is_big_endian] = process_fmt_string(fmt)
+        assert is_big_endian == expected_is_big_endian
+        assert array_name == expected_array_name