Enable libhdf5 and h5py (#2812)

docs/development/meta-yaml.md

@@ -154,6 +154,14 @@ Python package, the script section will be run before the build system runs
 `setup.py`. This script is run by `bash` in the directory where the tarball was
 extracted.
 
+There are special environment variables defined:
+
+- `$PKGDIR`: The directory in which the `meta.yaml` file resides.
+- `$PKG_VESRION`: The version of the package
+- `$PKG_BUILD_DIR`: The directory where the tarball was extracted.
+
+(These keys are not in the Conda spec).
+
 ### `build/cross-script`
 
 This script will run _after_ `build/script`. The difference is that it runs with
@@ -165,14 +173,13 @@ is the source directory.
 
 ### `build/post`
 
-Shell commands to run after building the library. These are run with `bash`, and
+Shell commands to run after building the library. This script is run by `bash`
-there are two special environment variables defined:
+in the directory where `meta.yaml` file resides. The `${PKG_BUILD_DIR}/dist`
-
+will contain the built wheel unpacked with `python -m wheel unpack`
-- `$SITEPACKAGES`: The `site-packages` directory into which the package has been
+so it's possible to manually add, delete, change, move files etc.
-  installed.
+See the [setuptools meta.yaml](https://github.com/pyodide/pyodide/
-- `$PKGDIR`: The directory in which the `meta.yaml` file resides.
+blob/main/packages/setuptools/meta.yaml)
-
+for an example of the usage of this key.
-(This key is not in the Conda spec).
 
 ### `build/unvendor-tests`
 

packages/h5py/meta.yaml

@@ -7,12 +7,14 @@ source:
   url: https://files.pythonhosted.org/packages/c5/40/7cf58e6230f0e76699f011c6d293dd47755997709a303a4e644823f3a753/h5py-3.7.0.tar.gz
   sha256: 3fcf37884383c5da64846ab510190720027dca0768def34dd8dcb659dbe5cbf3
   patches:
+    - patches/0001-Fix-incompatible-pointer-type.patch
     - patches/configure.patch
 
 requirements:
   run:
     - numpy
     - pkgconfig
+    - libhdf5
   host:
     - libhdf5
 
@@ -21,3 +23,4 @@ build:
     export HDF5_MPI=OFF
     export H5PY_SETUP_REQUIRES="0"
     export HDF5_VERSION=1.12.1
+    export HDF5_DIR=${WASM_LIBRARY_DIR}

packages/h5py/patches/0001-Fix-incompatible-pointer-type.patch

@@ -0,0 +1,57 @@
+From 30ba08deaa551b0920530f6b2edac59f2dd9e28e Mon Sep 17 00:00:00 2001
+From: Gyeongjae Choi <def6488@gmail.com>
+Date: Fri, 2 Sep 2022 07:54:44 +0000
+Subject: [PATCH] Fix incompatible pointer type
+
+---
+ h5py/_errors.pxd | 2 +-
+ h5py/_errors.pyx | 6 +++---
+ 2 files changed, 4 insertions(+), 4 deletions(-)
+
+diff --git a/h5py/_errors.pxd b/h5py/_errors.pxd
+index 13b299e2..e7791a6b 100644
+--- a/h5py/_errors.pxd
++++ b/h5py/_errors.pxd
+@@ -412,7 +412,7 @@ cdef extern from "hdf5.h":
+ 
+     herr_t    H5Eprint(hid_t estack_id, void *stream)
+ 
+-    ctypedef herr_t (*H5E_walk_t)(int n, H5E_error_t *err_desc, void* client_data)
++    ctypedef herr_t (*H5E_walk_t)(unsigned int n, const H5E_error_t *err_desc, void* client_data)
+     herr_t    H5Ewalk(hid_t estack_id, H5E_direction_t direction, H5E_walk_t func, void* client_data)
+ 
+ # --- Functions for managing the HDF5 error callback mechanism ---
+diff --git a/h5py/_errors.pyx b/h5py/_errors.pyx
+index 612052fb..dabbc632 100644
+--- a/h5py/_errors.pyx
++++ b/h5py/_errors.pyx
+@@ -94,7 +94,7 @@ cdef struct err_data_t:
+     H5E_error_t err
+     int n
+ 
+-cdef herr_t walk_cb(int n, H5E_error_t *desc, void *e) nogil:
++cdef herr_t walk_cb(unsigned int n, const H5E_error_t *desc, void *e) nogil:
+ 
+     cdef err_data_t *ee = <err_data_t*>e
+ 
+@@ -120,7 +120,7 @@ cdef int set_exception() except -1:
+ 
+     err.n = -1
+ 
+-    if H5Ewalk(<hid_t>H5E_DEFAULT, H5E_WALK_UPWARD, walk_cb, &err) < 0:
++    if H5Ewalk(<hid_t>H5E_DEFAULT, H5E_WALK_UPWARD, &walk_cb, &err) < 0:
+         raise RuntimeError("Failed to walk error stack")
+ 
+     if err.n < 0:   # No HDF5 exception information found
+@@ -137,7 +137,7 @@ cdef int set_exception() except -1:
+ 
+     err.n = -1
+ 
+-    if H5Ewalk(<hid_t>H5E_DEFAULT, H5E_WALK_DOWNWARD, walk_cb, &err) < 0:
++    if H5Ewalk(<hid_t>H5E_DEFAULT, H5E_WALK_DOWNWARD, &walk_cb, &err) < 0:
+         raise RuntimeError("Failed to walk error stack")
+ 
+     desc_bottom = err.err.desc
+-- 
+2.37.2
+

packages/h5py/patches/configure.patch

@@ -1,8 +1,17 @@
+From 7aea0dd0b4b86d9c607acc981d8055e55c9685bb Mon Sep 17 00:00:00 2001
+From: ryanking13 <def6488@gmail.com>
+Date: Thu, 30 Jun 2022 05:30:27 +0000
+Subject: [PATCH] Fix feature detection
+
+---
+ setup_configure.py | 26 ++++----------------------
+ 1 file changed, 4 insertions(+), 22 deletions(-)
+
 diff --git a/setup_configure.py b/setup_configure.py
-index 16c355b..85a4f90 100644
+index c16ddeef..1180d3fa 100644
 --- a/setup_configure.py
 +++ b/setup_configure.py
-@@ -183,7 +183,7 @@ class BuildConfig:
+@@ -206,7 +206,7 @@ class BuildConfig:
  class HDF5LibWrapper:
  
      def __init__(self, libdirs):
@@ -11,13 +20,15 @@ index 16c355b..85a4f90 100644
  
      def _load_hdf5_lib(self, libdirs):
          """
-@@ -245,23 +245,9 @@ class HDF5LibWrapper:
+@@ -268,25 +268,7 @@ class HDF5LibWrapper:
+         self._lib = lib
+ 
      def autodetect_version(self):
-         """
+-        """
-         Detect the current version of HDF5, and return X.Y.Z version string.
+-        Detect the current version of HDF5, and return X.Y.Z version string.
 -
 -        Raises an exception if anything goes wrong.
-         """
+-        """
 -        import ctypes
 -        from ctypes import byref
 -
@@ -30,13 +41,13 @@ index 16c355b..85a4f90 100644
 -        except Exception:
 -            print("error: Unable to find HDF5 version")
 -            raise
- 
+-
 -        return int(major.value), int(minor.value), int(release.value)
 +        return (1, 12, 1)
  
      def load_function(self, func_name):
          try:
-@@ -277,7 +263,7 @@ class HDF5LibWrapper:
+@@ -302,10 +284,10 @@ class HDF5LibWrapper:
          return True
  
      def has_mpi_support(self):
@@ -46,3 +57,9 @@ index 16c355b..85a4f90 100644
      def has_ros3_support(self):
 -        return self.has_functions("H5Pget_fapl_ros3", "H5Pset_fapl_ros3")
 +        return False
+ 
+     def has_direct_vfd_support(self):
+         return self.has_functions("H5Pget_fapl_direct", "H5Pset_fapl_direct")
+-- 
+2.35.1
+

packages/h5py/test_h5py.py

@@ -0,0 +1,18 @@
+from pytest_pyodide import run_in_pyodide
+
+
+@run_in_pyodide(packages=["h5py"])
+def test_h5py(selenium):
+    import h5py
+
+    with h5py.File("mytestfile.hdf5", "a") as f:
+        dset = f.create_dataset("mydataset", (100,), dtype="i")
+        grp = f.create_group("subgroup")
+        dset2 = grp.create_dataset("another_dataset", (50,), dtype="f")
+
+        assert f.name == "/"
+        assert dset.name == "/mydataset"
+        assert dset2.name == "/subgroup/another_dataset"
+
+    f = h5py.File("mytestfile.hdf5", "r")
+    assert sorted(list(f.keys())) == ["mydataset", "subgroup"]

packages/libhdf5/meta.yaml

@@ -1,32 +1,57 @@
 package:
   name: libhdf5
   version: 1.12.1
-  _disabled: true
 
 source:
   sha256: e6dde173c2d243551922d23a0387a79961205b018502e6a742acb30b61bc2d5f
   url: https://github.com/HDFGroup/hdf5/archive/refs/tags/hdf5-1_12_1.tar.gz
+  patches:
+    - patches/0001-Hardcode-float-precision.patch
 
 requirements:
   host:
     - zlib
 
 build:
-  library: true
+  sharedlibrary: true
   script: |
+    mkdir -p dist
+    export DISTDIR=$(pwd)/dist
+
     mkdir -p build;
+    echo "set_property(GLOBAL PROPERTY TARGET_SUPPORTS_SHARED_LIBS TRUE)" > build/SupportSharedLib.cmake
+
     cd build \
-        && LDFLAGS="-s NODERAWFS=1 -sUSE_ZLIB=1 -sFORCE_FILESYSTEM=1" emcmake cmake ../ \
+        && emcmake cmake \
         -DCMAKE_INSTALL_PREFIX=${WASM_LIBRARY_DIR} \
+        -DCMAKE_PROJECT_INCLUDE=$(pwd)/SupportSharedLib.cmake \
         -DH5_HAVE_GETPWUID=0 \
         -DH5_HAVE_SIGNAL=0 \
-        -DBUILD_SHARED_LIBS=0 \
+        -DCMAKE_BUILD_TYPE=Release \
-        -DBUILD_STATIC_LIBS=1 \
+        -DBUILD_SHARED_LIBS=1 \
+        -DBUILD_STATIC_LIBS=0 \
+        -DONLY_SHARED_LIBS=1 \
         -DBUILD_TESTING=0 \
-        -DCMAKE_C_FLAGS="-Wno-incompatible-pointer-types-discards-qualifiers" \
+        -DCMAKE_C_FLAGS="-fPIC -Wno-incompatible-pointer-types-discards-qualifiers" \
+        -DCMAKE_CXX_FLAGS="-fPIC -Wno-incompatible-pointer-types-discards-qualifiers" \
+        -DCMAKE_SHARED_LINKER_FLAGS="${SIDE_MODULE_LDFLAGS} -s NODERAWFS=1 -sFORCE_FILESYSTEM=1" \
         -DHDF5_BUILD_EXAMPLES=0 \
         -DHDF5_BUILD_TOOLS=0 \
         -DHDF5_BUILD_UTILS=0 \
         -DHDF5_ENABLE_Z_LIB_SUPPORT=1 \
-        -DHDF5_ENABLE_ROS3_VFD=0;
+        -DHDF5_ENABLE_ROS3_VFD=0 \
+        -DZLIB_INCLUDE_DIR=${WASM_LIBRARY_DIR}/include \
+        -DZLIB_LIBRARY=${WASM_LIBRARY_DIR}/lib/libz.a \
+        ../
+
+    # TODO(ryanking13): These files need to be generated dynamically during the build process
+    # by running a executable.
+    # However, since we build a side module, the emitted executable is a wasm object
+    # so it is not runnable even with the emulator (node).
+    # I think there should be a proper way to emit a js file which is runnable with node,
+    # But I was to exhausted to find it. So I created these files locally.
+    cp ${PKGDIR}/settings/* src/
+
     emmake make -j ${PYODIDE_JOBS:-3} install
+
+    cp -P ${WASM_LIBRARY_DIR}/lib/libhdf* ${DISTDIR}

packages/libhdf5/patches/0001-Fix-incompatible-pointer-type.patch

@@ -0,0 +1,39 @@
+From 873077a5b132c4294f0ea6af2eb29d61352c6688 Mon Sep 17 00:00:00 2001
+From: Gyeongjae Choi <def6488@gmail.com>
+Date: Fri, 2 Sep 2022 07:54:44 +0000
+Subject: [PATCH 1/1] Fix incompatible pointer type
+
+---
+ h5py/_errors.pxd | 2 +-
+ h5py/_errors.pyx | 2 +-
+ 2 files changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/h5py/_errors.pxd b/h5py/_errors.pxd
+index 13b299e2..e7791a6b 100644
+--- a/h5py/_errors.pxd
++++ b/h5py/_errors.pxd
+@@ -412,7 +412,7 @@ cdef extern from "hdf5.h":
+ 
+     herr_t    H5Eprint(hid_t estack_id, void *stream)
+ 
+-    ctypedef herr_t (*H5E_walk_t)(int n, H5E_error_t *err_desc, void* client_data)
++    ctypedef herr_t (*H5E_walk_t)(unsigned int n, const H5E_error_t *err_desc, void* client_data)
+     herr_t    H5Ewalk(hid_t estack_id, H5E_direction_t direction, H5E_walk_t func, void* client_data)
+ 
+ # --- Functions for managing the HDF5 error callback mechanism ---
+diff --git a/h5py/_errors.pyx b/h5py/_errors.pyx
+index 612052fb..ca7b1c48 100644
+--- a/h5py/_errors.pyx
++++ b/h5py/_errors.pyx
+@@ -94,7 +94,7 @@ cdef struct err_data_t:
+     H5E_error_t err
+     int n
+ 
+-cdef herr_t walk_cb(int n, H5E_error_t *desc, void *e) nogil:
++cdef herr_t walk_cb(unsigned int n, const H5E_error_t *desc, void *e) nogil:
+ 
+     cdef err_data_t *ee = <err_data_t*>e
+ 
+-- 
+2.37.0
+

packages/libhdf5/patches/0001-Hardcode-float-precision.patch

@@ -0,0 +1,29 @@
+From d6ef0fd66c42ffbba13b5d7a006457ca7f27fb44 Mon Sep 17 00:00:00 2001
+From: Gyeongjae Choi <def6488@gmail.com>
+Date: Fri, 2 Sep 2022 04:25:31 +0000
+Subject: [PATCH] Hardcode float precision
+
+HDF5 tries to check float precision by compiling and running
+a simple C program but it is not available with SIDE_MODULE setting.
+So we hardcode the value.
+---
+ config/cmake/ConfigureChecks.cmake | 3 ++-
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+
+diff --git a/config/cmake/ConfigureChecks.cmake b/config/cmake/ConfigureChecks.cmake
+index 902ddd3d3e..0a92cbb9ba 100644
+--- a/config/cmake/ConfigureChecks.cmake
++++ b/config/cmake/ConfigureChecks.cmake
+@@ -331,7 +331,8 @@ set (PROG_SRC
+ )
+ 
+ C_RUN ("maximum decimal precision for C" ${PROG_SRC} PROG_RES)
+-file (READ "${RUN_OUTPUT_PATH_DEFAULT}/pac_Cconftest.out" PROG_OUTPUT4)
++list(APPEND PROG_OUTPUT4 36)
++list(APPEND PROG_OUTPUT4 0)
+ message (STATUS "Testing maximum decimal precision for C - ${PROG_OUTPUT4}")
+ 
+ # dnl The output from the above program will be:
+-- 
+2.37.0
+

packages/libhdf5/settings/H5lib_settings.c

@@ -0,0 +1,29 @@
+/* Generated automatically by H5make_libsettings -- do not edit */
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group.                                               *
+ * Copyright by the Board of Trustees of the University of Illinois.         *
+ * All rights reserved.                                                      *
+ *                                                                           *
+ * This file is part of HDF5.  The full HDF5 copyright notice, including     *
+ * terms governing use, modification, and redistribution, is contained in    *
+ * the COPYING file, which can be found at the root of the source code       *
+ * distribution tree, or in https://www.hdfgroup.org/licenses.               *
+ * If you do not have access to either file, you may request a copy from     *
+ * help@hdfgroup.org.                                                        *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Created:		Sep  2, 2022
+ *			@emscripten
+ *
+ * Purpose:		This machine-generated source code contains
+ *			information about the library build configuration
+ *
+ * Modifications:
+ *
+ *	DO NOT MAKE MODIFICATIONS TO THIS FILE!
+ *	It was generated by code in `H5make_libsettings.c'.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+char H5libhdf5_settings[] = "";

pyodide-build/pyodide_build/buildpkg.py

@@ -526,7 +526,7 @@ def package_wheel(
     post = build_metadata.post
     if post:
         print("Running post script in ", str(Path.cwd().absolute()))
-        bash_runner.env.update({"PKGDIR": str(pkg_root), "WHEELDIR": str(wheel_dir)})
+        bash_runner.env.update({"WHEELDIR": str(wheel_dir)})
         result = bash_runner.run(post)
         if result.returncode != 0:
             print("ERROR: post failed")
@@ -759,6 +759,7 @@ def build_package(
     os.dup2(tee.stdin.fileno(), sys.stderr.fileno())  # type: ignore[union-attr]
 
     with chdir(pkg_root), get_bash_runner() as bash_runner:
+        bash_runner.env["PKGDIR"] = str(pkg_root)
         bash_runner.env["PKG_VERSION"] = version
         bash_runner.env["PKG_BUILD_DIR"] = str(srcpath)
         if not continue_: