Rooba
/
pyodide
mirror of https://github.com/pyodide/pyodide.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Enable libhdf5 and h5py (#2812)

This commit is contained in:
Gyeongjae Choi 2022-09-13 16:17:52 +09:00 committed by GitHub
parent c3a0ddb0c7
commit 2dc937c0b6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 1327 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
docs/development/meta-yaml.md
View File

@ -154,6 +154,14 @@ Python package, the script section will be run before the build system runs
`setup.py`. This script is run by `bash` in the directory where the tarball was
extracted.
There are special environment variables defined:
- `$PKGDIR`: The directory in which the `meta.yaml` file resides.
- `$PKG_VESRION`: The version of the package
- `$PKG_BUILD_DIR`: The directory where the tarball was extracted.
(These keys are not in the Conda spec).
### `build/cross-script`
This script will run _after_ `build/script`. The difference is that it runs with
@ -165,14 +173,13 @@ is the source directory.
### `build/post`
Shell commands to run after building the library. These are run with `bash`, and
there are two special environment variables defined:
- `$SITEPACKAGES`: The `site-packages` directory into which the package has been
installed.
- `$PKGDIR`: The directory in which the `meta.yaml` file resides.
(This key is not in the Conda spec).
Shell commands to run after building the library. This script is run by `bash`
in the directory where `meta.yaml` file resides. The `${PKG_BUILD_DIR}/dist`
will contain the built wheel unpacked with `python -m wheel unpack`
so it's possible to manually add, delete, change, move files etc.
See the [setuptools meta.yaml](https://github.com/pyodide/pyodide/
blob/main/packages/setuptools/meta.yaml)
for an example of the usage of this key.
### `build/unvendor-tests`

3
packages/h5py/meta.yaml
View File

@ -7,12 +7,14 @@ source:
url: https://files.pythonhosted.org/packages/c5/40/7cf58e6230f0e76699f011c6d293dd47755997709a303a4e644823f3a753/h5py-3.7.0.tar.gz
sha256: 3fcf37884383c5da64846ab510190720027dca0768def34dd8dcb659dbe5cbf3
patches:
- patches/0001-Fix-incompatible-pointer-type.patch
- patches/configure.patch
requirements:
run:
- numpy
- pkgconfig
- libhdf5
host:
- libhdf5
@ -21,3 +23,4 @@ build:
export HDF5_MPI=OFF
export H5PY_SETUP_REQUIRES="0"
export HDF5_VERSION=1.12.1
export HDF5_DIR=${WASM_LIBRARY_DIR}

57
packages/h5py/patches/0001-Fix-incompatible-pointer-type.patch Normal file
View File

@ -0,0 +1,57 @@
From 30ba08deaa551b0920530f6b2edac59f2dd9e28e Mon Sep 17 00:00:00 2001
From: Gyeongjae Choi <def6488@gmail.com>
Date: Fri, 2 Sep 2022 07:54:44 +0000
Subject: [PATCH] Fix incompatible pointer type
---
h5py/_errors.pxd | 2 +-
h5py/_errors.pyx | 6 +++---
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/h5py/_errors.pxd b/h5py/_errors.pxd
index 13b299e2..e7791a6b 100644
--- a/h5py/_errors.pxd
+++ b/h5py/_errors.pxd
@@ -412,7 +412,7 @@ cdef extern from "hdf5.h":
herr_t H5Eprint(hid_t estack_id, void *stream)
- ctypedef herr_t (*H5E_walk_t)(int n, H5E_error_t *err_desc, void* client_data)
+ ctypedef herr_t (*H5E_walk_t)(unsigned int n, const H5E_error_t *err_desc, void* client_data)
herr_t H5Ewalk(hid_t estack_id, H5E_direction_t direction, H5E_walk_t func, void* client_data)
# --- Functions for managing the HDF5 error callback mechanism ---
diff --git a/h5py/_errors.pyx b/h5py/_errors.pyx
index 612052fb..dabbc632 100644
--- a/h5py/_errors.pyx
+++ b/h5py/_errors.pyx
@@ -94,7 +94,7 @@ cdef struct err_data_t:
H5E_error_t err
int n
-cdef herr_t walk_cb(int n, H5E_error_t *desc, void *e) nogil:
+cdef herr_t walk_cb(unsigned int n, const H5E_error_t *desc, void *e) nogil:
cdef err_data_t *ee = <err_data_t*>e
@@ -120,7 +120,7 @@ cdef int set_exception() except -1:
err.n = -1
- if H5Ewalk(<hid_t>H5E_DEFAULT, H5E_WALK_UPWARD, walk_cb, &err) < 0:
+ if H5Ewalk(<hid_t>H5E_DEFAULT, H5E_WALK_UPWARD, &walk_cb, &err) < 0:
raise RuntimeError("Failed to walk error stack")
if err.n < 0: # No HDF5 exception information found
@@ -137,7 +137,7 @@ cdef int set_exception() except -1:
err.n = -1
- if H5Ewalk(<hid_t>H5E_DEFAULT, H5E_WALK_DOWNWARD, walk_cb, &err) < 0:
+ if H5Ewalk(<hid_t>H5E_DEFAULT, H5E_WALK_DOWNWARD, &walk_cb, &err) < 0:
raise RuntimeError("Failed to walk error stack")
desc_bottom = err.err.desc
--
2.37.2

33
packages/h5py/patches/configure.patch
View File

@ -1,8 +1,17 @@
From 7aea0dd0b4b86d9c607acc981d8055e55c9685bb Mon Sep 17 00:00:00 2001
From: ryanking13 <def6488@gmail.com>
Date: Thu, 30 Jun 2022 05:30:27 +0000
Subject: [PATCH] Fix feature detection
---
setup_configure.py | 26 ++++----------------------
1 file changed, 4 insertions(+), 22 deletions(-)
diff --git a/setup_configure.py b/setup_configure.py
index 16c355b..85a4f90 100644
index c16ddeef..1180d3fa 100644
--- a/setup_configure.py
+++ b/setup_configure.py
@@ -183,7 +183,7 @@ class BuildConfig:
@@ -206,7 +206,7 @@ class BuildConfig:
class HDF5LibWrapper:
def __init__(self, libdirs):
@ -11,13 +20,15 @@ index 16c355b..85a4f90 100644
def _load_hdf5_lib(self, libdirs):
"""
@@ -245,23 +245,9 @@ class HDF5LibWrapper:
@@ -268,25 +268,7 @@ class HDF5LibWrapper:
self._lib = lib
def autodetect_version(self):
"""
Detect the current version of HDF5, and return X.Y.Z version string.
- """
- Detect the current version of HDF5, and return X.Y.Z version string.
-
- Raises an exception if anything goes wrong.
"""
- """
- import ctypes
- from ctypes import byref
-
@ -30,13 +41,13 @@ index 16c355b..85a4f90 100644
- except Exception:
- print("error: Unable to find HDF5 version")
- raise
-
- return int(major.value), int(minor.value), int(release.value)
+ return (1, 12, 1)
def load_function(self, func_name):
try:
@@ -277,7 +263,7 @@ class HDF5LibWrapper:
@@ -302,10 +284,10 @@ class HDF5LibWrapper:
return True
def has_mpi_support(self):
@ -46,3 +57,9 @@ index 16c355b..85a4f90 100644
def has_ros3_support(self):
- return self.has_functions("H5Pget_fapl_ros3", "H5Pset_fapl_ros3")
+ return False
def has_direct_vfd_support(self):
return self.has_functions("H5Pget_fapl_direct", "H5Pset_fapl_direct")
--
2.35.1

18
packages/h5py/test_h5py.py Normal file
View File

@ -0,0 +1,18 @@
from pytest_pyodide import run_in_pyodide
@run_in_pyodide(packages=["h5py"])
def test_h5py(selenium):
import h5py
with h5py.File("mytestfile.hdf5", "a") as f:
dset = f.create_dataset("mydataset", (100,), dtype="i")
grp = f.create_group("subgroup")
dset2 = grp.create_dataset("another_dataset", (50,), dtype="f")
assert f.name == "/"
assert dset.name == "/mydataset"
assert dset2.name == "/subgroup/another_dataset"
f = h5py.File("mytestfile.hdf5", "r")
assert sorted(list(f.keys())) == ["mydataset", "subgroup"]

39
packages/libhdf5/meta.yaml
View File

@ -1,32 +1,57 @@
package:
name: libhdf5
version: 1.12.1
_disabled: true
source:
sha256: e6dde173c2d243551922d23a0387a79961205b018502e6a742acb30b61bc2d5f
url: https://github.com/HDFGroup/hdf5/archive/refs/tags/hdf5-1_12_1.tar.gz
patches:
- patches/0001-Hardcode-float-precision.patch
requirements:
host:
- zlib
build:
library: true
sharedlibrary: true
script: |
mkdir -p dist
export DISTDIR=$(pwd)/dist
mkdir -p build;
echo "set_property(GLOBAL PROPERTY TARGET_SUPPORTS_SHARED_LIBS TRUE)" > build/SupportSharedLib.cmake
cd build \
&& LDFLAGS="-s NODERAWFS=1 -sUSE_ZLIB=1 -sFORCE_FILESYSTEM=1" emcmake cmake ../ \
&& emcmake cmake \
-DCMAKE_INSTALL_PREFIX=${WASM_LIBRARY_DIR} \
-DCMAKE_PROJECT_INCLUDE=$(pwd)/SupportSharedLib.cmake \
-DH5_HAVE_GETPWUID=0 \
-DH5_HAVE_SIGNAL=0 \
-DBUILD_SHARED_LIBS=0 \
-DBUILD_STATIC_LIBS=1 \
-DCMAKE_BUILD_TYPE=Release \
-DBUILD_SHARED_LIBS=1 \
-DBUILD_STATIC_LIBS=0 \
-DONLY_SHARED_LIBS=1 \
-DBUILD_TESTING=0 \
-DCMAKE_C_FLAGS="-Wno-incompatible-pointer-types-discards-qualifiers" \
-DCMAKE_C_FLAGS="-fPIC -Wno-incompatible-pointer-types-discards-qualifiers" \
-DCMAKE_CXX_FLAGS="-fPIC -Wno-incompatible-pointer-types-discards-qualifiers" \
-DCMAKE_SHARED_LINKER_FLAGS="${SIDE_MODULE_LDFLAGS} -s NODERAWFS=1 -sFORCE_FILESYSTEM=1" \
-DHDF5_BUILD_EXAMPLES=0 \
-DHDF5_BUILD_TOOLS=0 \
-DHDF5_BUILD_UTILS=0 \
-DHDF5_ENABLE_Z_LIB_SUPPORT=1 \
-DHDF5_ENABLE_ROS3_VFD=0;
-DHDF5_ENABLE_ROS3_VFD=0 \
-DZLIB_INCLUDE_DIR=${WASM_LIBRARY_DIR}/include \
-DZLIB_LIBRARY=${WASM_LIBRARY_DIR}/lib/libz.a \
../
# TODO(ryanking13): These files need to be generated dynamically during the build process
# by running a executable.
# However, since we build a side module, the emitted executable is a wasm object
# so it is not runnable even with the emulator (node).
# I think there should be a proper way to emit a js file which is runnable with node,
# But I was to exhausted to find it. So I created these files locally.
cp ${PKGDIR}/settings/* src/
emmake make -j ${PYODIDE_JOBS:-3} install
cp -P ${WASM_LIBRARY_DIR}/lib/libhdf* ${DISTDIR}

39
packages/libhdf5/patches/0001-Fix-incompatible-pointer-type.patch Normal file
View File

@ -0,0 +1,39 @@
From 873077a5b132c4294f0ea6af2eb29d61352c6688 Mon Sep 17 00:00:00 2001
From: Gyeongjae Choi <def6488@gmail.com>
Date: Fri, 2 Sep 2022 07:54:44 +0000
Subject: [PATCH 1/1] Fix incompatible pointer type
---
h5py/_errors.pxd | 2 +-
h5py/_errors.pyx | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/h5py/_errors.pxd b/h5py/_errors.pxd
index 13b299e2..e7791a6b 100644
--- a/h5py/_errors.pxd
+++ b/h5py/_errors.pxd
@@ -412,7 +412,7 @@ cdef extern from "hdf5.h":
herr_t H5Eprint(hid_t estack_id, void *stream)
- ctypedef herr_t (*H5E_walk_t)(int n, H5E_error_t *err_desc, void* client_data)
+ ctypedef herr_t (*H5E_walk_t)(unsigned int n, const H5E_error_t *err_desc, void* client_data)
herr_t H5Ewalk(hid_t estack_id, H5E_direction_t direction, H5E_walk_t func, void* client_data)
# --- Functions for managing the HDF5 error callback mechanism ---
diff --git a/h5py/_errors.pyx b/h5py/_errors.pyx
index 612052fb..ca7b1c48 100644
--- a/h5py/_errors.pyx
+++ b/h5py/_errors.pyx
@@ -94,7 +94,7 @@ cdef struct err_data_t:
H5E_error_t err
int n
-cdef herr_t walk_cb(int n, H5E_error_t *desc, void *e) nogil:
+cdef herr_t walk_cb(unsigned int n, const H5E_error_t *desc, void *e) nogil:
cdef err_data_t *ee = <err_data_t*>e
--
2.37.0

29
packages/libhdf5/patches/0001-Hardcode-float-precision.patch Normal file
View File

@ -0,0 +1,29 @@
From d6ef0fd66c42ffbba13b5d7a006457ca7f27fb44 Mon Sep 17 00:00:00 2001
From: Gyeongjae Choi <def6488@gmail.com>
Date: Fri, 2 Sep 2022 04:25:31 +0000
Subject: [PATCH] Hardcode float precision
HDF5 tries to check float precision by compiling and running
a simple C program but it is not available with SIDE_MODULE setting.
So we hardcode the value.
---
config/cmake/ConfigureChecks.cmake | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/config/cmake/ConfigureChecks.cmake b/config/cmake/ConfigureChecks.cmake
index 902ddd3d3e..0a92cbb9ba 100644
--- a/config/cmake/ConfigureChecks.cmake
+++ b/config/cmake/ConfigureChecks.cmake
@@ -331,7 +331,8 @@ set (PROG_SRC
)
C_RUN ("maximum decimal precision for C" ${PROG_SRC} PROG_RES)
-file (READ "${RUN_OUTPUT_PATH_DEFAULT}/pac_Cconftest.out" PROG_OUTPUT4)
+list(APPEND PROG_OUTPUT4 36)
+list(APPEND PROG_OUTPUT4 0)
message (STATUS "Testing maximum decimal precision for C - ${PROG_OUTPUT4}")
# dnl The output from the above program will be:
--
2.37.0

1078
packages/libhdf5/settings/H5Tinit.c Normal file
View File

File diff suppressed because it is too large Load Diff

29
packages/libhdf5/settings/H5lib_settings.c Normal file
View File

@ -0,0 +1,29 @@
/* Generated automatically by H5make_libsettings -- do not edit */
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* Copyright by the Board of Trustees of the University of Illinois. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://www.hdfgroup.org/licenses. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Created: Sep 2, 2022
* @emscripten
*
* Purpose: This machine-generated source code contains
* information about the library build configuration
*
* Modifications:
*
* DO NOT MAKE MODIFICATIONS TO THIS FILE!
* It was generated by code in `H5make_libsettings.c'.
*
*-------------------------------------------------------------------------
*/
char H5libhdf5_settings[] = "";

3
pyodide-build/pyodide_build/buildpkg.py
View File

@ -526,7 +526,7 @@ def package_wheel(
post = build_metadata.post
if post:
print("Running post script in ", str(Path.cwd().absolute()))
bash_runner.env.update({"PKGDIR": str(pkg_root), "WHEELDIR": str(wheel_dir)})
bash_runner.env.update({"WHEELDIR": str(wheel_dir)})
result = bash_runner.run(post)
if result.returncode != 0:
print("ERROR: post failed")
@ -759,6 +759,7 @@ def build_package(
os.dup2(tee.stdin.fileno(), sys.stderr.fileno()) # type: ignore[union-attr]
with chdir(pkg_root), get_bash_runner() as bash_runner:
bash_runner.env["PKGDIR"] = str(pkg_root)
bash_runner.env["PKG_VERSION"] = version
bash_runner.env["PKG_BUILD_DIR"] = str(srcpath)
if not continue_: