diff --git a/docs/project/changelog.md b/docs/project/changelog.md index b5c79c73a..7825e7ef4 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -116,6 +116,7 @@ substitutions: - New packages: pycryptodomex {pr}`2966`, pycryptodome {pr}`2965`, coverage-py {pr}`3053`, bcrypt {pr}`3125`, lightgbm {pr}`3138`, pyheif, pillow_heif, libheif, libde265 {pr}`3161`, wordcloud {pr}`3173`, + gdal, fiona, geopandas {pr}`3213`, the standard library \_hashlib module {pr}`3206` - {{ Breaking }} Unvendored the sqlite3 module from the standard library. diff --git a/packages/_tests/test_packages_common.py b/packages/_tests/test_packages_common.py index 6e9995ce1..d7885adf2 100644 --- a/packages/_tests/test_packages_common.py +++ b/packages/_tests/test_packages_common.py @@ -49,7 +49,7 @@ def test_parse_package(name: str) -> None: @pytest.mark.skip_refcount_check -@pytest.mark.driver_timeout(60) +@pytest.mark.driver_timeout(120) @pytest.mark.parametrize("name", registered_packages()) @pytest.mark.benchmark( max_time=3, diff --git a/packages/click/meta.yaml b/packages/click/meta.yaml new file mode 100644 index 000000000..cedec3320 --- /dev/null +++ b/packages/click/meta.yaml @@ -0,0 +1,14 @@ +package: + name: click + version: 8.1.3 +source: + url: https://files.pythonhosted.org/packages/c2/f1/df59e28c642d583f7dacffb1e0965d0e00b218e0186d7858ac5233dce840/click-8.1.3-py3-none-any.whl + sha256: bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48 +test: + imports: + - click +about: + home: https://palletsprojects.com/p/click/ + PyPI: https://pypi.org/project/click + summary: Composable command line interface toolkit + license: BSD-3-Clause diff --git a/packages/cligj/meta.yaml b/packages/cligj/meta.yaml new file mode 100644 index 000000000..f456a2101 --- /dev/null +++ b/packages/cligj/meta.yaml @@ -0,0 +1,17 @@ +package: + name: cligj + version: 0.7.2 +source: + url: https://files.pythonhosted.org/packages/73/86/43fa9f15c5b9fb6e82620428827cd3c284aa933431405d1bcf5231ae3d3e/cligj-0.7.2-py3-none-any.whl + sha256: c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df +test: + imports: + - cligj +requirements: + run: + - click +about: + home: https://github.com/mapbox/cligj + PyPI: https://pypi.org/project/cligj + summary: Click params for command line interfaces to GeoJSON + license: BSD diff --git a/packages/fiona/meta.yaml b/packages/fiona/meta.yaml new file mode 100644 index 000000000..d9233f5f4 --- /dev/null +++ b/packages/fiona/meta.yaml @@ -0,0 +1,40 @@ +package: + name: fiona + version: 1.8.21 +source: + url: https://files.pythonhosted.org/packages/67/5c/4e028e84a1f0cb3f8a994217cf2366360ca984dfc1433f6171de527d0dca/Fiona-1.8.21.tar.gz + sha256: 3a0edca2a7a070db405d71187214a43d2333a57b4097544a3fcc282066a58bfc +test: + imports: + - fiona + - fiona.fio +requirements: + host: + - gdal + - geos + run: + - gdal + - geos + - attrs + - certifi + - setuptools + - six + - click + - cligj + # - click-plugins # only for fiona CLI? + +build: + script: | + export PACKAGE_DATA=1 + export GDAL_CONFIG=${WASM_LIBRARY_DIR}/bin/gdal-config + export GDAL_DATA=${WASM_LIBRARY_DIR}/share/gdal + export PROJ_LIB=${WASM_LIBRARY_DIR}/share/proj + echo ${GDAL_CONFIG} + echo ${GDAL_DATA} + echo ${PROJ_LIB} + +about: + home: http://github.com/Toblerity/Fiona + PyPI: https://pypi.org/project/fiona + summary: Fiona reads and writes spatial data files + license: BSD diff --git a/packages/fiona/test_data/fiona-tests-1.8.21.zip b/packages/fiona/test_data/fiona-tests-1.8.21.zip new file mode 100644 index 000000000..9255aec37 Binary files /dev/null and b/packages/fiona/test_data/fiona-tests-1.8.21.zip differ diff --git a/packages/fiona/test_fiona.py b/packages/fiona/test_fiona.py new file mode 100644 index 000000000..0a6e7722b --- /dev/null +++ b/packages/fiona/test_fiona.py @@ -0,0 +1,75 @@ +import pathlib + +import pytest +from pytest_pyodide import run_in_pyodide + +TEST_DATA_PATH = pathlib.Path(__file__).parent / "test_data" + + +@pytest.mark.driver_timeout(60) +@run_in_pyodide(packages=["fiona"]) +def test_supported_drivers(selenium): + import fiona + + assert fiona.driver_count() > 0 + + +@pytest.mark.driver_timeout(60) +def test_runtest(selenium): + @run_in_pyodide(packages=["fiona", "pytest"]) + def _run(selenium, data): + import zipfile + + with open("tests.zip", "wb") as f: + f.write(data) + + with zipfile.ZipFile("tests.zip", "r") as zf: + zf.extractall("tests") + + import sys + + sys.path.append("tests") + + import pytest + + def runtest(test_filter, ignore_filters): + + ignore_filter = [] + for ignore in ignore_filters: + ignore_filter.append("--ignore-glob") + ignore_filter.append(ignore) + + ret = pytest.main( + [ + "--pyargs", + "tests", + "--continue-on-collection-errors", + # "-v", + *ignore_filter, + "-k", + test_filter, + ] + ) + assert ret == 0 + + runtest( + ( + "not ordering " # hangs + "and not env " # No module named "boto3" + "and not slice " # GML file format not supported + "and not GML " # GML file format not supported + "and not TestNonCountingLayer " # GPX file format not supported + "and not test_schema_default_fields_wrong_type " # GPX file format not supported + "and not http " + "and not FlatGeobuf" # assertion error + ), + [ + "tests/test_fio*", # no CLI tests + "tests/test_data_paths.py", # no CLI tests + "tests/test_datetime.py", # no CLI tests + "tests/test_vfs.py", # No module named "boto3" + ], + ) + + TEST_DATA = (TEST_DATA_PATH / "fiona-tests-1.8.21.zip").read_bytes() + _run(selenium, TEST_DATA) diff --git a/packages/gdal/meta.yaml b/packages/gdal/meta.yaml new file mode 100644 index 000000000..45f176ab5 --- /dev/null +++ b/packages/gdal/meta.yaml @@ -0,0 +1,102 @@ +package: + name: gdal + version: 3.5.1 + +source: + url: https://github.com/OSGeo/gdal/releases/download/v3.5.2/gdal-3.5.2.tar.gz + sha256: fbd696e1b2a858fbd2eb3718db16b14ed9ba82521d3578770d480c74fe1146d2 +requirements: + host: + - libtiff + - libproj + - libiconv + - geos + run: + - geos + +build: + sharedlibrary: true + script: | + export EMSCRIPTEN_SYSROOT=$(em-config CACHE)/sysroot + export EMSCRIPTEN_INCLUDE=$EMSCRIPTEN_SYSROOT/include + export EMSCRIPTEN_BIN=$EMSCRIPTEN_SYSROOT/bin + export EMSCRIPTEN_LIB=$EMSCRIPTEN_SYSROOT/lib/wasm32-emscripten/pic + + mkdir -p dist + export DISTDIR=$(pwd)/dist + + embuilder build zlib --pic + embuilder build libjpeg --pic + embuilder build libpng --pic + embuilder build sqlite3 --pic + + mkdir -p build + cd build && emcmake cmake .. \ + -DCMAKE_INSTALL_PREFIX=$WASM_LIBRARY_DIR \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=True \ + -DBUILD_APPS=OFF \ + -DCMAKE_C_FLAGS="${SIDE_MODULE_CFLAGS} -Wno-deprecated-declarations -Wno-single-bit-bitfield-constant-conversion" \ + -DCMAKE_CXX_FLAGS="${SIDE_MODULE_CFLAGS} -Wno-deprecated-declarations -Wno-single-bit-bitfield-constant-conversion" \ + -DCMAKE_SHARED_LINKER_FLAGS="-sSIDE_MODULE=1 -sWASM_BIGINT" \ + -DGDAL_USE_EXTERNAL_LIBS=OFF \ + -DGDAL_USE_INTERNAL_LIBS=OFF \ + \ + -DPROJ_INCLUDE_DIR=$WASM_LIBRARY_DIR/include \ + -DPROJ_LIBRARY=$WASM_LIBRARY_DIR/lib/libproj.a \ + \ + -DGDAL_USE_ICONV=ON \ + -DIconv_INCLUDE_DIR=$WASM_LIBRARY_DIR/include \ + -DIconv_LIBRARY=$WASM_LIBRARY_DIR/lib/libiconv.a \ + \ + -DGDAL_USE_TIFF=ON \ + -DTIFF_INCLUDE_DIR=$WASM_LIBRARY_DIR/include \ + -DTIFF_LIBRARY=$WASM_LIBRARY_DIR/lib/libtiff.a \ + \ + -DGDAL_USE_GEOS=ON \ + -DGEOS_INCLUDE_DIR=$WASM_LIBRARY_DIR/include \ + -DGEOS_LIBRARY=$WASM_LIBRARY_DIR/lib/libgeos.so \ + \ + -DGDAL_USE_ZLIB=ON \ + -DZLIB_INCLUDE_DIR=$EMSCRIPTEN_INCLUDE \ + -DZLIB_LIBRARY=$EMSCRIPTEN_LIB/libz.a \ + \ + -DGDAL_USE_PNG=ON \ + -DPNG_PNG_INCLUDE_DIR=$EMSCRIPTEN_INCLUDE \ + -DPNG_LIBRARY_RELEASE=$EMSCRIPTEN_LIB/libpng.a \ + \ + -DGDAL_USE_JPEG=ON \ + -DJPEG_INCLUDE_DIR=$EMSCRIPTEN_INCLUDE \ + -DJPEG_LIBRARY_RELEASE=$EMSCRIPTEN_LIB/libjpeg.a \ + \ + -DGDAL_USE_SQLITE3=ON \ + -DSQLite3_INCLUDE_DIR=$EMSCRIPTEN_INCLUDE \ + -DSQLite3_LIBRARY=$EMSCRIPTEN_LIB/libsqlite3.a \ + \ + -DGDAL_USE_GEOTIFF_INTERNAL=ON \ + -DGDAL_USE_QHULL_INTERNAL=ON \ + -DGDAL_USE_LERC_INTERNAL=ON \ + -DGDAL_USE_JSONC_INTERNAL=ON \ + -DGDAL_USE_PCRE2=OFF \ + -DBUILD_TESTING=OFF + + + # Note: + # CMake tries to link the same libraries multiple times at the final link time. + # This is probably because the same libraries are used in multiple submodules. + # This behavior is okay when those libraries are "shared" libraries, + # but we often build static libraries and linking static libraries + # multiple times results in a duplicated symbol error. + # I wasn't able to find a way to prevent CMake from emitting duplicated libraries. + # This is a hack which removes all duplicated appearances of static libs at the final link time. + + export LINKLIBS=$(pwd)/CMakeFiles/GDAL.dir/link.txt + cat ${LINKLIBS} | grep -o '\S*' | grep "\.a$" | sort | uniq | tr "\n" " " > linked_static_libs.txt + cat ${LINKLIBS} | grep -o '\S*' | grep -v "\.a$" | tr "\n" " " > link_cmd.txt + cat link_cmd.txt > ${LINKLIBS} + cat linked_static_libs.txt >> ${LINKLIBS} + + emmake make -j ${PYODIDE_JOBS:-3} + emmake make install + + cp ${WASM_LIBRARY_DIR}/lib/libgdal.so ${DISTDIR} diff --git a/packages/geopandas/meta.yaml b/packages/geopandas/meta.yaml new file mode 100644 index 000000000..6942b963e --- /dev/null +++ b/packages/geopandas/meta.yaml @@ -0,0 +1,21 @@ +package: + name: geopandas + version: 0.11.1 + top-level: + - geopandas +source: + url: https://files.pythonhosted.org/packages/de/83/08b7284066b268e71019398b9094b1c860816aa9b9422f1c489459bfbdf1/geopandas-0.11.1-py3-none-any.whl + sha256: f3344937f3866e52996c7e505d56dae78be117dc840cd1c23507da0b33c0af71 +requirements: + run: + - shapely + - fiona + - pyproj + - packaging + - pandas + # - matplotlib # matplotlib is not a hard dependency +about: + home: http://geopandas.org + PyPI: https://pypi.org/project/geopandas + summary: Geographic pandas extensions + license: BSD diff --git a/packages/geopandas/test_geopandas.py b/packages/geopandas/test_geopandas.py new file mode 100644 index 000000000..de43d38b2 --- /dev/null +++ b/packages/geopandas/test_geopandas.py @@ -0,0 +1,63 @@ +import pytest +from pytest_pyodide import run_in_pyodide + + +@pytest.mark.driver_timeout(300) +@run_in_pyodide(packages=["geopandas", "geopandas-tests", "pytest"]) +def test_runtest(selenium): + from pathlib import Path + + import geopandas + import pytest + + test_path = Path(geopandas.__file__).parent / "tests" + + def runtest(test_filter, ignore_filters): + + ignore_filter = [] + for ignore in ignore_filters: + ignore_filter.append("--ignore-glob") + ignore_filter.append(ignore) + + ret = pytest.main( + [ + "--pyargs", + str(test_path), + "--continue-on-collection-errors", + # "-v", + *ignore_filter, + "-k", + test_filter, + # "--durations", + # "20", + ] + ) + assert ret == 0 + + runtest( + ( + "not test_transform2 " # CppException std::invalid_argument: non double value + "and not test_no_additional_imports " # subprocess + "and not test_pandas_kind " # scipy required + ), + [ + str( + test_path / "test_dissolve.py" + ), # CppException osgeo::proj::io::ParsingException: unrecognized format / unknown name + str( + test_path / "test_geodataframe.py" + ), # CppException osgeo::proj::io::ParsingException: unrecognized format / unknown name + str( + test_path / "test_testing.py" + ), # CppException osgeo::proj::io::ParsingException: unrecognized format / unknown name + str(test_path / "test_array.py"), # libc.so required + # These tests passes, but disabled because they takes too long to run in CI. + str(test_path / "test_plotting.py"), + str(test_path / "test_datasets.py"), + str(test_path / "test_extension_array.py"), + str(test_path / "test_crs.py"), + str(test_path / "test_testing.py"), + str(test_path / "test_merge.py"), + str(test_path / "test_explore.py"), + ], + ) diff --git a/packages/libproj/meta.yaml b/packages/libproj/meta.yaml index c073a4ecb..404a9a174 100644 --- a/packages/libproj/meta.yaml +++ b/packages/libproj/meta.yaml @@ -13,9 +13,13 @@ requirements: build: library: true script: | - mkdir -p build; + mkdir -p build + + embuilder build zlib --pic + embuilder build sqlite3 --pic + cd build \ - && LDFLAGS="-s NODERAWFS=1 -sUSE_ZLIB=1 -sUSE_SQLITE3 -sFORCE_FILESYSTEM=1" emcmake cmake ../ \ + && LDFLAGS="-s NODERAWFS=1 -sFORCE_FILESYSTEM=1" emcmake cmake ../ \ -DCMAKE_INSTALL_PREFIX=$WASM_LIBRARY_DIR \ -DCMAKE_BUILD_TYPE=Release \ -DENABLE_CURL=OFF \ diff --git a/packages/munch/meta.yaml b/packages/munch/meta.yaml new file mode 100644 index 000000000..2142b9577 --- /dev/null +++ b/packages/munch/meta.yaml @@ -0,0 +1,18 @@ +package: + name: munch + version: 2.5.0 +source: + url: https://files.pythonhosted.org/packages/cc/ab/85d8da5c9a45e072301beb37ad7f833cd344e04c817d97e0cc75681d248f/munch-2.5.0-py2.py3-none-any.whl + sha256: 6f44af89a2ce4ed04ff8de41f70b226b984db10a91dcc7b9ac2efc1c77022fdd +requirements: + run: + - setuptools + - six +test: + imports: + - munch +about: + home: https://github.com/Infinidat/munch + PyPI: https://pypi.org/project/munch + summary: A dot-accessible dictionary (a la JavaScript objects) + license: MIT diff --git a/packages/pyproj/meta.yaml b/packages/pyproj/meta.yaml index 0e0073eda..33ed13558 100644 --- a/packages/pyproj/meta.yaml +++ b/packages/pyproj/meta.yaml @@ -21,8 +21,11 @@ build: export PROJ_INCDIR=${WASM_LIBRARY_DIR}/include export PROJ_LIBDIR=${WASM_LIBRARY_DIR}/lib export PROJ_WHEEL=1 - mkdir -p pyproj/proj_dir - cp -r ${WASM_LIBRARY_DIR}/share pyproj/proj_dir + mkdir -p pyproj/proj_dir/share + cp -r ${WASM_LIBRARY_DIR}/share/proj pyproj/proj_dir/share +test: + imports: + - pyproj about: home: https://github.com/pyproj4/pyproj diff --git a/tools/cmake/Modules/Platform/Emscripten.cmake b/tools/cmake/Modules/Platform/Emscripten.cmake index 232300078..8917df6ac 100644 --- a/tools/cmake/Modules/Platform/Emscripten.cmake +++ b/tools/cmake/Modules/Platform/Emscripten.cmake @@ -42,6 +42,21 @@ if (NOT "$ENV{WASM_LIBRARY_DIR}" STREQUAL "") endif() endif() +# Note: Emscripten installs libraries into subdirectories such as: +# - Non PIC: /lib// +# - PIC: /lib//pic/ +# - LTO: /lib//lto/ +# - PIC+LTO: /lib//pic/lto/ +# We always wants to use a library built with "-fPIC", but +# CMake's find_library() will search Non-PIC dir only by default. +# This is a hack which overrides find_library() to tell CMake to look at PIC dirs first. +if ($ENV{CFLAGS} MATCHES "MEMORY64") + set(CMAKE_LIBRARY_ARCHITECTURE "wasm64-emscripten/pic") +else() + set(CMAKE_LIBRARY_ARCHITECTURE "wasm32-emscripten/pic") +endif() + + # Disable the usage of response file so objects are exposed to the commandline. # Our export calculation logic in pywasmcross needs to read object files. # TODO: support export calculation from the response file