PKG Add geopandas, fiona, and GDAL (#3213)

2022-11-07 13:44:23 +09:00 · 2022-11-07 13:44:23 +09:00 · 6bfb9ba329
parent d99c1fca0e
commit 6bfb9ba329
14 changed files with 378 additions and 5 deletions
--- a/docs/project/changelog.md
+++ b/docs/project/changelog.md
@ -116,6 +116,7 @@ substitutions:
 - New packages: pycryptodomex {pr}`2966`, pycryptodome {pr}`2965`,
  coverage-py {pr}`3053`, bcrypt {pr}`3125`, lightgbm {pr}`3138`,
  pyheif, pillow_heif, libheif, libde265 {pr}`3161`, wordcloud {pr}`3173`,
+  gdal, fiona, geopandas {pr}`3213`,
  the standard library \_hashlib module {pr}`3206`

 - {{ Breaking }} Unvendored the sqlite3 module from the standard library.
--- a/packages/_tests/test_packages_common.py
+++ b/packages/_tests/test_packages_common.py
@ -49,7 +49,7 @@ def test_parse_package(name: str) -> None:


@pytest.mark.skip_refcount_check
-@pytest.mark.driver_timeout(60)
+@pytest.mark.driver_timeout(120)
@pytest.mark.parametrize("name", registered_packages())
@pytest.mark.benchmark(
    max_time=3,
--- a/packages/click/meta.yaml
+++ b/packages/click/meta.yaml
@ -0,0 +1,14 @@
+package:
+  name: click
+  version: 8.1.3
+source:
+  url: https://files.pythonhosted.org/packages/c2/f1/df59e28c642d583f7dacffb1e0965d0e00b218e0186d7858ac5233dce840/click-8.1.3-py3-none-any.whl
+  sha256: bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48
+test:
+  imports:
+    - click
+about:
+  home: https://palletsprojects.com/p/click/
+  PyPI: https://pypi.org/project/click
+  summary: Composable command line interface toolkit
+  license: BSD-3-Clause
--- a/packages/cligj/meta.yaml
+++ b/packages/cligj/meta.yaml
@ -0,0 +1,17 @@
+package:
+  name: cligj
+  version: 0.7.2
+source:
+  url: https://files.pythonhosted.org/packages/73/86/43fa9f15c5b9fb6e82620428827cd3c284aa933431405d1bcf5231ae3d3e/cligj-0.7.2-py3-none-any.whl
+  sha256: c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df
+test:
+  imports:
+    - cligj
+requirements:
+  run:
+    - click
+about:
+  home: https://github.com/mapbox/cligj
+  PyPI: https://pypi.org/project/cligj
+  summary: Click params for command line interfaces to GeoJSON
+  license: BSD
--- a/packages/fiona/meta.yaml
+++ b/packages/fiona/meta.yaml
@ -0,0 +1,40 @@
+package:
+  name: fiona
+  version: 1.8.21
+source:
+  url: https://files.pythonhosted.org/packages/67/5c/4e028e84a1f0cb3f8a994217cf2366360ca984dfc1433f6171de527d0dca/Fiona-1.8.21.tar.gz
+  sha256: 3a0edca2a7a070db405d71187214a43d2333a57b4097544a3fcc282066a58bfc
+test:
+  imports:
+    - fiona
+    - fiona.fio
+requirements:
+  host:
+    - gdal
+    - geos
+  run:
+    - gdal
+    - geos
+    - attrs
+    - certifi
+    - setuptools
+    - six
+    - click
+    - cligj
+    # - click-plugins  # only for fiona CLI?
+
+build:
+  script: |
+    export PACKAGE_DATA=1
+    export GDAL_CONFIG=${WASM_LIBRARY_DIR}/bin/gdal-config
+    export GDAL_DATA=${WASM_LIBRARY_DIR}/share/gdal
+    export PROJ_LIB=${WASM_LIBRARY_DIR}/share/proj
+    echo ${GDAL_CONFIG}
+    echo ${GDAL_DATA}
+    echo ${PROJ_LIB}
+
+about:
+  home: http://github.com/Toblerity/Fiona
+  PyPI: https://pypi.org/project/fiona
+  summary: Fiona reads and writes spatial data files
+  license: BSD
--- a/packages/fiona/test_data/fiona-tests-1.8.21.zip
+++ b/packages/fiona/test_data/fiona-tests-1.8.21.zip
--- a/packages/fiona/test_fiona.py
+++ b/packages/fiona/test_fiona.py
@ -0,0 +1,75 @@
+import pathlib
+
+import pytest
+from pytest_pyodide import run_in_pyodide
+
+TEST_DATA_PATH = pathlib.Path(__file__).parent / "test_data"
+
+
+@pytest.mark.driver_timeout(60)
+@run_in_pyodide(packages=["fiona"])
+def test_supported_drivers(selenium):
+    import fiona
+
+    assert fiona.driver_count() > 0
+
+
+@pytest.mark.driver_timeout(60)
+def test_runtest(selenium):
+    @run_in_pyodide(packages=["fiona", "pytest"])
+    def _run(selenium, data):
+        import zipfile
+
+        with open("tests.zip", "wb") as f:
+            f.write(data)
+
+        with zipfile.ZipFile("tests.zip", "r") as zf:
+            zf.extractall("tests")
+
+        import sys
+
+        sys.path.append("tests")
+
+        import pytest
+
+        def runtest(test_filter, ignore_filters):
+
+            ignore_filter = []
+            for ignore in ignore_filters:
+                ignore_filter.append("--ignore-glob")
+                ignore_filter.append(ignore)
+
+            ret = pytest.main(
+                [
+                    "--pyargs",
+                    "tests",
+                    "--continue-on-collection-errors",
+                    # "-v",
+                    *ignore_filter,
+                    "-k",
+                    test_filter,
+                ]
+            )
+            assert ret == 0
+
+        runtest(
+            (
+                "not ordering "  # hangs
+                "and not env "  # No module named "boto3"
+                "and not slice "  # GML file format not supported
+                "and not GML "  # GML file format not supported
+                "and not TestNonCountingLayer "  # GPX file format not supported
+                "and not test_schema_default_fields_wrong_type "  # GPX file format not supported
+                "and not http "
+                "and not FlatGeobuf"  # assertion error
+            ),
+            [
+                "tests/test_fio*",  # no CLI tests
+                "tests/test_data_paths.py",  # no CLI tests
+                "tests/test_datetime.py",  # no CLI tests
+                "tests/test_vfs.py",  # No module named "boto3"
+            ],
+        )
+
+    TEST_DATA = (TEST_DATA_PATH / "fiona-tests-1.8.21.zip").read_bytes()
+    _run(selenium, TEST_DATA)
--- a/packages/gdal/meta.yaml
+++ b/packages/gdal/meta.yaml
@ -0,0 +1,102 @@
+package:
+  name: gdal
+  version: 3.5.1
+
+source:
+  url: https://github.com/OSGeo/gdal/releases/download/v3.5.2/gdal-3.5.2.tar.gz
+  sha256: fbd696e1b2a858fbd2eb3718db16b14ed9ba82521d3578770d480c74fe1146d2
+requirements:
+  host:
+    - libtiff
+    - libproj
+    - libiconv
+    - geos
+  run:
+    - geos
+
+build:
+  sharedlibrary: true
+  script: |
+    export EMSCRIPTEN_SYSROOT=$(em-config CACHE)/sysroot
+    export EMSCRIPTEN_INCLUDE=$EMSCRIPTEN_SYSROOT/include
+    export EMSCRIPTEN_BIN=$EMSCRIPTEN_SYSROOT/bin
+    export EMSCRIPTEN_LIB=$EMSCRIPTEN_SYSROOT/lib/wasm32-emscripten/pic
+
+    mkdir -p dist
+    export DISTDIR=$(pwd)/dist
+
+    embuilder build zlib --pic
+    embuilder build libjpeg --pic
+    embuilder build libpng --pic
+    embuilder build sqlite3 --pic
+
+    mkdir -p build
+    cd build && emcmake cmake .. \
+      -DCMAKE_INSTALL_PREFIX=$WASM_LIBRARY_DIR \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DBUILD_SHARED_LIBS=True \
+      -DBUILD_APPS=OFF \
+      -DCMAKE_C_FLAGS="${SIDE_MODULE_CFLAGS} -Wno-deprecated-declarations -Wno-single-bit-bitfield-constant-conversion" \
+      -DCMAKE_CXX_FLAGS="${SIDE_MODULE_CFLAGS} -Wno-deprecated-declarations -Wno-single-bit-bitfield-constant-conversion" \
+      -DCMAKE_SHARED_LINKER_FLAGS="-sSIDE_MODULE=1 -sWASM_BIGINT" \
+      -DGDAL_USE_EXTERNAL_LIBS=OFF \
+      -DGDAL_USE_INTERNAL_LIBS=OFF \
+      \
+      -DPROJ_INCLUDE_DIR=$WASM_LIBRARY_DIR/include \
+      -DPROJ_LIBRARY=$WASM_LIBRARY_DIR/lib/libproj.a \
+      \
+      -DGDAL_USE_ICONV=ON \
+      -DIconv_INCLUDE_DIR=$WASM_LIBRARY_DIR/include \
+      -DIconv_LIBRARY=$WASM_LIBRARY_DIR/lib/libiconv.a \
+      \
+      -DGDAL_USE_TIFF=ON \
+      -DTIFF_INCLUDE_DIR=$WASM_LIBRARY_DIR/include \
+      -DTIFF_LIBRARY=$WASM_LIBRARY_DIR/lib/libtiff.a \
+      \
+      -DGDAL_USE_GEOS=ON \
+      -DGEOS_INCLUDE_DIR=$WASM_LIBRARY_DIR/include \
+      -DGEOS_LIBRARY=$WASM_LIBRARY_DIR/lib/libgeos.so \
+      \
+      -DGDAL_USE_ZLIB=ON \
+      -DZLIB_INCLUDE_DIR=$EMSCRIPTEN_INCLUDE \
+      -DZLIB_LIBRARY=$EMSCRIPTEN_LIB/libz.a \
+      \
+      -DGDAL_USE_PNG=ON \
+      -DPNG_PNG_INCLUDE_DIR=$EMSCRIPTEN_INCLUDE \
+      -DPNG_LIBRARY_RELEASE=$EMSCRIPTEN_LIB/libpng.a \
+      \
+      -DGDAL_USE_JPEG=ON \
+      -DJPEG_INCLUDE_DIR=$EMSCRIPTEN_INCLUDE \
+      -DJPEG_LIBRARY_RELEASE=$EMSCRIPTEN_LIB/libjpeg.a \
+      \
+      -DGDAL_USE_SQLITE3=ON \
+      -DSQLite3_INCLUDE_DIR=$EMSCRIPTEN_INCLUDE \
+      -DSQLite3_LIBRARY=$EMSCRIPTEN_LIB/libsqlite3.a \
+      \
+      -DGDAL_USE_GEOTIFF_INTERNAL=ON \
+      -DGDAL_USE_QHULL_INTERNAL=ON \
+      -DGDAL_USE_LERC_INTERNAL=ON \
+      -DGDAL_USE_JSONC_INTERNAL=ON \
+      -DGDAL_USE_PCRE2=OFF \
+      -DBUILD_TESTING=OFF
+
+
+    # Note:
+    # CMake tries to link the same libraries multiple times at the final link time.
+    # This is probably because the same libraries are used in multiple submodules.
+    # This behavior is okay when those libraries are "shared" libraries,
+    # but we often build static libraries and linking static libraries
+    # multiple times results in a duplicated symbol error.
+    # I wasn't able to find a way to prevent CMake from emitting duplicated libraries.
+    # This is a hack which removes all duplicated appearances of static libs at the final link time.
+
+    export LINKLIBS=$(pwd)/CMakeFiles/GDAL.dir/link.txt
+    cat ${LINKLIBS} | grep -o '\S*' | grep "\.a$" | sort | uniq | tr "\n" " " > linked_static_libs.txt
+    cat ${LINKLIBS} | grep -o '\S*' | grep -v "\.a$" | tr "\n" " " > link_cmd.txt
+    cat link_cmd.txt > ${LINKLIBS}
+    cat linked_static_libs.txt >> ${LINKLIBS}
+
+    emmake make -j ${PYODIDE_JOBS:-3}
+    emmake make install
+
+    cp ${WASM_LIBRARY_DIR}/lib/libgdal.so ${DISTDIR}
--- a/packages/geopandas/meta.yaml
+++ b/packages/geopandas/meta.yaml
@ -0,0 +1,21 @@
+package:
+  name: geopandas
+  version: 0.11.1
+  top-level:
+    - geopandas
+source:
+  url: https://files.pythonhosted.org/packages/de/83/08b7284066b268e71019398b9094b1c860816aa9b9422f1c489459bfbdf1/geopandas-0.11.1-py3-none-any.whl
+  sha256: f3344937f3866e52996c7e505d56dae78be117dc840cd1c23507da0b33c0af71
+requirements:
+  run:
+    - shapely
+    - fiona
+    - pyproj
+    - packaging
+    - pandas
+    # - matplotlib # matplotlib is not a hard dependency
+about:
+  home: http://geopandas.org
+  PyPI: https://pypi.org/project/geopandas
+  summary: Geographic pandas extensions
+  license: BSD
--- a/packages/geopandas/test_geopandas.py
+++ b/packages/geopandas/test_geopandas.py
@ -0,0 +1,63 @@
+import pytest
+from pytest_pyodide import run_in_pyodide
+
+
+@pytest.mark.driver_timeout(300)
+@run_in_pyodide(packages=["geopandas", "geopandas-tests", "pytest"])
+def test_runtest(selenium):
+    from pathlib import Path
+
+    import geopandas
+    import pytest
+
+    test_path = Path(geopandas.__file__).parent / "tests"
+
+    def runtest(test_filter, ignore_filters):
+
+        ignore_filter = []
+        for ignore in ignore_filters:
+            ignore_filter.append("--ignore-glob")
+            ignore_filter.append(ignore)
+
+        ret = pytest.main(
+            [
+                "--pyargs",
+                str(test_path),
+                "--continue-on-collection-errors",
+                # "-v",
+                *ignore_filter,
+                "-k",
+                test_filter,
+                # "--durations",
+                # "20",
+            ]
+        )
+        assert ret == 0
+
+    runtest(
+        (
+            "not test_transform2 "  # CppException std::invalid_argument: non double value
+            "and not test_no_additional_imports "  # subprocess
+            "and not test_pandas_kind "  # scipy required
+        ),
+        [
+            str(
+                test_path / "test_dissolve.py"
+            ),  # CppException osgeo::proj::io::ParsingException: unrecognized format / unknown name
+            str(
+                test_path / "test_geodataframe.py"
+            ),  # CppException osgeo::proj::io::ParsingException: unrecognized format / unknown name
+            str(
+                test_path / "test_testing.py"
+            ),  # CppException osgeo::proj::io::ParsingException: unrecognized format / unknown name
+            str(test_path / "test_array.py"),  # libc.so required
+            # These tests passes, but disabled because they takes too long to run in CI.
+            str(test_path / "test_plotting.py"),
+            str(test_path / "test_datasets.py"),
+            str(test_path / "test_extension_array.py"),
+            str(test_path / "test_crs.py"),
+            str(test_path / "test_testing.py"),
+            str(test_path / "test_merge.py"),
+            str(test_path / "test_explore.py"),
+        ],
+    )
--- a/packages/libproj/meta.yaml
+++ b/packages/libproj/meta.yaml
@ -13,9 +13,13 @@ requirements:
 build:
  library: true
  script: |
-    mkdir -p build;
+    mkdir -p build
+
+    embuilder build zlib --pic
+    embuilder build sqlite3 --pic
+
    cd build \
-        && LDFLAGS="-s NODERAWFS=1 -sUSE_ZLIB=1 -sUSE_SQLITE3 -sFORCE_FILESYSTEM=1" emcmake cmake ../ \
+        && LDFLAGS="-s NODERAWFS=1 -sFORCE_FILESYSTEM=1" emcmake cmake ../ \
        -DCMAKE_INSTALL_PREFIX=$WASM_LIBRARY_DIR \
        -DCMAKE_BUILD_TYPE=Release \
        -DENABLE_CURL=OFF \
--- a/packages/munch/meta.yaml
+++ b/packages/munch/meta.yaml
@ -0,0 +1,18 @@
+package:
+  name: munch
+  version: 2.5.0
+source:
+  url: https://files.pythonhosted.org/packages/cc/ab/85d8da5c9a45e072301beb37ad7f833cd344e04c817d97e0cc75681d248f/munch-2.5.0-py2.py3-none-any.whl
+  sha256: 6f44af89a2ce4ed04ff8de41f70b226b984db10a91dcc7b9ac2efc1c77022fdd
+requirements:
+  run:
+    - setuptools
+    - six
+test:
+  imports:
+    - munch
+about:
+  home: https://github.com/Infinidat/munch
+  PyPI: https://pypi.org/project/munch
+  summary: A dot-accessible dictionary (a la JavaScript objects)
+  license: MIT
--- a/packages/pyproj/meta.yaml
+++ b/packages/pyproj/meta.yaml
@ -21,8 +21,11 @@ build:
    export PROJ_INCDIR=${WASM_LIBRARY_DIR}/include
    export PROJ_LIBDIR=${WASM_LIBRARY_DIR}/lib
    export PROJ_WHEEL=1
-    mkdir -p pyproj/proj_dir
-    cp -r ${WASM_LIBRARY_DIR}/share pyproj/proj_dir
+    mkdir -p pyproj/proj_dir/share
+    cp -r ${WASM_LIBRARY_DIR}/share/proj pyproj/proj_dir/share
+test:
+  imports:
+    - pyproj

 about:
  home: https://github.com/pyproj4/pyproj
--- a/tools/cmake/Modules/Platform/Emscripten.cmake
+++ b/tools/cmake/Modules/Platform/Emscripten.cmake
@ -42,6 +42,21 @@ if (NOT "$ENV{WASM_LIBRARY_DIR}" STREQUAL "")
  endif()
 endif()

+# Note: Emscripten installs libraries into subdirectories such as:
+# - Non PIC: <SYSROOT>/lib/<arch>/<lib>
+# - PIC: <SYSROOT>/lib/<arch>/pic/<lib>
+# - LTO: <SYSROOT>/lib/<arch>/lto/<lib>
+# - PIC+LTO: <SYSROOT>/lib/<arch>/pic/lto/<lib>
+# We always wants to use a library built with "-fPIC", but
+# CMake's find_library() will search Non-PIC dir only by default.
+# This is a hack which overrides find_library() to tell CMake to look at PIC dirs first.
+if ($ENV{CFLAGS} MATCHES "MEMORY64")
+  set(CMAKE_LIBRARY_ARCHITECTURE "wasm64-emscripten/pic")
+else()
+  set(CMAKE_LIBRARY_ARCHITECTURE "wasm32-emscripten/pic")
+endif()
+
+
 # Disable the usage of response file so objects are exposed to the commandline.
 # Our export calculation logic in pywasmcross needs to read object files.
 # TODO: support export calculation from the response file