From 994ce5111f25a5cc1f0d024c519a5308f9be8505 Mon Sep 17 00:00:00 2001 From: Gyeongjae Choi Date: Fri, 17 Feb 2023 08:25:42 +0900 Subject: [PATCH] BLD Add DISTDIR variable (#3584) --- docs/development/meta-yaml.md | 8 +++++--- packages/CLAPACK/meta.yaml | 5 ++--- packages/cpp-exceptions-test/meta.yaml | 5 ++--- packages/distutils/meta.yaml | 2 -- packages/gdal/meta.yaml | 3 --- packages/hashlib/meta.yaml | 2 -- packages/libhdf5/meta.yaml | 3 --- packages/libheif/meta.yaml | 3 +-- packages/libmagic/meta.yaml | 3 +-- packages/lzma/meta.yaml | 2 -- packages/openssl/meta.yaml | 3 +-- packages/pydecimal/meta.yaml | 2 -- packages/pydoc_data/meta.yaml | 2 -- packages/sharedlib-test/meta.yaml | 7 +++---- packages/sqlite3/meta.yaml | 2 -- packages/ssl/meta.yaml | 2 -- packages/suitesparse/meta.yaml | 3 +-- packages/test/meta.yaml | 2 -- pyodide-build/pyodide_build/buildpkg.py | 2 ++ 19 files changed, 18 insertions(+), 43 deletions(-) diff --git a/docs/development/meta-yaml.md b/docs/development/meta-yaml.md index cebed57a7..0a65b53b5 100644 --- a/docs/development/meta-yaml.md +++ b/docs/development/meta-yaml.md @@ -159,9 +159,8 @@ The difference between `static_library` and `shared_library` is that `static_library` is statically linked into the other packages, so it is required only in the build time, while `shared_library` is dynamically linked, so it is required in the runtime. When building -a shared library, you should copy the built libraries into the subfolder -of the source folder called `dist`. Files or folders in this folder will -be packaged to make the Pyodide package. +a shared library, you should copy the built libraries into the `$DISTDIR`. +Files or folders in this folder will be packaged to make the Pyodide package. See the [zlib meta.yaml](https://github.com/pyodide/pyodide/blob/main/packages/zlib/meta.yaml) @@ -182,6 +181,9 @@ There are special environment variables defined: - `$PKGDIR`: The directory in which the `meta.yaml` file resides. - `$PKG_VESRION`: The version of the package - `$PKG_BUILD_DIR`: The directory where the tarball was extracted. +- `$DISTDIR`: The directory where the built wheel or library should be placed. + If you are building a shared library, you should copy the built libraries into this + directory. (These keys are not in the Conda spec). diff --git a/packages/CLAPACK/meta.yaml b/packages/CLAPACK/meta.yaml index cdabe9430..14e05d2e5 100644 --- a/packages/CLAPACK/meta.yaml +++ b/packages/CLAPACK/meta.yaml @@ -33,8 +33,7 @@ build: sed -i 's/^ ld /^ $(LD)/' **/Makefile emmake make -j ${PYODIDE_JOBS:-3} blaslib lapacklib - mkdir -p dist - emcc blas_WA.a lapack_WA.a F2CLIBS/libf2c.a ${SIDE_MODULE_LDFLAGS} -o dist/clapack_all.so + emcc blas_WA.a lapack_WA.a F2CLIBS/libf2c.a ${SIDE_MODULE_LDFLAGS} -o ${DISTDIR}/clapack_all.so mkdir -p ${WASM_LIBRARY_DIR}/{lib,include} cp -r INCLUDE/* ${WASM_LIBRARY_DIR}/include - cp dist/clapack_all.so ${WASM_LIBRARY_DIR}/lib + cp ${DISTDIR}/clapack_all.so ${WASM_LIBRARY_DIR}/lib diff --git a/packages/cpp-exceptions-test/meta.yaml b/packages/cpp-exceptions-test/meta.yaml index 18f888b77..150c7ca8b 100644 --- a/packages/cpp-exceptions-test/meta.yaml +++ b/packages/cpp-exceptions-test/meta.yaml @@ -12,6 +12,5 @@ build: script: | em++ -c throw.cpp -o throw.o ${SIDE_MODULE_CFLAGS} -fexceptions em++ -c catch.cpp -o catch.o ${SIDE_MODULE_CFLAGS} -fexceptions - mkdir dist - em++ throw.o ${SIDE_MODULE_LDFLAGS} -o dist/cpp-exceptions-test-throw.so -fexceptions - em++ catch.o ${SIDE_MODULE_LDFLAGS} -o dist/cpp-exceptions-test-catch.so -fexceptions + em++ throw.o ${SIDE_MODULE_LDFLAGS} -o ${DISTDIR}/cpp-exceptions-test-throw.so -fexceptions + em++ catch.o ${SIDE_MODULE_LDFLAGS} -o ${DISTDIR}/cpp-exceptions-test-catch.so -fexceptions diff --git a/packages/distutils/meta.yaml b/packages/distutils/meta.yaml index ad77d480f..664b11eb4 100644 --- a/packages/distutils/meta.yaml +++ b/packages/distutils/meta.yaml @@ -8,8 +8,6 @@ package: build: type: cpython_module script: | - mkdir dist - export DISTDIR=$(pwd)/dist cd $CPYTHONLIB tar --exclude=__pycache__ -cf - distutils | tar -C $DISTDIR -xf - diff --git a/packages/gdal/meta.yaml b/packages/gdal/meta.yaml index a9b70071d..4afed0279 100644 --- a/packages/gdal/meta.yaml +++ b/packages/gdal/meta.yaml @@ -22,9 +22,6 @@ build: export EMSCRIPTEN_BIN=$EMSCRIPTEN_SYSROOT/bin export EMSCRIPTEN_LIB=$EMSCRIPTEN_SYSROOT/lib/wasm32-emscripten/pic - mkdir -p dist - export DISTDIR=$(pwd)/dist - embuilder build zlib --pic embuilder build libjpeg --pic embuilder build libpng --pic diff --git a/packages/hashlib/meta.yaml b/packages/hashlib/meta.yaml index 3963304e5..dac2e937c 100644 --- a/packages/hashlib/meta.yaml +++ b/packages/hashlib/meta.yaml @@ -8,8 +8,6 @@ package: build: type: cpython_module script: | - mkdir dist - export DISTDIR=$(pwd)/dist cd $CPYTHONBUILD emcc $STDLIB_MODULE_CFLAGS -c Modules/_hashopenssl.c -o Modules/_hashlib.o \ diff --git a/packages/libhdf5/meta.yaml b/packages/libhdf5/meta.yaml index 0e89e172b..597ce9611 100644 --- a/packages/libhdf5/meta.yaml +++ b/packages/libhdf5/meta.yaml @@ -15,9 +15,6 @@ requirements: build: type: shared_library script: | - mkdir -p dist - export DISTDIR=$(pwd)/dist - mkdir -p build; cd build \ diff --git a/packages/libheif/meta.yaml b/packages/libheif/meta.yaml index d4a80f4c3..8cf8e34ae 100644 --- a/packages/libheif/meta.yaml +++ b/packages/libheif/meta.yaml @@ -27,5 +27,4 @@ build: emmake make -j ${PYODIDE_JOBS:-3} emmake make install - mkdir -p dist - cp ${WASM_LIBRARY_DIR}/lib/libheif.so dist/ + cp ${WASM_LIBRARY_DIR}/lib/libheif.so ${DISTDIR} diff --git a/packages/libmagic/meta.yaml b/packages/libmagic/meta.yaml index 7bd8f6bd0..4acf1d949 100644 --- a/packages/libmagic/meta.yaml +++ b/packages/libmagic/meta.yaml @@ -12,7 +12,6 @@ source: build: type: shared_library script: | - mkdir -p dist autoreconf --install # build magic.mgc natively ./configure @@ -24,4 +23,4 @@ build: cd src emmake make -j ${PYODIDE_JOBS:-3} libmagic.la \ LDFLAGS="-Xcompiler '${SIDE_MODULE_LDFLAGS}'" - cp .libs/libmagic.so.1.0.0 ../dist/libmagic.so + cp .libs/libmagic.so.1.0.0 ${DISTDIR}/libmagic.so diff --git a/packages/lzma/meta.yaml b/packages/lzma/meta.yaml index af05fcbc9..f1477f14a 100644 --- a/packages/lzma/meta.yaml +++ b/packages/lzma/meta.yaml @@ -9,8 +9,6 @@ package: build: type: cpython_module script: | - mkdir dist - export DISTDIR=$(pwd)/dist cd $CPYTHONBUILD emcc $STDLIB_MODULE_CFLAGS -c Modules/_lzmamodule.c -o Modules/_lzmamodule.o \ $(pkg-config --cflags --dont-define-prefix liblzma) diff --git a/packages/openssl/meta.yaml b/packages/openssl/meta.yaml index 0ebdcb3d1..bb50db45e 100644 --- a/packages/openssl/meta.yaml +++ b/packages/openssl/meta.yaml @@ -23,8 +23,7 @@ build: make -j ${PYODIDE_JOBS:-3} libcrypto.a make -j ${PYODIDE_JOBS:-3} libssl.a emar -d libcrypto.a liblegacy-lib-bn_asm.o liblegacy-lib-des_enc.o liblegacy-lib-fcrypt_b.o - mkdir dist emcc ${SIDE_MODULE_LDFLAGS} libcrypto.a -o libcrypto.so emcc ${SIDE_MODULE_LDFLAGS} libssl.a -o libssl.so make install_sw - cp libcrypto.so libssl.so dist + cp libcrypto.so libssl.so ${DISTDIR} diff --git a/packages/pydecimal/meta.yaml b/packages/pydecimal/meta.yaml index 5b2deb61f..5e09dedf8 100644 --- a/packages/pydecimal/meta.yaml +++ b/packages/pydecimal/meta.yaml @@ -8,8 +8,6 @@ package: build: type: cpython_module script: | - mkdir dist - export DISTDIR=$(pwd)/dist cd $CPYTHONBUILD cp Lib/_pydecimal.py $DISTDIR diff --git a/packages/pydoc_data/meta.yaml b/packages/pydoc_data/meta.yaml index cd4349081..2a2c60e3c 100644 --- a/packages/pydoc_data/meta.yaml +++ b/packages/pydoc_data/meta.yaml @@ -8,8 +8,6 @@ package: build: type: cpython_module script: | - mkdir dist - export DISTDIR=$(pwd)/dist cd $CPYTHONBUILD/Lib tar --exclude=__pycache__ -cf - pydoc_data | tar -C $DISTDIR -xf - diff --git a/packages/sharedlib-test/meta.yaml b/packages/sharedlib-test/meta.yaml index 712c8bf33..4f8dc78c7 100644 --- a/packages/sharedlib-test/meta.yaml +++ b/packages/sharedlib-test/meta.yaml @@ -14,7 +14,6 @@ build: emcc -c dep.c -o dep.o ${SIDE_MODULE_CFLAGS} emcc -c dep2.c -o dep2.o ${SIDE_MODULE_CFLAGS} - mkdir -p dist - emcc dep2.o ${SIDE_MODULE_LDFLAGS} -o dist/sharedlib-test-dep2.so - emcc dep.o ${SIDE_MODULE_LDFLAGS} dist/sharedlib-test-dep2.so -o dist/sharedlib-test-dep.so - emcc main.o ${SIDE_MODULE_LDFLAGS} dist/sharedlib-test-dep.so -o dist/sharedlib-test.so + emcc dep2.o ${SIDE_MODULE_LDFLAGS} -o ${DISTDIR}/sharedlib-test-dep2.so + emcc dep.o ${SIDE_MODULE_LDFLAGS} ${DISTDIR}/sharedlib-test-dep2.so -o ${DISTDIR}/sharedlib-test-dep.so + emcc main.o ${SIDE_MODULE_LDFLAGS} ${DISTDIR}/sharedlib-test-dep.so -o ${DISTDIR}/sharedlib-test.so diff --git a/packages/sqlite3/meta.yaml b/packages/sqlite3/meta.yaml index 75b2b67ca..7dc5aba8a 100644 --- a/packages/sqlite3/meta.yaml +++ b/packages/sqlite3/meta.yaml @@ -9,8 +9,6 @@ package: build: type: cpython_module script: | - mkdir dist - export DISTDIR=$(pwd)/dist cd $CPYTHONBUILD export FILES=( diff --git a/packages/ssl/meta.yaml b/packages/ssl/meta.yaml index b5b4c8aeb..7964a570a 100644 --- a/packages/ssl/meta.yaml +++ b/packages/ssl/meta.yaml @@ -9,8 +9,6 @@ package: build: type: cpython_module script: | - mkdir dist - export DISTDIR=$(pwd)/dist cd $CPYTHONBUILD emcc $STDLIB_MODULE_CFLAGS -c Modules/_ssl.c -o Modules/_ssl.o \ $(pkg-config --cflags --dont-define-prefix openssl) \ diff --git a/packages/suitesparse/meta.yaml b/packages/suitesparse/meta.yaml index f0e59b15d..21e78ab74 100644 --- a/packages/suitesparse/meta.yaml +++ b/packages/suitesparse/meta.yaml @@ -26,7 +26,6 @@ build: BLAS="${WASM_LIBRARY_DIR}/lib/clapack_all.so" \ LAPACK="${WASM_LIBRARY_DIR}/lib/clapack_all.so" \ INSTALL=${WASM_LIBRARY_DIR} - mkdir -p dist cp ${WASM_LIBRARY_DIR}/lib/libsuitesparseconfig.so \ ${WASM_LIBRARY_DIR}/lib/libamd.so \ @@ -36,4 +35,4 @@ build: ${WASM_LIBRARY_DIR}/lib/libmetis.so \ ${WASM_LIBRARY_DIR}/lib/libcholmod.so \ ${WASM_LIBRARY_DIR}/lib/libspqr.so \ - dist/ + ${DISTDIR} diff --git a/packages/test/meta.yaml b/packages/test/meta.yaml index c06f247ae..45df00ca9 100644 --- a/packages/test/meta.yaml +++ b/packages/test/meta.yaml @@ -10,8 +10,6 @@ package: build: type: cpython_module script: | - mkdir dist - export DISTDIR=$(pwd)/dist cd $CPYTHONBUILD export TEST_EXTENSIONS="\ diff --git a/pyodide-build/pyodide_build/buildpkg.py b/pyodide-build/pyodide_build/buildpkg.py index 06887e103..59512f18c 100755 --- a/pyodide-build/pyodide_build/buildpkg.py +++ b/pyodide-build/pyodide_build/buildpkg.py @@ -772,11 +772,13 @@ def _build_package_inner( bash_runner.env["PKGDIR"] = str(pkg_root) bash_runner.env["PKG_VERSION"] = version bash_runner.env["PKG_BUILD_DIR"] = str(srcpath) + bash_runner.env["DISTDIR"] = str(src_dist_dir) if not continue_: clear_only = package_type == "cpython_module" prepare_source(build_dir, srcpath, source_metadata, clear_only=clear_only) patch(pkg_root, srcpath, source_metadata) + src_dist_dir.mkdir(exist_ok=True, parents=True) run_script(build_dir, srcpath, build_metadata, bash_runner) if package_type == "static_library":