BLD Add DISTDIR variable (#3584)

docs/development/meta-yaml.md

@@ -159,9 +159,8 @@ The difference between `static_library` and `shared_library` is that
 `static_library` is statically linked into the other packages,
 so it is required only in the build time, while `shared_library` is
 dynamically linked, so it is required in the runtime. When building
-a shared library, you should copy the built libraries into the subfolder
-of the source folder called `dist`. Files or folders in this folder will
-be packaged to make the Pyodide package.
+a shared library, you should copy the built libraries into the `$DISTDIR`.
+Files or folders in this folder will be packaged to make the Pyodide package.
 
 See the [zlib
 meta.yaml](https://github.com/pyodide/pyodide/blob/main/packages/zlib/meta.yaml)
@@ -182,6 +181,9 @@ There are special environment variables defined:
 - `$PKGDIR`: The directory in which the `meta.yaml` file resides.
 - `$PKG_VESRION`: The version of the package
 - `$PKG_BUILD_DIR`: The directory where the tarball was extracted.
+- `$DISTDIR`: The directory where the built wheel or library should be placed.
+  If you are building a shared library, you should copy the built libraries into this
+  directory.
 
 (These keys are not in the Conda spec).
 

packages/CLAPACK/meta.yaml

@@ -33,8 +33,7 @@ build:
     sed -i 's/^	ld /^	$(LD)/' **/Makefile
 
     emmake make -j ${PYODIDE_JOBS:-3} blaslib lapacklib
-    mkdir -p dist
-    emcc blas_WA.a lapack_WA.a F2CLIBS/libf2c.a ${SIDE_MODULE_LDFLAGS} -o dist/clapack_all.so
+    emcc blas_WA.a lapack_WA.a F2CLIBS/libf2c.a ${SIDE_MODULE_LDFLAGS} -o ${DISTDIR}/clapack_all.so
     mkdir -p ${WASM_LIBRARY_DIR}/{lib,include}
     cp -r INCLUDE/* ${WASM_LIBRARY_DIR}/include
-    cp dist/clapack_all.so ${WASM_LIBRARY_DIR}/lib
+    cp ${DISTDIR}/clapack_all.so ${WASM_LIBRARY_DIR}/lib

packages/cpp-exceptions-test/meta.yaml

@@ -12,6 +12,5 @@ build:
   script: |
     em++ -c throw.cpp -o throw.o ${SIDE_MODULE_CFLAGS} -fexceptions
     em++ -c catch.cpp -o catch.o ${SIDE_MODULE_CFLAGS} -fexceptions
-    mkdir dist
-    em++ throw.o ${SIDE_MODULE_LDFLAGS} -o dist/cpp-exceptions-test-throw.so -fexceptions
-    em++ catch.o ${SIDE_MODULE_LDFLAGS} -o dist/cpp-exceptions-test-catch.so -fexceptions
+    em++ throw.o ${SIDE_MODULE_LDFLAGS} -o ${DISTDIR}/cpp-exceptions-test-throw.so -fexceptions
+    em++ catch.o ${SIDE_MODULE_LDFLAGS} -o ${DISTDIR}/cpp-exceptions-test-catch.so -fexceptions

packages/distutils/meta.yaml

@@ -8,8 +8,6 @@ package:
 build:
   type: cpython_module
   script: |
-    mkdir dist
-    export DISTDIR=$(pwd)/dist
     cd $CPYTHONLIB
 
     tar --exclude=__pycache__ -cf - distutils | tar -C $DISTDIR -xf -

packages/gdal/meta.yaml

@@ -22,9 +22,6 @@ build:
     export EMSCRIPTEN_BIN=$EMSCRIPTEN_SYSROOT/bin
     export EMSCRIPTEN_LIB=$EMSCRIPTEN_SYSROOT/lib/wasm32-emscripten/pic
 
-    mkdir -p dist
-    export DISTDIR=$(pwd)/dist
-
     embuilder build zlib --pic
     embuilder build libjpeg --pic
     embuilder build libpng --pic

packages/hashlib/meta.yaml

@@ -8,8 +8,6 @@ package:
 build:
   type: cpython_module
   script: |
-    mkdir dist
-    export DISTDIR=$(pwd)/dist
     cd $CPYTHONBUILD
 
     emcc $STDLIB_MODULE_CFLAGS -c Modules/_hashopenssl.c -o Modules/_hashlib.o \

packages/libhdf5/meta.yaml

@@ -15,9 +15,6 @@ requirements:
 build:
   type: shared_library
   script: |
-    mkdir -p dist
-    export DISTDIR=$(pwd)/dist
-
     mkdir -p build;
 
     cd build \

packages/libheif/meta.yaml

@@ -27,5 +27,4 @@ build:
 
     emmake make -j ${PYODIDE_JOBS:-3}
     emmake make install
-    mkdir -p dist
-    cp ${WASM_LIBRARY_DIR}/lib/libheif.so dist/
+    cp ${WASM_LIBRARY_DIR}/lib/libheif.so ${DISTDIR}

packages/libmagic/meta.yaml

@@ -12,7 +12,6 @@ source:
 build:
   type: shared_library
   script: |
-    mkdir -p dist
     autoreconf --install
     # build magic.mgc natively
     ./configure
@@ -24,4 +23,4 @@ build:
     cd src
     emmake make -j ${PYODIDE_JOBS:-3} libmagic.la \
       LDFLAGS="-Xcompiler '${SIDE_MODULE_LDFLAGS}'"
-    cp .libs/libmagic.so.1.0.0 ../dist/libmagic.so
+    cp .libs/libmagic.so.1.0.0 ${DISTDIR}/libmagic.so

packages/lzma/meta.yaml

@@ -9,8 +9,6 @@ package:
 build:
   type: cpython_module
   script: |
-    mkdir dist
-    export DISTDIR=$(pwd)/dist
     cd $CPYTHONBUILD
     emcc $STDLIB_MODULE_CFLAGS -c Modules/_lzmamodule.c -o Modules/_lzmamodule.o \
       $(pkg-config --cflags --dont-define-prefix liblzma)

packages/openssl/meta.yaml

@@ -23,8 +23,7 @@ build:
     make -j ${PYODIDE_JOBS:-3} libcrypto.a
     make -j ${PYODIDE_JOBS:-3} libssl.a
     emar -d libcrypto.a liblegacy-lib-bn_asm.o liblegacy-lib-des_enc.o liblegacy-lib-fcrypt_b.o
-    mkdir dist
     emcc ${SIDE_MODULE_LDFLAGS} libcrypto.a -o libcrypto.so
     emcc ${SIDE_MODULE_LDFLAGS} libssl.a -o libssl.so
     make install_sw
-    cp libcrypto.so libssl.so dist
+    cp libcrypto.so libssl.so ${DISTDIR}

packages/pydecimal/meta.yaml

@@ -8,8 +8,6 @@ package:
 build:
   type: cpython_module
   script: |
-    mkdir dist
-    export DISTDIR=$(pwd)/dist
     cd $CPYTHONBUILD
 
     cp Lib/_pydecimal.py $DISTDIR

packages/pydoc_data/meta.yaml

@@ -8,8 +8,6 @@ package:
 build:
   type: cpython_module
   script: |
-    mkdir dist
-    export DISTDIR=$(pwd)/dist
     cd $CPYTHONBUILD/Lib
 
     tar --exclude=__pycache__ -cf - pydoc_data | tar -C $DISTDIR -xf -

packages/sharedlib-test/meta.yaml

@@ -14,7 +14,6 @@ build:
     emcc -c dep.c -o dep.o ${SIDE_MODULE_CFLAGS}
     emcc -c dep2.c -o dep2.o ${SIDE_MODULE_CFLAGS}
 
-    mkdir -p dist
-    emcc dep2.o ${SIDE_MODULE_LDFLAGS} -o dist/sharedlib-test-dep2.so
-    emcc dep.o ${SIDE_MODULE_LDFLAGS} dist/sharedlib-test-dep2.so -o dist/sharedlib-test-dep.so
-    emcc main.o ${SIDE_MODULE_LDFLAGS} dist/sharedlib-test-dep.so -o dist/sharedlib-test.so
+    emcc dep2.o ${SIDE_MODULE_LDFLAGS} -o ${DISTDIR}/sharedlib-test-dep2.so
+    emcc dep.o ${SIDE_MODULE_LDFLAGS} ${DISTDIR}/sharedlib-test-dep2.so -o ${DISTDIR}/sharedlib-test-dep.so
+    emcc main.o ${SIDE_MODULE_LDFLAGS} ${DISTDIR}/sharedlib-test-dep.so -o ${DISTDIR}/sharedlib-test.so

packages/sqlite3/meta.yaml

@@ -9,8 +9,6 @@ package:
 build:
   type: cpython_module
   script: |
-    mkdir dist
-    export DISTDIR=$(pwd)/dist
     cd $CPYTHONBUILD
 
     export FILES=(

packages/ssl/meta.yaml

@@ -9,8 +9,6 @@ package:
 build:
   type: cpython_module
   script: |
-    mkdir dist
-    export DISTDIR=$(pwd)/dist
     cd $CPYTHONBUILD
     emcc $STDLIB_MODULE_CFLAGS -c Modules/_ssl.c -o Modules/_ssl.o \
       $(pkg-config --cflags --dont-define-prefix openssl) \

packages/suitesparse/meta.yaml

@@ -26,7 +26,6 @@ build:
       BLAS="${WASM_LIBRARY_DIR}/lib/clapack_all.so" \
       LAPACK="${WASM_LIBRARY_DIR}/lib/clapack_all.so" \
       INSTALL=${WASM_LIBRARY_DIR}
-    mkdir -p dist
 
     cp ${WASM_LIBRARY_DIR}/lib/libsuitesparseconfig.so \
        ${WASM_LIBRARY_DIR}/lib/libamd.so \
@@ -36,4 +35,4 @@ build:
        ${WASM_LIBRARY_DIR}/lib/libmetis.so \
        ${WASM_LIBRARY_DIR}/lib/libcholmod.so \
        ${WASM_LIBRARY_DIR}/lib/libspqr.so \
-       dist/
+       ${DISTDIR}

packages/test/meta.yaml

@@ -10,8 +10,6 @@ package:
 build:
   type: cpython_module
   script: |
-    mkdir dist
-    export DISTDIR=$(pwd)/dist
     cd $CPYTHONBUILD
 
     export TEST_EXTENSIONS="\

pyodide-build/pyodide_build/buildpkg.py

@@ -772,11 +772,13 @@ def _build_package_inner(
         bash_runner.env["PKGDIR"] = str(pkg_root)
         bash_runner.env["PKG_VERSION"] = version
         bash_runner.env["PKG_BUILD_DIR"] = str(srcpath)
+        bash_runner.env["DISTDIR"] = str(src_dist_dir)
         if not continue_:
             clear_only = package_type == "cpython_module"
             prepare_source(build_dir, srcpath, source_metadata, clear_only=clear_only)
             patch(pkg_root, srcpath, source_metadata)
 
+        src_dist_dir.mkdir(exist_ok=True, parents=True)
         run_script(build_dir, srcpath, build_metadata, bash_runner)
 
         if package_type == "static_library":