Load shared libraries locally (#4876)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

docs/project/changelog.md

@@ -21,6 +21,10 @@ myst:
   timezone handling.
   {pr}`4889`
 
+- {{ Breaking }} Shared libraries are now loaded locally. This means that packages that
+  depend on shared libraries link to the shared libraries explicitly.
+  {pr}`4876`
+
 - {{ Enhancement }} Added implementation to abort `pyfetch` and `FetchResponse`
   manually or automatically.
   {pr}`4846`

emsdk/patches/0006-Load-dependent-libs-globally.patch

@@ -1,35 +0,0 @@
-From 2f3ae3b30cf8ce8cd04d512ebb4c8b2218563085 Mon Sep 17 00:00:00 2001
-From: Hood Chatham <roberthoodchatham@gmail.com>
-Date: Mon, 17 Jun 2024 18:32:07 -0700
-Subject: [PATCH 6/6] Load dependent libs globally
-
-I'm not 100% sure why, but we need all dependent shared libs to be loaded
-globally.
----
- src/library_dylink.js | 6 +++++-
- 1 file changed, 5 insertions(+), 1 deletion(-)
-
-diff --git a/src/library_dylink.js b/src/library_dylink.js
-index e0fa4721c..502bc7b25 100644
---- a/src/library_dylink.js
-+++ b/src/library_dylink.js
-@@ -880,11 +880,15 @@ var LibraryDylink = {
- 
-     // now load needed libraries and the module itself.
-     if (flags.loadAsync) {
-+      const origGlobal = flags.global;
-+      flags.global = true;
-       return metadata.neededDynlibs
-         .reduce((chain, dynNeeded) => chain.then(() =>
-           loadDynamicLibrary(dynNeeded, flags)
-         ), Promise.resolve())
--        .then(loadModule);
-+        .then(() => {
-+          flags.global = origGlobal;
-+        }).then(loadModule);
-     }
- 
-     metadata.neededDynlibs.forEach((needed) => loadDynamicLibrary(needed, flags, localScope));
--- 
-2.34.1
-

emsdk/patches/0006-Make-RTLD_LOCAL-work-correctly-for-preloaded-DSOs-21.patch

@@ -0,0 +1,46 @@
+From 4a3956693056b3ad4f6541589c5ab44df7a7fc39 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Morten=20Johan=20S=C3=B8rvig?=
+ <msorvig@users.noreply.github.com>
+Date: Wed, 12 Jun 2024 20:10:35 +0200
+Subject: [PATCH 6/6] Make RTLD_LOCAL work correctly for preloaded DSOs
+ (#21985)
+
+Copied from: https://github.com/emscripten-core/emscripten/pull/21985
+This patch can be removed when updating Emscripten version to >= 3.1.62
+
+Follow-up to commit c9a5e63c, for the FS.createPreloadedFile()
+case.
+
+Make sure loadWebAssemblyModule() and loadDynamicLibrary()
+are called with a valid 'localScope' object when invoked
+from the .so file type preload handler.
+---
+ src/library_dylink.js |  4 ++--
+ test/test_core.py     | 46 +++++++++++++++++++++++++++++++++++++++++++
+ 2 files changed, 48 insertions(+), 2 deletions(-)
+
+diff --git a/src/library_dylink.js b/src/library_dylink.js
+index 965b119a4..211f4030c 100644
+--- a/src/library_dylink.js
++++ b/src/library_dylink.js
+@@ -26,7 +26,7 @@ var LibraryDylink = {
+         // than just running the promises in parallel, this makes a chain of
+         // promises to run in series.
+         wasmPlugin['promiseChainEnd'] = wasmPlugin['promiseChainEnd'].then(
+-          () => loadWebAssemblyModule(byteArray, {loadAsync: true, nodelete: true}, name)).then(
++          () => loadWebAssemblyModule(byteArray, {loadAsync: true, nodelete: true}, name, {})).then(
+             (exports) => {
+ #if DYLINK_DEBUG
+               dbg(`registering preloadedWasm: ${name}`);
+@@ -877,7 +877,7 @@ var LibraryDylink = {
+     if (flags.loadAsync) {
+       return metadata.neededDynlibs
+         .reduce((chain, dynNeeded) => chain.then(() =>
+-          loadDynamicLibrary(dynNeeded, flags)
++          loadDynamicLibrary(dynNeeded, flags, localScope)
+         ), Promise.resolve())
+         .then(loadModule);
+     }
+-- 
+2.29.2.windows.2
+

packages/scipy/meta.yaml

@@ -30,7 +30,8 @@ source:
     - patches/0007-Fix-gees-calls.patch
     - patches/0008-MAINT-linalg-Remove-id_dist-Fortran-files.patch
     - patches/0009-Mark-mvndst-functions-recursive.patch
-    - patches/0001-Make-sreorth-recursive.patch
+    - patches/0010-Make-sreorth-recursive.patch
+    - patches/0011-Link-openblas-with-modules-that-require-f2c.patch
 
 build:
   cflags: |

packages/scipy/patches/0011-Link-openblas-with-modules-that-require-f2c.patch

@@ -0,0 +1,80 @@
+From 091fb1486525e310a9b8e41d6b4d7a70c965c649 Mon Sep 17 00:00:00 2001
+From: ryanking13 <def6488@gmail.com>
+Date: Sun, 28 Jul 2024 18:15:17 +0900
+Subject: [PATCH 1/1] Link openblas with modules that require f2c
+
+Some fortran modules require symbols from f2c, which is provided by
+openblas.
+This patch adds openblas as a dependency to the modules that require f2c
+symbols.
+---
+ scipy/integrate/meson.build | 2 +-
+ scipy/optimize/meson.build  | 4 ++--
+ scipy/special/meson.build   | 2 +-
+ scipy/stats/meson.build     | 2 +-
+ 4 files changed, 5 insertions(+), 5 deletions(-)
+
+diff --git a/scipy/integrate/meson.build b/scipy/integrate/meson.build
+index 4acc119a5..3c0e0f695 100644
+--- a/scipy/integrate/meson.build
++++ b/scipy/integrate/meson.build
+@@ -167,7 +167,7 @@ py3.extension_module('_dop',
+   _dop_module,
+   link_with: [dop_lib],
+   c_args: [numpy_nodepr_api, Wno_unused_variable],
+-  dependencies: [fortranobject_dep],
++  dependencies: [lapack, fortranobject_dep],
+   link_args: version_link_args,
+   install: true,
+   link_language: 'fortran',
+diff --git a/scipy/optimize/meson.build b/scipy/optimize/meson.build
+index 964edad36..3ca17e3dc 100644
+--- a/scipy/optimize/meson.build
++++ b/scipy/optimize/meson.build
+@@ -138,7 +138,7 @@ _cobyla = py3.extension_module('_cobyla',
+   c_args: [numpy_nodepr_api, Wno_unused_variable],
+   fortran_args: fortran_ignore_warnings,
+   link_args: version_link_args,
+-  dependencies: [fortranobject_dep],
++  dependencies: [lapack, fortranobject_dep],
+   install: true,
+   link_language: 'fortran',
+   subdir: 'scipy/optimize'
+@@ -172,7 +172,7 @@ _slsqp = py3.extension_module('_slsqp',
+   c_args: numpy_nodepr_api,
+   fortran_args: fortran_ignore_warnings,
+   link_args: version_link_args,
+-  dependencies: [fortranobject_dep],
++  dependencies: [lapack, fortranobject_dep],
+   install: true,
+   link_language: 'fortran',
+   subdir: 'scipy/optimize'
+diff --git a/scipy/special/meson.build b/scipy/special/meson.build
+index d166555f5..c0ca05f77 100644
+--- a/scipy/special/meson.build
++++ b/scipy/special/meson.build
+@@ -276,7 +276,7 @@ py3.extension_module('_specfun',
+   specfun_module,
+   c_args: numpy_nodepr_api,
+   link_args: version_link_args,
+-  dependencies: [fortranobject_dep],
++  dependencies: [lapack, fortranobject_dep],
+   link_with: specfun_lib,
+   link_language: 'fortran',
+   install: true,
+diff --git a/scipy/stats/meson.build b/scipy/stats/meson.build
+index 9460f5338..145436fb4 100644
+--- a/scipy/stats/meson.build
++++ b/scipy/stats/meson.build
+@@ -44,7 +44,7 @@ mvn = py3.extension_module('_mvn',
+   # Wno-surprising is to suppress a pointless warning with GCC 10-12
+   # (see GCC bug 98411: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=98411)
+   fortran_args: [fortran_ignore_warnings, _fflag_Wno_surprising],
+-  dependencies: [fortranobject_dep],
++  dependencies: [lapack, fortranobject_dep],
+   link_args: version_link_args,
+   install: true,
+   link_language: 'fortran',
+-- 
+2.29.2.windows.2
+

src/js/dynload.ts

@@ -7,8 +7,36 @@ import { memoize } from "./common/memoize";
 import { InternalPackageData } from "./load-package";
 import { LoadDynlibFS, ReadFileType } from "./types";
 
-// File System-like type which can be passed to
-// Module.loadDynamicLibrary or Module.loadWebAssemblyModule
+/**
+ * Recursively get all subdirectories of a directory
+ *
+ * @param dir The absolute path to the directory
+ * @returns A list of absolute paths to the subdirectories
+ * @private
+ */
+function* getSubDirs(dir: string): Generator<string> {
+  const dirs = Module.FS.readdir(dir);
+
+  for (const d of dirs) {
+    if (d === "." || d === "..") {
+      continue;
+    }
+
+    const subdir: string = Module.PATH.join2(dir, d);
+    const lookup = Module.FS.lookupPath(subdir);
+    if (lookup.node === null) {
+      continue;
+    }
+
+    const mode = lookup.node.mode;
+    if (!Module.FS.isDir(mode)) {
+      continue;
+    }
+
+    yield subdir;
+    yield* getSubDirs(subdir);
+  }
+}
 
 /**
  * Creates a filesystem-like object to be passed to Module.loadDynamicLibrary or Module.loadWebAssemblyModule
@@ -16,20 +44,16 @@ import { LoadDynlibFS, ReadFileType } from "./types";
  *
  * @param lib The path to the library to load
  * @param searchDirs The list of directories to search for the library
- * @param readFileFunc The function to read a file, if not provided, Module.FS.readFile will be used
  * @returns A filesystem-like object
  * @private
  */
-function createDynlibFS(
-  lib: string,
-  searchDirs?: string[],
-  readFileFunc?: ReadFileType,
-): LoadDynlibFS {
+function createDynlibFS(lib: string, searchDirs?: string[]): LoadDynlibFS {
   const dirname = lib.substring(0, lib.lastIndexOf("/"));
 
   let _searchDirs = searchDirs || [];
   _searchDirs = _searchDirs.concat(API.defaultLdLibraryPath, [dirname]);
 
+  // TODO: add rpath to Emscripten dsos and remove this logic
   const resolvePath = (path: string) => {
     if (DEBUG) {
       if (Module.PATH.basename(path) !== Module.PATH.basename(lib)) {
@@ -37,20 +61,34 @@ function createDynlibFS(
       }
     }
 
+    // If the path is absolute, we don't need to search for it.
+    if (Module.PATH.isAbs(path)) {
+      return path;
+    }
+
+    // Step 1) Try to find the library in the search directories
     for (const dir of _searchDirs) {
       const fullPath = Module.PATH.join2(dir, path);
+
       if (Module.FS.findObject(fullPath) !== null) {
         return fullPath;
       }
     }
+
+    // Step 2) try to find the library by searching child directories of the library directory
+    //         (This should not be necessary in most cases, but some libraries have dependencies in the child directories)
+    for (const childDir of getSubDirs(dirname)) {
+      const fullPath = Module.PATH.join2(childDir, path);
+      if (Module.FS.findObject(fullPath) !== null) {
+        return fullPath;
+      }
+    }
+
     return path;
   };
 
-  let readFile: ReadFileType = (path: string) =>
+  const readFile: ReadFileType = (path: string) =>
     Module.FS.readFile(resolvePath(path));
-  if (readFileFunc !== undefined) {
-    readFile = (path: string) => readFileFunc(resolvePath(path));
-  }
 
   const fs: LoadDynlibFS = {
     findObject: (path: string, dontResolveLastLink: boolean) => {
@@ -82,14 +120,12 @@ const acquireDynlibLock = createLock();
  * @param lib The file system path to the library.
  * @param global Whether to make the symbols available globally.
  * @param searchDirs Directories to search for the library.
- * @param readFileFunc The function to read a file, if not provided, Module.FS.readFile will be used
  * @private
  */
 export async function loadDynlib(
   lib: string,
   global: boolean,
   searchDirs?: string[],
-  readFileFunc?: ReadFileType,
 ) {
   const releaseDynlibLock = await acquireDynlibLock();
 
@@ -97,22 +133,27 @@ export async function loadDynlib(
     console.debug(`Loading a dynamic library ${lib} (global: ${global})`);
   }
 
-  const fs = createDynlibFS(lib, searchDirs, readFileFunc);
+  const fs = createDynlibFS(lib, searchDirs);
+  const localScope = global ? null : {};
 
   try {
-    await Module.loadDynamicLibrary(lib, {
-      loadAsync: true,
-      nodelete: true,
-      allowUndefined: true,
-      global,
-      fs,
-    });
+    await Module.loadDynamicLibrary(
+      lib,
+      {
+        loadAsync: true,
+        nodelete: true,
+        allowUndefined: true,
+        global,
+        fs,
+      },
+      localScope,
+    );
 
     // Emscripten saves the list of loaded libraries in LDSO.loadedLibsByName.
     // However, since emscripten dylink metadata only contains the name of the
     // library not the full path, we need to update it manually in order to
     // prevent loading same library twice.
-    if (global && Module.PATH.isAbs(lib)) {
+    if (Module.PATH.isAbs(lib)) {
       const libName: string = Module.PATH.basename(lib);
       const dso: any = Module.LDSO.loadedLibsByName[libName];
       if (!dso) {
@@ -157,74 +198,9 @@ export async function loadDynlibsFromPackage(
     pkg.file_name.split("-")[0]
   }.libs`;
 
-  // This prevents from reading large libraries multiple times. We may read it
-  // once in calculateGlobalLibs and again in loadDynlib.
-  // Memoized function is intentionally instanced per call to
-  // loadDynlibsFromPackage since we won't need the data again after we're done.
-  const readFileMemoized: ReadFileType = memoize(Module.FS.readFile);
-
-  type Dynlib = { path: string; global: boolean };
-  let dynlibs: Dynlib[];
-
-  if (pkg.shared_library) {
-    // All dynamic libs in a shared library should be global.
-    dynlibs = dynlibPaths.map((path) => ({
-      path,
-      global: true,
-    }));
-  } else {
-    // Otherwise, they should only be global if they are in the DT_NEEDED list
-    // of another dynlib.
-    const globalLibs: Set<string> = calculateGlobalLibs(
-      dynlibPaths,
-      readFileMemoized,
-    );
-
-    dynlibs = dynlibPaths.map((path) => ({
-      path,
-      global: globalLibs.has(Module.PATH.basename(path)),
-    }));
-    // Sort libraries so that global libraries can be loaded first.
-    // TODO(ryanking13): It is not clear why global libraries should be loaded first.
-    // But without this, we get a fatal error when loading the libraries.
-    type T = { global: boolean };
-    dynlibs.sort(
-      (lib1: T, lib2: T) => Number(lib2.global) - Number(lib1.global),
-    );
+  for (const path of dynlibPaths) {
+    await loadDynlib(path, false, [auditWheelLibDir]);
   }
-
-  for (const { path, global } of dynlibs) {
-    await loadDynlib(path, global, [auditWheelLibDir], readFileMemoized);
-  }
-}
-
-/**
- * Given a list of libraries, return a list of libraries to load globally.
- * @param libs The list of path to libraries
- * @param readFileFunc A function to read the file, if not provided, use Module.FS.readFile
- * @returns A list of libraries needed to be loaded globally
- * @private
- */
-function calculateGlobalLibs(
-  libs: string[],
-  readFileFunc?: ReadFileType,
-): Set<string> {
-  let readFile: ReadFileType = Module.FS.readFile;
-  if (readFileFunc !== undefined) {
-    readFile = readFileFunc;
-  }
-
-  const globalLibs = new Set<string>();
-
-  libs.forEach((lib: string) => {
-    const binary = readFile(lib);
-    const needed = Module.getDylinkMetadata(binary).neededDynlibs;
-    needed.forEach((lib: string) => {
-      globalLibs.add(lib);
-    });
-  });
-
-  return globalLibs;
 }
 
 API.loadDynlib = loadDynlib;

src/js/types.ts

@@ -230,7 +230,7 @@ export interface FS {
   mkdev: (path: string, dev: number) => FSNode;
   filesystems: any;
   stat: (path: string, dontFollow?: boolean) => any;
-  readdir: (node: FSNode) => string[];
+  readdir: (path: string) => string[];
   isDir: (mode: number) => boolean;
   isMountpoint: (mode: FSNode) => boolean;
   lookupPath: (
@@ -265,6 +265,8 @@ export type PreRunFunc = (Module: Module) => void;
 
 export type ReadFileType = (path: string) => Uint8Array;
 
+// File System-like type which can be passed to
+// Module.loadDynamicLibrary or Module.loadWebAssemblyModule
 export type LoadDynlibFS = {
   readFile: ReadFileType;
   findObject: (path: string, dontResolveLastLink: boolean) => any;
@@ -300,6 +302,8 @@ export interface Module {
       global?: boolean;
       fs: LoadDynlibFS;
     },
+    localScope?: object | null,
+    handle?: number,
   ): void;
   getDylinkMetadata(binary: Uint8Array | WebAssembly.Module): {
     neededDynlibs: string[];