Extract out JS API for package installation (#5215)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

src/js/dynload.ts

@@ -197,7 +197,8 @@ export class DynlibLoader {
    * @private
    */
   public async loadDynlibsFromPackage(
-    pkg: InternalPackageData,
+    // TODO: Simplify the type of pkg after removing usage of this function in micropip.
+    pkg: { file_name: string },
     dynlibPaths: string[],
   ) {
     // assume that shared libraries of a package are located in <package-name>.libs directory,

src/js/installer.ts

@@ -0,0 +1,61 @@
+import { DynlibLoader } from "./dynload";
+import { uriToPackageData } from "./packaging-utils";
+import { PackageManagerAPI, PackageManagerModule } from "./types";
+
+/**
+ * The Installer class is responsible for installing packages into the Pyodide filesystem.
+ * This includes
+ * - extracting the package into the filesystem
+ * - storing metadata about the Package
+ * - loading shared libraries
+ * - installing data files
+ */
+export class Installer {
+  #api: PackageManagerAPI;
+  #dynlibLoader: DynlibLoader;
+
+  constructor(api: PackageManagerAPI, pyodideModule: PackageManagerModule) {
+    this.#api = api;
+    this.#dynlibLoader = new DynlibLoader(api, pyodideModule);
+  }
+
+  async install(
+    buffer: Uint8Array,
+    filename: string,
+    installDir: string,
+    installer: string,
+    source: string,
+  ) {
+    const dynlibs: string[] = this.#api.package_loader.unpack_buffer.callKwargs(
+      {
+        buffer,
+        filename,
+        extract_dir: installDir,
+        installer,
+        source,
+        calculate_dynlibs: true,
+      },
+    );
+
+    DEBUG &&
+      console.debug(
+        `Found ${dynlibs.length} dynamic libraries inside ${filename}`,
+      );
+
+    await this.#dynlibLoader.loadDynlibsFromPackage(
+      { file_name: filename },
+      dynlibs,
+    );
+  }
+}
+
+export let install: typeof Installer.prototype.install;
+
+if (typeof API !== "undefined" && typeof Module !== "undefined") {
+  const singletonInstaller = new Installer(API, Module);
+
+  install = singletonInstaller.install.bind(singletonInstaller);
+
+  // TODO: Find a better way to register these functions
+  API.install = install;
+}

src/js/load-package.ts

@@ -23,7 +23,7 @@ import {
   initNodeModules,
   ensureDirNode,
 } from "./compat";
-import { DynlibLoader } from "./dynload";
+import { Installer } from "./installer";
 
 /**
  * Initialize the packages index. This is called as early as possible in
@@ -94,6 +94,7 @@ export async function initializePackageIndex(
 }
 
 const DEFAULT_CHANNEL = "default channel";
+const INSTALLER = "pyodide.loadPackage";
 
 /**
  * @hidden
@@ -102,7 +103,7 @@ const DEFAULT_CHANNEL = "default channel";
 export class PackageManager {
   #api: PackageManagerAPI;
   #module: PackageManagerModule;
-  #dynlibLoader: DynlibLoader;
+  #installer: Installer;
 
   /**
    * Only used in Node. If we can't find a package in node_modules, we'll use this
@@ -132,7 +133,7 @@ export class PackageManager {
   constructor(api: PackageManagerAPI, pyodideModule: PackageManagerModule) {
     this.#api = api;
     this.#module = pyodideModule;
-    this.#dynlibLoader = new DynlibLoader(api, pyodideModule);
+    this.#installer = new Installer(api, pyodideModule);
   }
 
   /**
@@ -444,30 +445,19 @@ export class PackageManager {
     }
 
     const filename = pkg.file_name;
+
     // This Python helper function unpacks the buffer and lists out any .so files in it.
     const installDir: string = this.#api.package_loader.get_install_dir(
       pkg.install_dir,
     );
-    const dynlibs: string[] = this.#api.package_loader.unpack_buffer.callKwargs(
-      {
-        buffer,
-        filename,
-        extract_dir: installDir,
-        calculate_dynlibs: true,
-        installer: "pyodide.loadPackage",
-        source:
-          metadata.channel === this.defaultChannel
-            ? "pyodide"
-            : metadata.channel,
-      },
+
+    await this.#installer.install(
+      buffer,
+      filename,
+      installDir,
+      INSTALLER,
+      metadata.channel === this.defaultChannel ? "pyodide" : metadata.channel,
     );
-
-    DEBUG &&
-      console.debug(
-        `Found ${dynlibs.length} dynamic libraries inside ${filename}`,
-      );
-
-    await this.#dynlibLoader.loadDynlibsFromPackage(pkg, dynlibs);
   }
 
   /**

src/js/test/unit/installer.test.ts

@@ -0,0 +1,47 @@
+import * as chai from "chai";
+import sinon from "sinon";
+import { genMockAPI, genMockModule } from "./test-helper.ts";
+import { Installer } from "../../installer.ts";
+
+describe("Installer", () => {
+  it("should initialize with API and Module", () => {
+    const mockApi = genMockAPI();
+    const mockMod = genMockModule();
+    const _ = new Installer(mockApi, mockMod);
+  });
+
+  it("should call package_loader.unpack_buffer.callKwargs", async () => {
+    // @ts-ignore
+    globalThis.DEBUG = false;
+
+    const mockApi = genMockAPI();
+    const mockMod = genMockModule();
+    const installer = new Installer(mockApi, mockMod);
+
+    const unpackBufferSpy = sinon
+      .stub(mockApi.package_loader.unpack_buffer, "callKwargs")
+      .returns([]);
+
+    await installer.install(
+      new Uint8Array(),
+      "filename",
+      "installDir",
+      "installer",
+      "source",
+    );
+
+    chai.assert.isTrue(unpackBufferSpy.calledOnce);
+    chai.assert.isTrue(
+      unpackBufferSpy.calledWith({
+        buffer: new Uint8Array(),
+        filename: "filename",
+        extract_dir: "installDir",
+        installer: "installer",
+        source: "source",
+        calculate_dynlibs: true,
+      }),
+    );
+
+    unpackBufferSpy.restore();
+  });
+});

src/js/types.ts

@@ -454,10 +454,18 @@ export interface API {
     searchDirs?: string[] | undefined,
     readFileFunc?: (path: string) => Uint8Array,
   ) => Promise<void>;
+  // TODO: Remove this from the API after migrating micropip to use the `install` API instead.
   loadDynlibsFromPackage: (
-    pkg: InternalPackageData,
+    pkg: { file_name: string },
     dynlibPaths: string[],
   ) => Promise<void>;
+  install: (
+    buffer: Uint8Array,
+    filename: string,
+    installDir: string,
+    installer: string,
+    source: string,
+  ) => Promise<void>;
   recursiveDependencies: (
     names: string[],
     errorCallback: (err: string) => void,

src/tests/test_package_loading.py

@@ -747,3 +747,47 @@ def test_data_files_support(selenium_standalone, httpserver):
         assert (Path(sys.prefix) / "etc" / "datafile2").is_file(), "datafile2 not found"
 
     _run(selenium)
+
+
+def test_install_api(selenium_standalone, httpserver):
+    selenium = selenium_standalone
+
+    test_file_name = "dummy_pkg-0.1.0-py3-none-any.whl"
+    test_file_path = Path(__file__).parent / "wheels" / test_file_name
+    test_file_data = test_file_path.read_bytes()
+    install_dir = "/random_install_dir"
+
+    httpserver.expect_oneshot_request("/" + test_file_name).respond_with_data(
+        test_file_data,
+        content_type="application/zip",
+        headers={"Access-Control-Allow-Origin": "*"},
+        status=200,
+    )
+    request_url = httpserver.url_for("/" + test_file_name)
+
+    selenium.run_js(
+        f"""
+        wheelData = await fetch("{request_url}");
+        wheelDataArr = new Uint8Array(await wheelData.arrayBuffer());
+        await pyodide._api.install(
+          wheelDataArr,
+          "{test_file_name}",
+          "{install_dir}",
+          "pytest",
+          "pytest",
+        );
+        """
+    )
+
+    @run_in_pyodide
+    def _run(selenium, pkg_dir):
+        import pathlib
+
+        d = pathlib.Path(pkg_dir)
+        assert d.is_dir(), f"Directory {d} not found"
+        assert (
+            d / "dummy_pkg-0.1.0.dist-info"
+        ).is_dir(), "dist-info directory not found"
+        assert (d / "dummy_pkg").is_dir(), "package directory not found"
+
+    _run(selenium, install_dir)