diff --git a/src/js/dynload.ts b/src/js/dynload.ts index cb55cad77..406030e59 100644 --- a/src/js/dynload.ts +++ b/src/js/dynload.ts @@ -197,7 +197,8 @@ export class DynlibLoader { * @private */ public async loadDynlibsFromPackage( - pkg: InternalPackageData, + // TODO: Simplify the type of pkg after removing usage of this function in micropip. + pkg: { file_name: string }, dynlibPaths: string[], ) { // assume that shared libraries of a package are located in .libs directory, diff --git a/src/js/installer.ts b/src/js/installer.ts new file mode 100644 index 000000000..e5801e39a --- /dev/null +++ b/src/js/installer.ts @@ -0,0 +1,61 @@ +import { DynlibLoader } from "./dynload"; +import { uriToPackageData } from "./packaging-utils"; +import { PackageManagerAPI, PackageManagerModule } from "./types"; + +/** + * The Installer class is responsible for installing packages into the Pyodide filesystem. + * This includes + * - extracting the package into the filesystem + * - storing metadata about the Package + * - loading shared libraries + * - installing data files + */ +export class Installer { + #api: PackageManagerAPI; + #dynlibLoader: DynlibLoader; + + constructor(api: PackageManagerAPI, pyodideModule: PackageManagerModule) { + this.#api = api; + this.#dynlibLoader = new DynlibLoader(api, pyodideModule); + } + + async install( + buffer: Uint8Array, + filename: string, + installDir: string, + installer: string, + source: string, + ) { + const dynlibs: string[] = this.#api.package_loader.unpack_buffer.callKwargs( + { + buffer, + filename, + extract_dir: installDir, + installer, + source, + calculate_dynlibs: true, + }, + ); + + DEBUG && + console.debug( + `Found ${dynlibs.length} dynamic libraries inside ${filename}`, + ); + + await this.#dynlibLoader.loadDynlibsFromPackage( + { file_name: filename }, + dynlibs, + ); + } +} + +export let install: typeof Installer.prototype.install; + +if (typeof API !== "undefined" && typeof Module !== "undefined") { + const singletonInstaller = new Installer(API, Module); + + install = singletonInstaller.install.bind(singletonInstaller); + + // TODO: Find a better way to register these functions + API.install = install; +} diff --git a/src/js/load-package.ts b/src/js/load-package.ts index ebce7d752..f94abf23e 100644 --- a/src/js/load-package.ts +++ b/src/js/load-package.ts @@ -23,7 +23,7 @@ import { initNodeModules, ensureDirNode, } from "./compat"; -import { DynlibLoader } from "./dynload"; +import { Installer } from "./installer"; /** * Initialize the packages index. This is called as early as possible in @@ -94,6 +94,7 @@ export async function initializePackageIndex( } const DEFAULT_CHANNEL = "default channel"; +const INSTALLER = "pyodide.loadPackage"; /** * @hidden @@ -102,7 +103,7 @@ const DEFAULT_CHANNEL = "default channel"; export class PackageManager { #api: PackageManagerAPI; #module: PackageManagerModule; - #dynlibLoader: DynlibLoader; + #installer: Installer; /** * Only used in Node. If we can't find a package in node_modules, we'll use this @@ -132,7 +133,7 @@ export class PackageManager { constructor(api: PackageManagerAPI, pyodideModule: PackageManagerModule) { this.#api = api; this.#module = pyodideModule; - this.#dynlibLoader = new DynlibLoader(api, pyodideModule); + this.#installer = new Installer(api, pyodideModule); } /** @@ -444,30 +445,19 @@ export class PackageManager { } const filename = pkg.file_name; + // This Python helper function unpacks the buffer and lists out any .so files in it. const installDir: string = this.#api.package_loader.get_install_dir( pkg.install_dir, ); - const dynlibs: string[] = this.#api.package_loader.unpack_buffer.callKwargs( - { - buffer, - filename, - extract_dir: installDir, - calculate_dynlibs: true, - installer: "pyodide.loadPackage", - source: - metadata.channel === this.defaultChannel - ? "pyodide" - : metadata.channel, - }, + + await this.#installer.install( + buffer, + filename, + installDir, + INSTALLER, + metadata.channel === this.defaultChannel ? "pyodide" : metadata.channel, ); - - DEBUG && - console.debug( - `Found ${dynlibs.length} dynamic libraries inside ${filename}`, - ); - - await this.#dynlibLoader.loadDynlibsFromPackage(pkg, dynlibs); } /** diff --git a/src/js/test/unit/installer.test.ts b/src/js/test/unit/installer.test.ts new file mode 100644 index 000000000..7ecc55002 --- /dev/null +++ b/src/js/test/unit/installer.test.ts @@ -0,0 +1,47 @@ +import * as chai from "chai"; +import sinon from "sinon"; +import { genMockAPI, genMockModule } from "./test-helper.ts"; +import { Installer } from "../../installer.ts"; + +describe("Installer", () => { + it("should initialize with API and Module", () => { + const mockApi = genMockAPI(); + const mockMod = genMockModule(); + const _ = new Installer(mockApi, mockMod); + }); + + it("should call package_loader.unpack_buffer.callKwargs", async () => { + // @ts-ignore + globalThis.DEBUG = false; + + const mockApi = genMockAPI(); + const mockMod = genMockModule(); + const installer = new Installer(mockApi, mockMod); + + const unpackBufferSpy = sinon + .stub(mockApi.package_loader.unpack_buffer, "callKwargs") + .returns([]); + + await installer.install( + new Uint8Array(), + "filename", + "installDir", + "installer", + "source", + ); + + chai.assert.isTrue(unpackBufferSpy.calledOnce); + chai.assert.isTrue( + unpackBufferSpy.calledWith({ + buffer: new Uint8Array(), + filename: "filename", + extract_dir: "installDir", + installer: "installer", + source: "source", + calculate_dynlibs: true, + }), + ); + + unpackBufferSpy.restore(); + }); +}); diff --git a/src/js/types.ts b/src/js/types.ts index 986c07c3e..ae2295ab7 100644 --- a/src/js/types.ts +++ b/src/js/types.ts @@ -454,10 +454,18 @@ export interface API { searchDirs?: string[] | undefined, readFileFunc?: (path: string) => Uint8Array, ) => Promise; + // TODO: Remove this from the API after migrating micropip to use the `install` API instead. loadDynlibsFromPackage: ( - pkg: InternalPackageData, + pkg: { file_name: string }, dynlibPaths: string[], ) => Promise; + install: ( + buffer: Uint8Array, + filename: string, + installDir: string, + installer: string, + source: string, + ) => Promise; recursiveDependencies: ( names: string[], errorCallback: (err: string) => void, diff --git a/src/tests/test_package_loading.py b/src/tests/test_package_loading.py index a9e15ba17..30998a87a 100644 --- a/src/tests/test_package_loading.py +++ b/src/tests/test_package_loading.py @@ -747,3 +747,47 @@ def test_data_files_support(selenium_standalone, httpserver): assert (Path(sys.prefix) / "etc" / "datafile2").is_file(), "datafile2 not found" _run(selenium) + + +def test_install_api(selenium_standalone, httpserver): + selenium = selenium_standalone + + test_file_name = "dummy_pkg-0.1.0-py3-none-any.whl" + test_file_path = Path(__file__).parent / "wheels" / test_file_name + test_file_data = test_file_path.read_bytes() + install_dir = "/random_install_dir" + + httpserver.expect_oneshot_request("/" + test_file_name).respond_with_data( + test_file_data, + content_type="application/zip", + headers={"Access-Control-Allow-Origin": "*"}, + status=200, + ) + request_url = httpserver.url_for("/" + test_file_name) + + selenium.run_js( + f""" + wheelData = await fetch("{request_url}"); + wheelDataArr = new Uint8Array(await wheelData.arrayBuffer()); + await pyodide._api.install( + wheelDataArr, + "{test_file_name}", + "{install_dir}", + "pytest", + "pytest", + ); + """ + ) + + @run_in_pyodide + def _run(selenium, pkg_dir): + import pathlib + + d = pathlib.Path(pkg_dir) + assert d.is_dir(), f"Directory {d} not found" + assert ( + d / "dummy_pkg-0.1.0.dist-info" + ).is_dir(), "dist-info directory not found" + assert (d / "dummy_pkg").is_dir(), "package directory not found" + + _run(selenium, install_dir)