Allow customizing cache location for packages in Node (#3967)

This change allows users to customize where wheels are cached in Node.
This is important in the context of docker images and other cases where
the `node_modules` folder isn't writable.

docs/project/changelog.md

@@ -15,6 +15,9 @@ myst:
 
 ## Unreleased
 
+- {{ Enhancement }} Allow customizing installation location for packages
+  {pr}`3967`
+
 - {{ Enhancement }} ABI Break: Updated Emscripten to version 3.1.39
   {pr}`3665`, {pr}`3659`, {pr}`3822`, {pr}`3889`, {pr}`3890`
 

src/js/load-package.ts

@@ -214,11 +214,11 @@ function recursiveDependencies(
 
 /**
  * Download a package. If `channel` is `DEFAULT_CHANNEL`, look up the wheel URL
- * relative to indexURL from `pyodide-lock.json`, otherwise use the URL specified by
+ * relative to packageCacheDir (when IN_NODE), or indexURL from `pyodide-lock.json`, otherwise use the URL specified by
  * `channel`.
  * @param name The name of the package
  * @param channel Either `DEFAULT_CHANNEL` or the absolute URL to the
- * wheel or the path to the wheel relative to indexURL.
+ * wheel or the path to the wheel relative to packageCacheDir (when IN_NODE), or indexURL.
  * @param checkIntegrity Whether to check the integrity of the downloaded
  * package.
  * @returns The binary data for the package
@@ -229,13 +229,24 @@ async function downloadPackage(
   channel: string,
   checkIntegrity: boolean = true,
 ): Promise<Uint8Array> {
+  let installBaseUrl: string;
+  if (IN_NODE) {
+    installBaseUrl = API.config.packageCacheDir;
+    // ensure that the directory exists before trying to download files into it
+    await nodeFsPromisesMod.mkdir(API.config.packageCacheDir, {
+      recursive: true,
+    });
+  } else {
+    installBaseUrl = API.config.indexURL;
+  }
+
   let file_name, uri, file_sub_resource_hash;
   if (channel === DEFAULT_CHANNEL) {
     if (!(name in API.lockfile_packages)) {
       throw new Error(`Internal error: no entry for package named ${name}`);
     }
     file_name = API.lockfile_packages[name].file_name;
-    uri = resolvePath(file_name, API.config.indexURL);
+    uri = resolvePath(file_name, installBaseUrl);
     file_sub_resource_hash = API.package_loader.sub_resource_hash(
       API.lockfile_packages[name].sha256,
     );

src/js/pyodide.ts

@@ -177,6 +177,7 @@ function calculateIndexURL(): string {
  */
 export type ConfigType = {
   indexURL: string;
+  packageCacheDir: string;
   lockFileURL: string;
   homedir: string;
   fullStdLib?: boolean;
@@ -209,6 +210,16 @@ export async function loadPyodide(
      */
     indexURL?: string;
 
+    /**
+     * The file path where packages will be cached in `node.js`. If a package
+     * exists in `packageCacheDir` it is loaded from there, otherwise it is
+     * downloaded from the JsDelivr CDN and then cached into `packageCacheDir`.
+     * Only applies when running in node.js. Ignored in browsers.
+     *
+     * Default: same as indexURL
+     */
+    packageCacheDir?: string;
+
     /**
      * The URL from which Pyodide will load the Pyodide ``pyodide-lock.json`` lock
      * file. You can produce custom lock files with :py:func:`micropip.freeze`.
@@ -293,6 +304,7 @@ export async function loadPyodide(
     args: [],
     _node_mounts: [],
     env: {},
+    packageCacheDir: indexURL,
   };
   const config = Object.assign(default_config, options) as ConfigType;
   if (options.homedir) {