Update docs on NativeFS and NodeFS (#4562)

Also I added a useful mountDirectory method to console.html

docs/index.rst

@@ -44,8 +44,9 @@ Using Pyodide
    usage/downloading-and-deploying.md
    usage/index.md
    usage/loading-packages.md
-   usage/type-conversions.md
+   usage/loading-files.md
    usage/wasm-constraints.md
+   usage/type-conversions.md
    usage/keyboard-interrupts.md
    usage/streams.md
    usage/api-reference.md

docs/usage/accessing-files.md

@@ -0,0 +1,96 @@
+(accessing_files_quickref)=
+
+# Accessing Files Quick Reference
+
+For development of modules to use in Pyodide, the best experience comes from
+using Pyodide in Node and mounting the development directory into Pyodide using
+the NodeFS. In the NodeFS, all changes to your native file system are
+immediately reflected in Pyodide and vice versa.
+
+If your code is browser-only, you can use the Chrome `NativeFS` for development.
+This will not automatically sync up with your native file system, but it is
+still quite convenient.
+
+## In Node.js
+
+It's recommended to use {js:func}`pyodide.mountNodeFS` to mount the host file
+system so that it is accessible from inside of Pyodide. For example if you have
+a Python package in a folder called `my_package`, you can do:
+
+```pyodide
+pyodide.mountNodeFS("my_package", "/path/to/my_package");
+pyodide.runPython(`
+import my_package
+# ... use it
+`);
+```
+
+## In the browser
+
+To access local files in Chrome, you can use the File System Access API to
+acquire a directory handle and then mount the directory into the Pyodide file
+system with {js:func}`pyodide.mountNativeFS`. To acquire the directory handle,
+you have to fill out a folder picker the first time. The handle can subsequently
+be stored in the `IndexedDB`. You will still be prompted for read and write
+access, but you don't have to deal with the folder picker again.
+
+The following code is a good starting point:
+
+```js
+const { get, set } = await import(
+  "https://unpkg.com/idb-keyval@5.0.2/dist/esm/index.js"
+);
+
+/**
+ * Mount a folder from your native filesystem as the directory
+ * `pyodideDirectory`. If `directoryKey` was used previously, then it will reuse
+ * the same folder as last time. Otherwise, it will show a directory picker.
+ */
+async function mountDirectory(pyodideDirectory, directoryKey) {
+  let directoryHandle = await get(directoryKey);
+  const opts = {
+    id: "mountdirid",
+    mode: "readwrite",
+  };
+  if (!directoryHandle) {
+    directoryHandle = await showDirectoryPicker(opts);
+    await set(directoryKey, directoryHandle);
+  }
+  const permissionStatus = await directoryHandle.requestPermission(opts);
+  if (permissionStatus !== "granted") {
+    throw new Error("readwrite access to directory not granted");
+  }
+  const { syncfs } = await pyodide.mountNativeFS(
+    pyodideDirectory,
+    directoryHandle,
+  );
+  return syncfs;
+}
+```
+
+See {ref}`nativefs-api` for more information.
+
+## Downloading external archives
+
+If you are using Pyodide in the browser, you should download external files and
+save them to the virtual file system. The recommended way to do this is to zip
+the files and unpack them into the file system with
+{js:func}`pyodide.unpackArchive`:
+
+```pyodide
+let zipResponse = await fetch("myfiles.zip");
+let zipBinary = await zipResponse.arrayBuffer();
+pyodide.unpackArchive(zipBinary, "zip");
+```
+
+You can also download the files from Python using
+{py:func}`~pyodide.http.pyfetch`, which is a convenient wrapper of JavaScript
+{js:func}`fetch`:
+
+```pyodide
+await pyodide.runPythonAsync(`
+  from pyodide.http import pyfetch
+  response = await pyfetch("https://some_url/myfiles.zip")
+  await response.unpack_archive()
+`)
+```

docs/usage/faq.md

@@ -4,46 +4,7 @@
 
 ## How can I load external files in Pyodide?
 
-If you are using Pyodide in the browser, you should download external files and
-save them to the virtual file system. The recommended way to do this is to zip
-the files and unpack them into the file system with
-{js:func}`pyodide.unpackArchive`:
-
-```pyodide
-let zipResponse = await fetch("myfiles.zip");
-let zipBinary = await zipResponse.arrayBuffer();
-pyodide.unpackArchive(zipBinary, "zip");
-```
-
-You can also download the files from Python using
-{py:func}`~pyodide.http.pyfetch`, which is a convenient wrapper of JavaScript
-{js:func}`fetch`:
-
-```pyodide
-await pyodide.runPythonAsync(`
-  from pyodide.http import pyfetch
-  response = await pyfetch("https://some_url/myfiles.zip")
-  await response.unpack_archive()
-`)
-```
-
-If you are working in Node.js, you can mount a native folder into the file
-system as follows:
-
-```pyodide
-FS.mkdir("/local_directory");
-FS.mount(NODEFS, { root: "some/local/filepath" }, "/local_directory");
-```
-
-Then you can access the mounted folder from Python via the `/local_directory`
-mount.
-
-```{admonition} Why can't I just use urllib or requests?
-:class: warning
-
-We currently can’t use such packages since sockets are not available in Pyodide.
-See {ref}`http-client-limit` for more information.
-```
+See {accessing_files_quickref}`accessing_files_quickref`.
 
 ## Why can't I load files from the local file system?
 

docs/usage/file-system.md

@@ -9,10 +9,11 @@ API](https://emscripten.org/docs/api_reference/Filesystem-API.html#filesystem-ap
 
 **Example: Reading from the file system**
 
-```js
+```pyodide
 pyodide.runPython(`
-  with open("/hello.txt", "w") as fh:
-      fh.write("hello world!")
+  from pathlib import Path
+
+  Path("/hello.txt").write_text("hello world!")
 `);
 
 let file = pyodide.FS.readFile("/hello.txt", { encoding: "utf8" });
@@ -21,13 +22,13 @@ console.log(file); // ==> "hello world!"
 
 **Example: Writing to the file system**
 
-```js
+```pyodide
 let data = "hello world!";
 pyodide.FS.writeFile("/hello.txt", data, { encoding: "utf8" });
 pyodide.runPython(`
-  with open("/hello.txt", "r") as fh:
-        data = fh.read()
-  print(data)
+  from pathlib import Path
+
+  print(Path("/hello.txt").read_text())
 `);
 ```
 
@@ -46,15 +47,15 @@ a folder with the
 
 ```js
 let mountDir = "/mnt";
-pyodide.FS.mkdir(mountDir);
+pyodide.FS.mkdirTree(mountDir);
 pyodide.FS.mount(pyodide.FS.filesystems.IDBFS, { root: "." }, mountDir);
 ```
 
 If you are using Node.js you can access the native file system by mounting `NODEFS`.
 
-```js
+```pyodide
 let mountDir = "/mnt";
-pyodide.FS.mkdir(mountDir);
+pyodide.FS.mkdirTree(mountDir);
 pyodide.FS.mount(pyodide.FS.filesystems.NODEFS, { root: "." }, mountDir);
 pyodide.runPython("import os; print(os.listdir('/mnt'))");
 // ==> The list of files in the Node working directory
@@ -78,15 +79,14 @@ The File System Access API is only supported in Chromium based browsers: Chrome
 Pyodide provides an API {js:func}`pyodide.mountNativeFS` which mounts a
 {js:class}`FileSystemDirectoryHandle` into the Pyodide Python file system.
 
-```js
+```pyodide
 const dirHandle = await showDirectoryPicker();
+const permissionStatus = await dirHandle.requestPermission({
+  mode: "readwrite",
+});
 
-if ((await dirHandle.queryPermission({ mode: "readwrite" })) !== "granted") {
-  if (
-    (await dirHandle.requestPermission({ mode: "readwrite" })) !== "granted"
-  ) {
-    throw Error("Unable to read and write directory");
-  }
+if (permissionStatus !== "granted") {
+  throw new Error("readwrite access to directory not granted");
 }
 
 const nativefs = await pyodide.mountNativeFS("/mount_dir", dirHandle);
@@ -103,7 +103,7 @@ Due to browser limitations, the changes in the mounted file system
 is not synchronized by default. In order to persist any operations
 to an native file system, you must call
 
-```js
+```pyodide
 // nativefs is the returned from: await pyodide.mountNativeFS('/mount_dir', dirHandle)
 pyodide.runPython(`
   with open('/mount_dir/new_file.txt', 'w') as f:

docs/usage/index.md

@@ -129,8 +129,9 @@ undefined
 .. toctree::
    :hidden:
 
-   webworker.md
    loading-custom-python-code.md
    file-system.md
+   accessing-files.md
+   webworker.md
    service-worker.md
 ```

src/templates/console.html

@@ -253,6 +253,39 @@
         if (searchParams.has("noblink")) {
           $(".cmd-cursor").addClass("noblink");
         }
+
+        let idbkvPromise;
+        async function getIDBKV() {
+          if (!idbkvPromise) {
+            idbkvPromise = await import(
+              "https://unpkg.com/idb-keyval@5.0.2/dist/esm/index.js"
+            );
+          }
+          return idbkvPromise;
+        }
+
+        async function mountDirectory(pyodideDirectory, directoryKey) {
+          if (pyodide.FS.analyzePath(pyodideDirectory).exists) {
+            return;
+          }
+          const { get, set } = await getIDBKV();
+          const opts = {
+            id: "mountdirid",
+            mode: "readwrite",
+          };
+          let directoryHandle = await get(directoryKey);
+          if (!directoryHandle) {
+            directoryHandle = await showDirectoryPicker(opts);
+            await set(directoryKey, directoryHandle);
+          }
+          const permissionStatus =
+            await directoryHandle.requestPermission(opts);
+          if (permissionStatus !== "granted") {
+            throw new Error("readwrite access to directory not granted");
+          }
+          await pyodide.mountNativeFS(pyodideDirectory, directoryHandle);
+        }
+        globalThis.mountDirectory = mountDirectory;
       }
       window.console_ready = main();
     </script>