Enable all emscripten file systems: IDBFS, NODEFS, PROXYFS and WORKERFS (#1596)

Makefile

@@ -51,6 +51,10 @@ build/pyodide.asm.js: \
 	[ -d build ] || mkdir build
 	$(CXX) -s EXPORT_NAME="'_createPyodideModule'" -o build/pyodide.asm.js $(filter %.o,$^) \
 		$(MAIN_MODULE_LDFLAGS) -s FORCE_FILESYSTEM=1 \
+		-lidbfs.js \
+		-lnodefs.js \
+		-lproxyfs.js \
+		-lworkerfs.js \
 		--preload-file $(CPYTHONLIB)@/lib/python$(PYMAJOR).$(PYMINOR) \
 		--preload-file src/webbrowser.py@/lib/python$(PYMAJOR).$(PYMINOR)/webbrowser.py \
 		--preload-file src/_testcapi.py@/lib/python$(PYMAJOR).$(PYMINOR)/_testcapi.py \

docs/project/changelog.md

@@ -47,9 +47,14 @@ substitutions:
 
 - {{ Enhancement }} Pyodide can experimentally be used in Node.js {pr}`1689`
 
-- {{ Enhancement }} Pyodide now exposes the emscripten `FS` module as `fileSystem`,
+- {{ Enhancement }} Pyodide now directly exposes the emscripten `FS` API,
   allowing for direct manipulation of the in-memory filesystem {pr}`1692`
 
+- {{ Enhancement }} Pyodide's support of emscripten file systems is expanded from
+  the default `MEMFS` to include `IDBFS`, `NODEFS`, `PROXYFS`, and `WORKERFS`,
+  allowing for custom persistence strategies depending on execution environment
+  {pr}`1596`
+
 ## Standard library
 
 - The following standard library modules are now available as standalone packages

src/js/api.js

@@ -320,13 +320,18 @@ export function makePublicAPI() {
    * which can be used to extend the in-memory filesystem with features like `persistence
    * <https://emscripten.org/docs/api_reference/Filesystem-API.html#persistent-data>`_.
    *
+   * While all of the file systems implementations are enabled, only the default
+   * ``MEMFS`` is guaranteed to work in all runtime settings. The implementations
+   * are available as members of ``FS.filesystems``:
+   * ``IDBFS``, ``NODEFS``, ``PROXYFS``, ``WORKERFS``.
+   *
    * @type {FS} The Emscripten File System API.
    */
-  const fileSystem = Module.FS;
+  const FS = Module.FS;
 
   let namespace = {
     globals,
-    fileSystem,
+    FS,
     pyodide_py,
     version,
     loadPackage,

src/js/test/filesystem.test.js

@@ -1,10 +1,12 @@
 import assert from "assert";
 
-describe("fileSystem", () => {
+// for a persistence-related browser test see /src/tests/test_filesystem.py
+
+describe("FS", () => {
   const exists = () => {
     return pyodide.runPython("import os; os.path.exists('/tmp/js-test')");
   };
-  it("no dir", async () => assert.equal(exists(), false));
-  it("mkdir", async () => pyodide.fileSystem.mkdir("/tmp/js-test"));
-  it("made dir", async () => assert.equal(exists(), true));
+  it("no dir", async () => assert.strictEqual(exists(), false));
+  it("mkdir", async () => pyodide.FS.mkdir("/tmp/js-test"));
+  it("made dir", async () => assert.strictEqual(exists(), true));
 });

src/tests/test_filesystem.py

@@ -0,0 +1,94 @@
+"""tests of using the emscripten filesystem API with pyodide
+
+for a basic nodejs-based test, see src/js/test/filesystem.test.js
+"""
+import pytest
+
+
+@pytest.mark.skip_refcount_check
+@pytest.mark.skip_pyproxy_check
+def test_idbfs_persist_code(selenium_standalone):
+    """can we persist files created by user python code?"""
+    selenium = selenium_standalone
+    # create mount
+    selenium.run_js(
+        """
+        pyodide.FS.mkdir('/lib/python3.9/site-packages/test_idbfs');
+        pyodide.FS.mount(pyodide.FS.filesystems.IDBFS, {}, "/lib/python3.9/site-packages/test_idbfs")
+        """
+    )
+    # create file in mount
+    selenium.run_js(
+        """
+        pyodide.runPython(`
+            import pathlib
+            p = pathlib.Path('/lib/python3.9/site-packages/test_idbfs/__init__.py')
+            p.write_text("def test(): return 7")
+            from importlib import invalidate_caches
+            invalidate_caches()
+            from test_idbfs import test
+            assert test() == 7
+        `);
+        """
+    )
+    # sync TO idbfs
+    selenium.run_js(
+        """
+        const error = await new Promise(
+            (resolve, reject) => pyodide.FS.syncfs(false, resolve)
+        );
+        assert(() => error == null);
+        """
+    )
+    # refresh page and re-fixture
+    selenium.driver.refresh()
+    selenium.javascript_setup()
+    selenium.run_js(
+        """
+        window.pyodide = await loadPyodide({ indexURL : './', fullStdLib: false });
+        """
+    )
+    selenium.save_state()
+    selenium.restore_state()
+    # idbfs isn't magically loaded
+    selenium.run_js(
+        """
+        pyodide.runPython(`
+            from importlib import invalidate_caches
+            invalidate_caches()
+            err_type = None
+            try:
+                from test_idbfs import test
+            except Exception as err:
+                err_type = type(err)
+            assert err_type is ModuleNotFoundError, err_type
+        `);
+        """
+    )
+    # re-mount
+    selenium.run_js(
+        """
+        pyodide.FS.mkdir('/lib/python3.9/site-packages/test_idbfs');
+        pyodide.FS.mount(pyodide.FS.filesystems.IDBFS, {}, "/lib/python3.9/site-packages/test_idbfs");
+        """
+    )
+    # sync FROM idbfs
+    selenium.run_js(
+        """
+        const error = await new Promise(
+            (resolve, reject) => pyodide.FS.syncfs(true, resolve)
+        );
+        assert(() => error == null);
+        """
+    )
+    # import file persisted above
+    selenium.run_js(
+        """
+        pyodide.runPython(`
+            from importlib import invalidate_caches
+            invalidate_caches()
+            from test_idbfs import test
+            assert test() == 7
+        `);
+        """
+    )