From f13f3340e0458bb78ff653383352b61a57cb6c17 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Fri, 16 Jul 2021 17:33:25 -0400 Subject: [PATCH] Enable all emscripten file systems: IDBFS, NODEFS, PROXYFS and WORKERFS (#1596) --- Makefile | 4 ++ docs/project/changelog.md | 7 ++- src/js/api.js | 9 +++- src/js/test/filesystem.test.js | 10 ++-- src/tests/test_filesystem.py | 94 ++++++++++++++++++++++++++++++++++ 5 files changed, 117 insertions(+), 7 deletions(-) create mode 100644 src/tests/test_filesystem.py diff --git a/Makefile b/Makefile index 95a36c820..cc22088a0 100644 --- a/Makefile +++ b/Makefile @@ -51,6 +51,10 @@ build/pyodide.asm.js: \ [ -d build ] || mkdir build $(CXX) -s EXPORT_NAME="'_createPyodideModule'" -o build/pyodide.asm.js $(filter %.o,$^) \ $(MAIN_MODULE_LDFLAGS) -s FORCE_FILESYSTEM=1 \ + -lidbfs.js \ + -lnodefs.js \ + -lproxyfs.js \ + -lworkerfs.js \ --preload-file $(CPYTHONLIB)@/lib/python$(PYMAJOR).$(PYMINOR) \ --preload-file src/webbrowser.py@/lib/python$(PYMAJOR).$(PYMINOR)/webbrowser.py \ --preload-file src/_testcapi.py@/lib/python$(PYMAJOR).$(PYMINOR)/_testcapi.py \ diff --git a/docs/project/changelog.md b/docs/project/changelog.md index 5aee8142f..cef418308 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -47,9 +47,14 @@ substitutions: - {{ Enhancement }} Pyodide can experimentally be used in Node.js {pr}`1689` -- {{ Enhancement }} Pyodide now exposes the emscripten `FS` module as `fileSystem`, +- {{ Enhancement }} Pyodide now directly exposes the emscripten `FS` API, allowing for direct manipulation of the in-memory filesystem {pr}`1692` +- {{ Enhancement }} Pyodide's support of emscripten file systems is expanded from + the default `MEMFS` to include `IDBFS`, `NODEFS`, `PROXYFS`, and `WORKERFS`, + allowing for custom persistence strategies depending on execution environment + {pr}`1596` + ## Standard library - The following standard library modules are now available as standalone packages diff --git a/src/js/api.js b/src/js/api.js index adf909fc0..cb2b0dffc 100644 --- a/src/js/api.js +++ b/src/js/api.js @@ -320,13 +320,18 @@ export function makePublicAPI() { * which can be used to extend the in-memory filesystem with features like `persistence * `_. * + * While all of the file systems implementations are enabled, only the default + * ``MEMFS`` is guaranteed to work in all runtime settings. The implementations + * are available as members of ``FS.filesystems``: + * ``IDBFS``, ``NODEFS``, ``PROXYFS``, ``WORKERFS``. + * * @type {FS} The Emscripten File System API. */ - const fileSystem = Module.FS; + const FS = Module.FS; let namespace = { globals, - fileSystem, + FS, pyodide_py, version, loadPackage, diff --git a/src/js/test/filesystem.test.js b/src/js/test/filesystem.test.js index 7b4602a0e..a3c6d38ec 100644 --- a/src/js/test/filesystem.test.js +++ b/src/js/test/filesystem.test.js @@ -1,10 +1,12 @@ import assert from "assert"; -describe("fileSystem", () => { +// for a persistence-related browser test see /src/tests/test_filesystem.py + +describe("FS", () => { const exists = () => { return pyodide.runPython("import os; os.path.exists('/tmp/js-test')"); }; - it("no dir", async () => assert.equal(exists(), false)); - it("mkdir", async () => pyodide.fileSystem.mkdir("/tmp/js-test")); - it("made dir", async () => assert.equal(exists(), true)); + it("no dir", async () => assert.strictEqual(exists(), false)); + it("mkdir", async () => pyodide.FS.mkdir("/tmp/js-test")); + it("made dir", async () => assert.strictEqual(exists(), true)); }); diff --git a/src/tests/test_filesystem.py b/src/tests/test_filesystem.py new file mode 100644 index 000000000..0dacebbf3 --- /dev/null +++ b/src/tests/test_filesystem.py @@ -0,0 +1,94 @@ +"""tests of using the emscripten filesystem API with pyodide + +for a basic nodejs-based test, see src/js/test/filesystem.test.js +""" +import pytest + + +@pytest.mark.skip_refcount_check +@pytest.mark.skip_pyproxy_check +def test_idbfs_persist_code(selenium_standalone): + """can we persist files created by user python code?""" + selenium = selenium_standalone + # create mount + selenium.run_js( + """ + pyodide.FS.mkdir('/lib/python3.9/site-packages/test_idbfs'); + pyodide.FS.mount(pyodide.FS.filesystems.IDBFS, {}, "/lib/python3.9/site-packages/test_idbfs") + """ + ) + # create file in mount + selenium.run_js( + """ + pyodide.runPython(` + import pathlib + p = pathlib.Path('/lib/python3.9/site-packages/test_idbfs/__init__.py') + p.write_text("def test(): return 7") + from importlib import invalidate_caches + invalidate_caches() + from test_idbfs import test + assert test() == 7 + `); + """ + ) + # sync TO idbfs + selenium.run_js( + """ + const error = await new Promise( + (resolve, reject) => pyodide.FS.syncfs(false, resolve) + ); + assert(() => error == null); + """ + ) + # refresh page and re-fixture + selenium.driver.refresh() + selenium.javascript_setup() + selenium.run_js( + """ + window.pyodide = await loadPyodide({ indexURL : './', fullStdLib: false }); + """ + ) + selenium.save_state() + selenium.restore_state() + # idbfs isn't magically loaded + selenium.run_js( + """ + pyodide.runPython(` + from importlib import invalidate_caches + invalidate_caches() + err_type = None + try: + from test_idbfs import test + except Exception as err: + err_type = type(err) + assert err_type is ModuleNotFoundError, err_type + `); + """ + ) + # re-mount + selenium.run_js( + """ + pyodide.FS.mkdir('/lib/python3.9/site-packages/test_idbfs'); + pyodide.FS.mount(pyodide.FS.filesystems.IDBFS, {}, "/lib/python3.9/site-packages/test_idbfs"); + """ + ) + # sync FROM idbfs + selenium.run_js( + """ + const error = await new Promise( + (resolve, reject) => pyodide.FS.syncfs(true, resolve) + ); + assert(() => error == null); + """ + ) + # import file persisted above + selenium.run_js( + """ + pyodide.runPython(` + from importlib import invalidate_caches + invalidate_caches() + from test_idbfs import test + assert test() == 7 + `); + """ + )