From fcc2b7a03623c5af596f3c3cf56574ba12931e65 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Wed, 19 Sep 2018 10:42:49 -0400 Subject: [PATCH 1/2] Reorg pyodide ns to separate emscripten details from public API --- Makefile | 2 +- src/pyodide.js | 46 ++++++++++++++++++++++++++++++++------------- test/test_python.py | 4 ++-- tools/buildpkg.py | 2 +- 4 files changed, 37 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index e18c45604..f0f8e20e2 100644 --- a/Makefile +++ b/Makefile @@ -139,7 +139,7 @@ build/test.data: $(CPYTHONLIB) ) ( \ cd build; \ - python $(FILEPACKAGER) test.data --preload ../$(CPYTHONLIB)/test@/lib/python3.7/test --js-output=test.js --export-name=pyodide --exclude \*.wasm.pre --exclude __pycache__ \ + python $(FILEPACKAGER) test.data --preload ../$(CPYTHONLIB)/test@/lib/python3.7/test --js-output=test.js --export-name=pyodide._module --exclude \*.wasm.pre --exclude __pycache__ \ ) uglifyjs build/test.js -o build/test.js diff --git a/src/pyodide.js b/src/pyodide.js index 52f308e31..492570738 100644 --- a/src/pyodide.js +++ b/src/pyodide.js @@ -2,12 +2,6 @@ * The main bootstrap script for loading pyodide. */ -// Regexp for validating package name and URI -var package_name_regexp = '[a-z0-9_][a-z0-9_\-]*' -var package_uri_regexp = - new RegExp('^https?://.*?(' + package_name_regexp + ').js$', 'i'); -var package_name_regexp = new RegExp('^' + package_name_regexp + '$', 'i'); - var languagePluginLoader = new Promise((resolve, reject) => { // This is filled in by the Makefile to be either a local file or the // deployed location. TODO: This should be done in a less hacky @@ -16,9 +10,13 @@ var languagePluginLoader = new Promise((resolve, reject) => { //////////////////////////////////////////////////////////// // Package loading - var packages = undefined; let loadedPackages = new Array(); var loadPackagePromise = new Promise((resolve) => resolve()); + // Regexp for validating package name and URI + var package_name_regexp = '[a-z0-9_][a-z0-9_\-]*' + var package_uri_regexp = + new RegExp('^https?://.*?(' + package_name_regexp + ').js$', 'i'); + var package_name_regexp = new RegExp('^' + package_name_regexp + '$', 'i'); let _uri_to_package_name = (package_uri) => { // Generate a unique package name from URI @@ -36,7 +34,7 @@ var languagePluginLoader = new Promise((resolve, reject) => { let _loadPackage = (names) => { // DFS to find all dependencies of the requested packages - let packages = window.pyodide.packages.dependencies; + let packages = window.pyodide._module.packages.dependencies; let queue = [].concat(names || []); let toLoad = new Array(); while (queue.length) { @@ -86,12 +84,12 @@ var languagePluginLoader = new Promise((resolve, reject) => { resolve('No new packages to load'); } - pyodide.monitorRunDependencies = (n) => { + window.pyodide._module.monitorRunDependencies = (n) => { if (n === 0) { for (let package in toLoad) { loadedPackages[package] = toLoad[package]; } - delete pyodide.monitorRunDependencies; + delete window.pyodide._module.monitorRunDependencies; const packageList = Array.from(Object.keys(toLoad)).join(', '); resolve(`Loaded ${packageList}`); } @@ -130,6 +128,8 @@ var languagePluginLoader = new Promise((resolve, reject) => { return loadPackagePromise; }; + //////////////////////////////////////////////////////////// + // Fix Python recursion limit function fixRecursionLimit(pyodide) { // The Javascript/Wasm call stack may be too small to handle the default // Python call stack limit of 1000 frames. This is generally the case on @@ -156,6 +156,24 @@ var languagePluginLoader = new Promise((resolve, reject) => { `import sys; sys.setrecursionlimit(int(${recursionLimit}))`); }; + //////////////////////////////////////////////////////////// + // Rearrange namespace for public API + let PUBLIC_API = [ + 'loadPackage', + 'pyimport', + 'repr', + 'runPython', + 'version', + ]; + + function makePublicAPI(module, public_api) { + var namespace = {_module : module}; + for (let name of public_api) { + namespace[name] = module[name]; + } + return namespace; + } + //////////////////////////////////////////////////////////// // Loading Pyodide let wasmURL = `${baseURL}pyodide.asm.wasm`; @@ -184,8 +202,9 @@ var languagePluginLoader = new Promise((resolve, reject) => { fetch(`${baseURL}packages.json`) .then((response) => response.json()) .then((json) => { - window.pyodide.packages = json; - fixRecursionLimit(pyodide); + fixRecursionLimit(window.pyodide); + window.pyodide = makePublicAPI(window.pyodide, PUBLIC_API); + window.pyodide._module.packages = json; resolve(); }); }; @@ -231,7 +250,8 @@ var languagePluginLoader = new Promise((resolve, reject) => { // Add a custom output handler for Python objects window.iodide.addOutputHandler({ shouldHandle : (val) => { - return (typeof val === 'function' && pyodide.PyProxy.isPyProxy(val)); + return (typeof val === 'function' && + pyodide._module.PyProxy.isPyProxy(val)); }, render : (val) => { diff --git a/test/test_python.py b/test/test_python.py index 90ab9f8b3..c2cb38dc8 100644 --- a/test/test_python.py +++ b/test/test_python.py @@ -143,10 +143,10 @@ def test_typed_arrays(selenium, wasm_heap, jstype, pytype): else: selenium.run_js( f""" - var buffer = pyodide._malloc( + var buffer = pyodide._module._malloc( 4 * {jstype}.BYTES_PER_ELEMENT); window.array = new {jstype}( - pyodide.HEAPU8.buffer, buffer, 4); + pyodide._module.HEAPU8.buffer, buffer, 4); window.array[0] = 1; window.array[1] = 2; window.array[2] = 3; diff --git a/tools/buildpkg.py b/tools/buildpkg.py index 1ed53df06..bf5f3a306 100755 --- a/tools/buildpkg.py +++ b/tools/buildpkg.py @@ -129,7 +129,7 @@ def package_files(buildpath, srcpath, pkg, args): '--preload', '{}@/'.format(install_prefix), '--js-output={}'.format(name + '.js'), - '--export-name=pyodide', + '--export-name=pyodide._module', '--exclude', '*.wasm.pre', '--exclude', '__pycache__', '--use-preload-plugins'], From 83fa468df3da19e926fc00c94d96d3d8bbad046d Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Wed, 19 Sep 2018 11:34:28 -0400 Subject: [PATCH 2/2] Add comment about bootstrapping --- src/pyodide.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/pyodide.js b/src/pyodide.js index 492570738..a0756e67f 100644 --- a/src/pyodide.js +++ b/src/pyodide.js @@ -228,6 +228,9 @@ var languagePluginLoader = new Promise((resolve, reject) => { let script = document.createElement('script'); script.src = `${baseURL}pyodide.asm.js`; script.onload = () => { + // The emscripten module needs to be at this location for the core + // filesystem to install itself. Once that's complete, it will be replaced + // by the call to `makePublicAPI` with a more limited public API. window.pyodide = pyodide(Module); window.pyodide.loadPackage = loadPackage; };