diff --git a/docs/api_reference.md b/docs/api_reference.md index 5f2ecc6ef..feb40dc8b 100644 --- a/docs/api_reference.md +++ b/docs/api_reference.md @@ -49,19 +49,19 @@ Either the resulting object or `None`. Load a package or a list of packages over the network. -This makes the files for the package available in the virtual filesystem. +This makes the files for the package available in the virtual filesystem. The package needs to be imported from Python before it can be used. *Parameters* -| name | type | description | -|---------|-----------------|---------------------------------------| -| *names* | {String, Array} | package name, or URL. Can be either a single element, or an array. | - +| name | type | description | +|-------------------|-----------------|---------------------------------------| +| *names* | {String, Array} | package name, or URL. Can be either a single element, or an array. | +| *messageCallback* | function | A callback, called with progress messages. (optional) | *Returns* -Loading is asynchronous, therefore, this returns a promise. +Loading is asynchronous, therefore, this returns a `Promise`. ### pyodide.loadedPackage @@ -137,6 +137,43 @@ Runs a string of code. The last part of the string may be an expression, in whic | *jsresult* | *any* | Result, converted to Javascript | +### pyodide.runPythonAsync(code, messageCallback) + +Runs Python code, possibly asynchronously loading any known packages that the code +chunk imports. + +For example, given the following code chunk + +```python +import numpy as np +x = np.array([1, 2, 3]) +``` + +pyodide will first call `pyodide.loadPackage(['numpy'])`, and then run the code +chunk, returning the result. Since package fetching must happen asyncronously, +this function returns a `Promise` which resolves to the output. For example, to +use: + +```javascript +pyodide.runPythonAsync(code, messageCallback) + .then((output) => handleOutput(output)) +``` + +*Parameters* + +| name | type | description | +|-------------------|----------|--------------------------------| +| *code* | String | Python code to evaluate | +| *messageCallback* | function | Callback given status messages | +| | | (optional) | + +*Returns* + +| name | type | description | +|------------|---------|------------------------------------------| +| *result* | Promise | Resolves to the result of the code chunk | + + ### pyodide.version() Returns the pyodide version. diff --git a/packages/matplotlib/meta.yaml b/packages/matplotlib/meta.yaml index 4f2650ccf..e437119bd 100644 --- a/packages/matplotlib/meta.yaml +++ b/packages/matplotlib/meta.yaml @@ -41,3 +41,4 @@ requirements: test: imports: - matplotlib + - mpl_toolkits diff --git a/pyodide_build/buildall.py b/pyodide_build/buildall.py index cc74b439b..e237e4973 100755 --- a/pyodide_build/buildall.py +++ b/pyodide_build/buildall.py @@ -34,6 +34,7 @@ def build_packages(packagesdir, outputdir, args): # We have to build the packages in the correct order (dependencies first), # so first load in all of the package metadata and build a dependency map. dependencies = {} + import_name_to_package_name = {} for pkgdir in packagesdir.iterdir(): pkgpath = pkgdir / 'meta.yaml' if pkgdir.is_dir() and pkgpath.is_file(): @@ -41,6 +42,9 @@ def build_packages(packagesdir, outputdir, args): name = pkg['package']['name'] reqs = pkg.get('requirements', {}).get('run', []) dependencies[name] = reqs + imports = pkg.get('test', {}).get('imports', [name]) + for imp in imports: + import_name_to_package_name[imp] = name for pkgname in dependencies.keys(): build_package(pkgname, dependencies, packagesdir, outputdir, args) @@ -51,7 +55,10 @@ def build_packages(packagesdir, outputdir, args): # This is done last so the Makefile can use it as a completion token. with open(outputdir / 'packages.json', 'w') as fd: - json.dump({'dependencies': dependencies}, fd) + json.dump({ + 'dependencies': dependencies, + 'import_name_to_package_name': import_name_to_package_name, + }, fd) def make_parser(parser): diff --git a/src/pyodide.js b/src/pyodide.js index 7da634363..e0188221e 100644 --- a/src/pyodide.js +++ b/src/pyodide.js @@ -75,7 +75,7 @@ var languagePluginLoader = new Promise((resolve, reject) => { } // clang-format on - let _loadPackage = (names) => { + let _loadPackage = (names, messageCallback) => { // DFS to find all dependencies of the requested packages let packages = window.pyodide._module.packages.dependencies; let loadedPackages = window.pyodide.loadedPackages; @@ -140,13 +140,17 @@ var languagePluginLoader = new Promise((resolve, reject) => { resolve('No new packages to load'); } + const packageList = Array.from(Object.keys(toLoad)).join(', '); + if (messageCallback !== undefined) { + messageCallback(`Loading ${packageList}`); + } + window.pyodide._module.monitorRunDependencies = (n) => { if (n === 0) { for (let package in toLoad) { window.pyodide.loadedPackages[package] = toLoad[package]; } delete window.pyodide._module.monitorRunDependencies; - const packageList = Array.from(Object.keys(toLoad)).join(', '); if (!isFirefox) { preloadWasm().then(() => {resolve(`Loaded ${packageList}`)}); } else { @@ -181,10 +185,11 @@ var languagePluginLoader = new Promise((resolve, reject) => { return promise; }; - let loadPackage = (names) => { + let loadPackage = (names, messageCallback) => { /* We want to make sure that only one loadPackage invocation runs at any * given time, so this creates a "chain" of promises. */ - loadPackagePromise = loadPackagePromise.then(() => _loadPackage(names)); + loadPackagePromise = + loadPackagePromise.then(() => _loadPackage(names, messageCallback)); return loadPackagePromise; }; @@ -224,6 +229,7 @@ var languagePluginLoader = new Promise((resolve, reject) => { 'pyimport', 'repr', 'runPython', + 'runPythonAsync', 'version', ]; diff --git a/src/pyodide.py b/src/pyodide.py index 099d9487c..4dda010ec 100644 --- a/src/pyodide.py +++ b/src/pyodide.py @@ -2,8 +2,6 @@ A library of helper utilities for connecting Python to the browser environment. """ -from js import XMLHttpRequest - import ast import io @@ -14,6 +12,8 @@ def open_url(url): """ Fetches a given *url* and returns a io.StringIO to access its contents. """ + from js import XMLHttpRequest + req = XMLHttpRequest.new() req.open('GET', url, False) req.send(None) @@ -39,4 +39,22 @@ def eval_code(code, ns): return None -__all__ = ['open_url', 'eval_code'] +def find_imports(code): + """ + Finds the imports in a string of code and returns a list of their package + names. + """ + mod = ast.parse(code) + imports = set() + for node in ast.walk(mod): + if isinstance(node, ast.Import): + for name in node.names: + name = name.name + imports.add(name.split('.')[0]) + elif isinstance(node, ast.ImportFrom): + name = node.module + imports.add(name.split('.')[0]) + return list(imports) + + +__all__ = ['open_url', 'eval_code', 'find_imports'] diff --git a/src/runpython.c b/src/runpython.c index 269f86c40..ca8484b3a 100644 --- a/src/runpython.c +++ b/src/runpython.c @@ -10,6 +10,7 @@ extern PyObject* globals; PyObject* eval_code; +PyObject* find_imports; int _runPython(char* code) @@ -32,18 +33,79 @@ _runPython(char* code) return id; } +int +_findImports(char* code) +{ + PyObject* py_code; + py_code = PyUnicode_FromString(code); + if (py_code == NULL) { + return pythonexc2js(); + } + + PyObject* ret = PyObject_CallFunctionObjArgs(find_imports, py_code, NULL); + + if (ret == NULL) { + return pythonexc2js(); + } + + int id = python2js(ret); + Py_DECREF(ret); + return id; +} + EM_JS(int, runpython_init_js, (), { - Module.runPython = function(code) + Module._runPythonInternal = function(pycode) { - var pycode = allocate(intArrayFromString(code), 'i8', ALLOC_NORMAL); var idresult = Module.__runPython(pycode); - jsresult = Module.hiwire_get_value(idresult); + var jsresult = Module.hiwire_get_value(idresult); Module.hiwire_decref(idresult); _free(pycode); return jsresult; }; - return 0; + Module.runPython = function(code) + { + var pycode = allocate(intArrayFromString(code), 'i8', ALLOC_NORMAL); + return Module._runPythonInternal(pycode); + }; + + Module.runPythonAsync = function(code, messageCallback) + { + var pycode = allocate(intArrayFromString(code), 'i8', ALLOC_NORMAL); + + var idimports = Module.__findImports(pycode); + var jsimports = Module.hiwire_get_value(idimports); + Module.hiwire_decref(idimports); + + var internal = function(resolve, reject) + { + try { + resolve(Module._runPythonInternal(pycode)); + } catch (e) { + reject(e); + } + }; + + if (jsimports.length) { + var packageNames = + window.pyodide._module.packages.import_name_to_package_name; + var packages = {}; + for (var i = 0; i < jsimports.length; ++i) { + var name = jsimports[i]; + // clang-format off + if (packageNames[name] !== undefined) { + // clang-format on + packages[packageNames[name]] = undefined; + } + } + if (Object.keys(packages).length) { + var runInternal = function() { return new Promise(internal); }; + return Module.loadPackage(Object.keys(packages), messageCallback) + .then(runInternal); + } + } + return new Promise(internal); + }; }); int @@ -64,6 +126,11 @@ runpython_init_py() return 1; } + find_imports = PyDict_GetItemString(d, "find_imports"); + if (find_imports == NULL) { + return 1; + } + Py_DECREF(m); Py_DECREF(d); return 0; diff --git a/test/conftest.py b/test/conftest.py index 02096475a..5aaa29636 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -105,6 +105,37 @@ class SeleniumWrapper: return self.run_js( 'return pyodide.runPython({!r})'.format(code)) + def run_async(self, code): + from selenium.common.exceptions import TimeoutException + if isinstance(code, str) and code.startswith('\n'): + # we have a multiline string, fix indentation + code = textwrap.dedent(code) + self.run_js( + """ + window.done = false; + pyodide.runPythonAsync({!r}) + .then(function(output) + {{ window.output = output; window.error = false; }}, + function(output) + {{ window.output = output; window.error = true; }}) + .finally(() => window.done = true); + """.format(code) + ) + try: + self.wait.until(PackageLoaded()) + except TimeoutException as exc: + _display_driver_logs(self.browser, self.driver) + print(self.logs) + raise TimeoutException('runPythonAsync timed out') + return self.run_js( + """ + if (window.error) { + throw window.output; + } + return window.output; + """ + ) + def run_js(self, code): if isinstance(code, str) and code.startswith('\n'): # we have a multiline string, fix indentation diff --git a/test/test_python.py b/test/test_python.py index 313430d53..9435ca9e2 100644 --- a/test/test_python.py +++ b/test/test_python.py @@ -448,3 +448,73 @@ def test_recursive_dict(selenium_standalone): """ ) selenium_standalone.run_js("x = pyodide.pyimport('x')") + + +def test_runpythonasync(selenium_standalone): + output = selenium_standalone.run_async( + """ + import numpy as np + np.zeros(5) + """ + ) + assert list(output) == [0, 0, 0, 0, 0] + + +def test_runpythonasync_different_package_name(selenium_standalone): + output = selenium_standalone.run_async( + """ + import dateutil + dateutil.__version__ + """ + ) + assert isinstance(output, str) + + +def test_runpythonasync_no_imports(selenium_standalone): + output = selenium_standalone.run_async( + """ + 42 + """ + ) + assert output == 42 + + +def test_runpythonasync_missing_import(selenium_standalone): + try: + selenium_standalone.run_async( + """ + import foo + """ + ) + except selenium_standalone.JavascriptException as e: + assert "ModuleNotFoundError" in str(e) + else: + assert False + + +def test_runpythonasync_exception(selenium_standalone): + try: + selenium_standalone.run_async( + """ + 42 / 0 + """ + ) + except selenium_standalone.JavascriptException as e: + assert "ZeroDivisionError" in str(e) + else: + assert False + + +def test_runpythonasync_exception_after_import(selenium_standalone): + try: + selenium_standalone.run_async( + """ + import numpy as np + x = np.empty(5) + 42 / 0 + """ + ) + except selenium_standalone.JavascriptException as e: + assert "ZeroDivisionError" in str(e) + else: + assert False