From d70f4833e1f3860962194fa94c2713b06e832064 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Tue, 9 Oct 2018 18:47:24 -0400 Subject: [PATCH 1/7] Add a package-loading version of runPython --- docs/api_reference.md | 49 ++++++++++++++++++++++++++++---- src/pyodide.js | 13 ++++++--- src/pyodide.py | 24 ++++++++++++++-- src/runpython.c | 66 +++++++++++++++++++++++++++++++++++++++---- 4 files changed, 134 insertions(+), 18 deletions(-) diff --git a/docs/api_reference.md b/docs/api_reference.md index 5f2ecc6ef..feb40dc8b 100644 --- a/docs/api_reference.md +++ b/docs/api_reference.md @@ -49,19 +49,19 @@ Either the resulting object or `None`. Load a package or a list of packages over the network. -This makes the files for the package available in the virtual filesystem. +This makes the files for the package available in the virtual filesystem. The package needs to be imported from Python before it can be used. *Parameters* -| name | type | description | -|---------|-----------------|---------------------------------------| -| *names* | {String, Array} | package name, or URL. Can be either a single element, or an array. | - +| name | type | description | +|-------------------|-----------------|---------------------------------------| +| *names* | {String, Array} | package name, or URL. Can be either a single element, or an array. | +| *messageCallback* | function | A callback, called with progress messages. (optional) | *Returns* -Loading is asynchronous, therefore, this returns a promise. +Loading is asynchronous, therefore, this returns a `Promise`. ### pyodide.loadedPackage @@ -137,6 +137,43 @@ Runs a string of code. The last part of the string may be an expression, in whic | *jsresult* | *any* | Result, converted to Javascript | +### pyodide.runPythonAsync(code, messageCallback) + +Runs Python code, possibly asynchronously loading any known packages that the code +chunk imports. + +For example, given the following code chunk + +```python +import numpy as np +x = np.array([1, 2, 3]) +``` + +pyodide will first call `pyodide.loadPackage(['numpy'])`, and then run the code +chunk, returning the result. Since package fetching must happen asyncronously, +this function returns a `Promise` which resolves to the output. For example, to +use: + +```javascript +pyodide.runPythonAsync(code, messageCallback) + .then((output) => handleOutput(output)) +``` + +*Parameters* + +| name | type | description | +|-------------------|----------|--------------------------------| +| *code* | String | Python code to evaluate | +| *messageCallback* | function | Callback given status messages | +| | | (optional) | + +*Returns* + +| name | type | description | +|------------|---------|------------------------------------------| +| *result* | Promise | Resolves to the result of the code chunk | + + ### pyodide.version() Returns the pyodide version. diff --git a/src/pyodide.js b/src/pyodide.js index 7da634363..590c51886 100644 --- a/src/pyodide.js +++ b/src/pyodide.js @@ -75,7 +75,7 @@ var languagePluginLoader = new Promise((resolve, reject) => { } // clang-format on - let _loadPackage = (names) => { + let _loadPackage = (names, messageCallback) => { // DFS to find all dependencies of the requested packages let packages = window.pyodide._module.packages.dependencies; let loadedPackages = window.pyodide.loadedPackages; @@ -140,13 +140,17 @@ var languagePluginLoader = new Promise((resolve, reject) => { resolve('No new packages to load'); } + const packageList = Array.from(Object.keys(toLoad)).join(', '); + if (messageCallback !== undefined) { + messageCallback(`Loading ${packageList}`); + } + window.pyodide._module.monitorRunDependencies = (n) => { if (n === 0) { for (let package in toLoad) { window.pyodide.loadedPackages[package] = toLoad[package]; } delete window.pyodide._module.monitorRunDependencies; - const packageList = Array.from(Object.keys(toLoad)).join(', '); if (!isFirefox) { preloadWasm().then(() => {resolve(`Loaded ${packageList}`)}); } else { @@ -181,10 +185,10 @@ var languagePluginLoader = new Promise((resolve, reject) => { return promise; }; - let loadPackage = (names) => { + let loadPackage = (names, messageCallback) => { /* We want to make sure that only one loadPackage invocation runs at any * given time, so this creates a "chain" of promises. */ - loadPackagePromise = loadPackagePromise.then(() => _loadPackage(names)); + loadPackagePromise = loadPackagePromise.then(() => _loadPackage(names, messageCallback)); return loadPackagePromise; }; @@ -224,6 +228,7 @@ var languagePluginLoader = new Promise((resolve, reject) => { 'pyimport', 'repr', 'runPython', + 'runPythonAsync', 'version', ]; diff --git a/src/pyodide.py b/src/pyodide.py index 099d9487c..4dda010ec 100644 --- a/src/pyodide.py +++ b/src/pyodide.py @@ -2,8 +2,6 @@ A library of helper utilities for connecting Python to the browser environment. """ -from js import XMLHttpRequest - import ast import io @@ -14,6 +12,8 @@ def open_url(url): """ Fetches a given *url* and returns a io.StringIO to access its contents. """ + from js import XMLHttpRequest + req = XMLHttpRequest.new() req.open('GET', url, False) req.send(None) @@ -39,4 +39,22 @@ def eval_code(code, ns): return None -__all__ = ['open_url', 'eval_code'] +def find_imports(code): + """ + Finds the imports in a string of code and returns a list of their package + names. + """ + mod = ast.parse(code) + imports = set() + for node in ast.walk(mod): + if isinstance(node, ast.Import): + for name in node.names: + name = name.name + imports.add(name.split('.')[0]) + elif isinstance(node, ast.ImportFrom): + name = node.module + imports.add(name.split('.')[0]) + return list(imports) + + +__all__ = ['open_url', 'eval_code', 'find_imports'] diff --git a/src/runpython.c b/src/runpython.c index 269f86c40..4dff683eb 100644 --- a/src/runpython.c +++ b/src/runpython.c @@ -10,6 +10,7 @@ extern PyObject* globals; PyObject* eval_code; +PyObject* find_imports; int _runPython(char* code) @@ -32,18 +33,68 @@ _runPython(char* code) return id; } +int +_findImports(char *code) +{ + PyObject* py_code; + py_code = PyUnicode_FromString(code); + if (py_code == NULL) { + return pythonexc2js(); + } + + PyObject* ret = + PyObject_CallFunctionObjArgs(find_imports, py_code, NULL); + + if (ret == NULL) { + return pythonexc2js(); + } + + int id = python2js(ret); + Py_DECREF(ret); + return id; +} + EM_JS(int, runpython_init_js, (), { - Module.runPython = function(code) - { - var pycode = allocate(intArrayFromString(code), 'i8', ALLOC_NORMAL); + Module._runPythonInternal = function(pycode) { var idresult = Module.__runPython(pycode); - jsresult = Module.hiwire_get_value(idresult); + var jsresult = Module.hiwire_get_value(idresult); Module.hiwire_decref(idresult); _free(pycode); return jsresult; }; - return 0; + Module.runPython = function(code) + { + var pycode = allocate(intArrayFromString(code), 'i8', ALLOC_NORMAL); + return Module._runPythonInternal(pycode); + }; + + Module.runPythonAsync = function(code, messageCallback) + { + var pycode = allocate(intArrayFromString(code), 'i8', ALLOC_NORMAL); + + var idimports = Module.__findImports(pycode); + var jsimports = Module.hiwire_get_value(idimports); + Module.hiwire_decref(idimports); + + var internal = function() { + return Module._runPythonInternal(pycode); + }; + + if (jsimports.length) { + var packages = window.pyodide._module.packages.dependencies; + var packageFilter = function(name) { + return Object.prototype.hasOwnProperty(packages, name); + }; + jsimports = jsimports.filter(packageFilter); + return Module.loadPackage(jsimports, messageCallback).then(internal); + } else { + var resolve = function(resolve) { + return resolve(); + }; + return new Promise(resolve).then(internal); + } + }; }); int @@ -64,6 +115,11 @@ runpython_init_py() return 1; } + find_imports = PyDict_GetItemString(d, "find_imports"); + if (find_imports == NULL) { + return 1; + } + Py_DECREF(m); Py_DECREF(d); return 0; From 9a926ce896e855dcc4f67ae8bcd11c5d6a6d2611 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Tue, 9 Oct 2018 18:59:38 -0400 Subject: [PATCH 2/7] LINT --- src/pyodide.js | 3 ++- src/runpython.c | 19 ++++++++----------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/pyodide.js b/src/pyodide.js index 590c51886..e0188221e 100644 --- a/src/pyodide.js +++ b/src/pyodide.js @@ -188,7 +188,8 @@ var languagePluginLoader = new Promise((resolve, reject) => { let loadPackage = (names, messageCallback) => { /* We want to make sure that only one loadPackage invocation runs at any * given time, so this creates a "chain" of promises. */ - loadPackagePromise = loadPackagePromise.then(() => _loadPackage(names, messageCallback)); + loadPackagePromise = + loadPackagePromise.then(() => _loadPackage(names, messageCallback)); return loadPackagePromise; }; diff --git a/src/runpython.c b/src/runpython.c index 4dff683eb..693f85c7d 100644 --- a/src/runpython.c +++ b/src/runpython.c @@ -34,7 +34,7 @@ _runPython(char* code) } int -_findImports(char *code) +_findImports(char* code) { PyObject* py_code; py_code = PyUnicode_FromString(code); @@ -42,8 +42,7 @@ _findImports(char *code) return pythonexc2js(); } - PyObject* ret = - PyObject_CallFunctionObjArgs(find_imports, py_code, NULL); + PyObject* ret = PyObject_CallFunctionObjArgs(find_imports, py_code, NULL); if (ret == NULL) { return pythonexc2js(); @@ -55,7 +54,8 @@ _findImports(char *code) } EM_JS(int, runpython_init_js, (), { - Module._runPythonInternal = function(pycode) { + Module._runPythonInternal = function(pycode) + { var idresult = Module.__runPython(pycode); var jsresult = Module.hiwire_get_value(idresult); Module.hiwire_decref(idresult); @@ -77,21 +77,18 @@ EM_JS(int, runpython_init_js, (), { var jsimports = Module.hiwire_get_value(idimports); Module.hiwire_decref(idimports); - var internal = function() { - return Module._runPythonInternal(pycode); - }; + var internal = function() { return Module._runPythonInternal(pycode); }; if (jsimports.length) { var packages = window.pyodide._module.packages.dependencies; - var packageFilter = function(name) { + var packageFilter = function(name) + { return Object.prototype.hasOwnProperty(packages, name); }; jsimports = jsimports.filter(packageFilter); return Module.loadPackage(jsimports, messageCallback).then(internal); } else { - var resolve = function(resolve) { - return resolve(); - }; + var resolve = function(resolve) { return resolve(); }; return new Promise(resolve).then(internal); } }; From 97829d86b5e23facd6fc3c9d9735ff154ca25262 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Wed, 10 Oct 2018 14:16:02 -0400 Subject: [PATCH 3/7] Add another subpackage to matplotlib --- packages/matplotlib/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/matplotlib/meta.yaml b/packages/matplotlib/meta.yaml index 4f2650ccf..e437119bd 100644 --- a/packages/matplotlib/meta.yaml +++ b/packages/matplotlib/meta.yaml @@ -41,3 +41,4 @@ requirements: test: imports: - matplotlib + - mpl_toolkits From 3498159efbfd197ba98a8f3047c9462c7c5f6a61 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Wed, 10 Oct 2018 14:16:14 -0400 Subject: [PATCH 4/7] Record mapping from import name to package name --- pyodide_build/buildall.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pyodide_build/buildall.py b/pyodide_build/buildall.py index cc74b439b..e237e4973 100755 --- a/pyodide_build/buildall.py +++ b/pyodide_build/buildall.py @@ -34,6 +34,7 @@ def build_packages(packagesdir, outputdir, args): # We have to build the packages in the correct order (dependencies first), # so first load in all of the package metadata and build a dependency map. dependencies = {} + import_name_to_package_name = {} for pkgdir in packagesdir.iterdir(): pkgpath = pkgdir / 'meta.yaml' if pkgdir.is_dir() and pkgpath.is_file(): @@ -41,6 +42,9 @@ def build_packages(packagesdir, outputdir, args): name = pkg['package']['name'] reqs = pkg.get('requirements', {}).get('run', []) dependencies[name] = reqs + imports = pkg.get('test', {}).get('imports', [name]) + for imp in imports: + import_name_to_package_name[imp] = name for pkgname in dependencies.keys(): build_package(pkgname, dependencies, packagesdir, outputdir, args) @@ -51,7 +55,10 @@ def build_packages(packagesdir, outputdir, args): # This is done last so the Makefile can use it as a completion token. with open(outputdir / 'packages.json', 'w') as fd: - json.dump({'dependencies': dependencies}, fd) + json.dump({ + 'dependencies': dependencies, + 'import_name_to_package_name': import_name_to_package_name, + }, fd) def make_parser(parser): From 4124f18d1a182d0be222e0a42c7aa7ccc3726498 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Wed, 10 Oct 2018 14:17:29 -0400 Subject: [PATCH 5/7] Fix package name / import name dichotomy. Add testing. --- src/runpython.c | 33 ++++++++++++++------- test/conftest.py | 29 +++++++++++++++++++ test/test_python.py | 70 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 121 insertions(+), 11 deletions(-) diff --git a/src/runpython.c b/src/runpython.c index 693f85c7d..ce086db41 100644 --- a/src/runpython.c +++ b/src/runpython.c @@ -77,20 +77,31 @@ EM_JS(int, runpython_init_js, (), { var jsimports = Module.hiwire_get_value(idimports); Module.hiwire_decref(idimports); - var internal = function() { return Module._runPythonInternal(pycode); }; + var internal = function(resolve, reject) { + try { + resolve(Module._runPythonInternal(pycode)); + } catch (e) { + reject(e); + } + }; if (jsimports.length) { - var packages = window.pyodide._module.packages.dependencies; - var packageFilter = function(name) - { - return Object.prototype.hasOwnProperty(packages, name); - }; - jsimports = jsimports.filter(packageFilter); - return Module.loadPackage(jsimports, messageCallback).then(internal); - } else { - var resolve = function(resolve) { return resolve(); }; - return new Promise(resolve).then(internal); + var packageNames = window.pyodide._module.packages.import_name_to_package_name; + var packages = {}; + for (var i = 0; i < jsimports.length; ++i) { + var name = jsimports[i]; + if (packageNames[name] !== undefined) { + packages[packageNames[name]] = undefined; + } + } + if (Object.keys(packages).length) { + var runInternal = function() { + return new Promise(internal); + }; + return Module.loadPackage(Object.keys(packages), messageCallback).then(runInternal); + } } + return new Promise(internal); }; }); diff --git a/test/conftest.py b/test/conftest.py index 02096475a..1c926d38e 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -105,6 +105,35 @@ class SeleniumWrapper: return self.run_js( 'return pyodide.runPython({!r})'.format(code)) + def run_async(self, code): + from selenium.common.exceptions import TimeoutException + if isinstance(code, str) and code.startswith('\n'): + # we have a multiline string, fix indentation + code = textwrap.dedent(code) + self.run_js( + """ + window.done = false; + pyodide.runPythonAsync({!r}) + .then(function(output) {{ window.output = output; window.error = false; }}, + function(output) {{ window.output = output; window.error = true; }}) + .finally(() => window.done = true); + """.format(code) + ) + try: + self.wait.until(PackageLoaded()) + except TimeoutException as exc: + _display_driver_logs(self.browser, self.driver) + print(self.logs) + raise TimeoutException('runPythonAsync timed out') + return self.run_js( + """ + if (window.error) { + throw window.output; + } + return window.output; + """ + ) + def run_js(self, code): if isinstance(code, str) and code.startswith('\n'): # we have a multiline string, fix indentation diff --git a/test/test_python.py b/test/test_python.py index 83ef16f2c..37783b36d 100644 --- a/test/test_python.py +++ b/test/test_python.py @@ -442,3 +442,73 @@ def test_recursive_dict(selenium_standalone): """ ) selenium_standalone.run_js("x = pyodide.pyimport('x')") + + +def test_runpythonasync(selenium_standalone): + output = selenium_standalone.run_async( + """ + import numpy as np + np.zeros(5) + """ + ) + assert list(output) == [0, 0, 0, 0, 0] + + +def test_runpythonasync_different_package_name(selenium_standalone): + output = selenium_standalone.run_async( + """ + import dateutil + dateutil.__version__ + """ + ) + assert isinstance(output, str) + + +def test_runpythonasync_no_imports(selenium_standalone): + output = selenium_standalone.run_async( + """ + 42 + """ + ) + assert output == 42 + + +def test_runpythonasync_missing_import(selenium_standalone): + try: + selenium_standalone.run_async( + """ + import foo + """ + ) + except selenium_standalone.JavascriptException as e: + assert "ModuleNotFoundError" in str(e) + else: + assert False + + +def test_runpythonasync_exception(selenium_standalone): + try: + selenium_standalone.run_async( + """ + 42 / 0 + """ + ) + except selenium_standalone.JavascriptException as e: + assert "ZeroDivisionError" in str(e) + else: + assert False + + +def test_runpythonasync_exception_after_import(selenium_standalone): + try: + selenium_standalone.run_async( + """ + import numpy as np + x = np.empty(5) + 42 / 0 + """ + ) + except selenium_standalone.JavascriptException as e: + assert "ZeroDivisionError" in str(e) + else: + assert False From 0809f7462249381697be5e46996458dc8b61928d Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Wed, 10 Oct 2018 14:21:45 -0400 Subject: [PATCH 6/7] LINT --- src/runpython.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/runpython.c b/src/runpython.c index ce086db41..ca8484b3a 100644 --- a/src/runpython.c +++ b/src/runpython.c @@ -77,7 +77,8 @@ EM_JS(int, runpython_init_js, (), { var jsimports = Module.hiwire_get_value(idimports); Module.hiwire_decref(idimports); - var internal = function(resolve, reject) { + var internal = function(resolve, reject) + { try { resolve(Module._runPythonInternal(pycode)); } catch (e) { @@ -86,19 +87,21 @@ EM_JS(int, runpython_init_js, (), { }; if (jsimports.length) { - var packageNames = window.pyodide._module.packages.import_name_to_package_name; + var packageNames = + window.pyodide._module.packages.import_name_to_package_name; var packages = {}; for (var i = 0; i < jsimports.length; ++i) { var name = jsimports[i]; + // clang-format off if (packageNames[name] !== undefined) { + // clang-format on packages[packageNames[name]] = undefined; } } if (Object.keys(packages).length) { - var runInternal = function() { - return new Promise(internal); - }; - return Module.loadPackage(Object.keys(packages), messageCallback).then(runInternal); + var runInternal = function() { return new Promise(internal); }; + return Module.loadPackage(Object.keys(packages), messageCallback) + .then(runInternal); } } return new Promise(internal); From 6aa6c9464450a9ca2991b9d0ebb8012694b3c03a Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Wed, 10 Oct 2018 14:38:58 -0400 Subject: [PATCH 7/7] LINT --- test/conftest.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 1c926d38e..5aaa29636 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -114,8 +114,10 @@ class SeleniumWrapper: """ window.done = false; pyodide.runPythonAsync({!r}) - .then(function(output) {{ window.output = output; window.error = false; }}, - function(output) {{ window.output = output; window.error = true; }}) + .then(function(output) + {{ window.output = output; window.error = false; }}, + function(output) + {{ window.output = output; window.error = true; }}) .finally(() => window.done = true); """.format(code) )