Rooba
/
pyodide
mirror of https://github.com/pyodide/pyodide.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #286 from madhur-tandon/master 

Check packages against Pyodide ABI version
This commit is contained in:
Michael Droettboom 2019-01-22 14:37:03 -05:00 committed by GitHub
parent 0127416633 cf7072d37b
commit 1049ba78fb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 81 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Makefile
View File

@ -72,7 +72,7 @@ build/pyodide.asm.js: src/main.bc src/jsimport.bc src/jsproxy.bc src/js2python.b
build/pyodide.asm.data: root/.built
( \
cd build; \
python $(FILEPACKAGER) pyodide.asm.data --lz4 --preload ../root/lib@lib --js-output=pyodide.asm.data.js --use-preload-plugins \
python $(FILEPACKAGER) pyodide.asm.data --abi=$(PYODIDE_PACKAGE_ABI) --lz4 --preload ../root/lib@lib --js-output=pyodide.asm.data.js --use-preload-plugins \
)
uglifyjs build/pyodide.asm.data.js -o build/pyodide.asm.data.js
@ -80,11 +80,13 @@ build/pyodide.asm.data: root/.built
build/pyodide_dev.js: src/pyodide.js
cp $< $@
sed -i -e "s#{{DEPLOY}}##g" $@
sed -i -e "s#{{ABI}}#$(PYODIDE_PACKAGE_ABI)#g" $@
build/pyodide.js: src/pyodide.js
cp $< $@
sed -i -e 's#{{DEPLOY}}#https://iodide.io/pyodide-demo/#g' $@
sed -i -e "s#{{ABI}}#$(PYODIDE_PACKAGE_ABI)#g" $@
build/python.html: src/python.html
@ -145,7 +147,7 @@ build/test.data: $(CPYTHONLIB)
)
( \
cd build; \
python $(FILEPACKAGER) test.data --lz4 --preload ../$(CPYTHONLIB)/test@/lib/python3.7/test --js-output=test.js --export-name=pyodide._module --exclude __pycache__ \
python $(FILEPACKAGER) test.data --abi=$(PYODIDE_PACKAGE_ABI) --lz4 --preload ../$(CPYTHONLIB)/test@/lib/python3.7/test --js-output=test.js --export-name=pyodide._module --exclude __pycache__ \
)
uglifyjs build/test.js -o build/test.js

2
Makefile.envs
View File

@ -15,6 +15,8 @@ export HOSTPYTHON=$(HOSTPYTHONROOT)/bin/python3
export TARGETPYTHONROOT=$(PYODIDE_ROOT)/cpython/installs/python-$(PYVERSION)
export PYTHONINCLUDE=$(PYODIDE_ROOT)/cpython/installs/python-$(PYVERSION)/include/python$(PYMINOR)
export PYODIDE_PACKAGE_ABI=1
export SIDE_LDFLAGS=\
-O3 \
-s "BINARYEN_METHOD='native-wasm'" \

2
packages/Makefile
View File

@ -3,7 +3,7 @@ include ../Makefile.envs
all: deps
../bin/pyodide buildall . ../build \
--ldflags="$(SIDE_LDFLAGS)" --host=$(HOSTPYTHONROOT) --target=$(TARGETPYTHONROOT)
--package_abi=$(PYODIDE_PACKAGE_ABI) --ldflags="$(SIDE_LDFLAGS)" --host=$(HOSTPYTHONROOT) --target=$(TARGETPYTHONROOT)
deps:
# Install build dependencies
$(HOSTPYTHON) -m pip install Cython Tempita

3
pyodide_build/buildall.py
View File

@ -69,6 +69,9 @@ def make_parser(parser):
parser.add_argument(
'output', type=str, nargs=1,
help='Output directory in which to put all built packages')
parser.add_argument(
'--package_abi', type=int, required=True,
help='The ABI number for the packages to be built')
parser.add_argument(
'--cflags', type=str, nargs='?', default=common.DEFAULTCFLAGS,
help='Extra compiling flags')

4
pyodide_build/buildpkg.py
View File

@ -127,6 +127,7 @@ def package_files(buildpath, srcpath, pkg, args):
'python',
common.ROOTDIR / 'file_packager.py',
name + '.data',
'--abi={0}'.format(args.package_abi),
'--lz4',
'--preload',
'{}@/'.format(install_prefix),
@ -169,6 +170,9 @@ def make_parser(parser):
parser.add_argument(
'package', type=str, nargs=1,
help="Path to meta.yaml package description")
parser.add_argument(
'--package_abi', type=int, required=True,
help='The ABI number for the package to be built')
parser.add_argument(
'--cflags', type=str, nargs='?', default=common.DEFAULTCFLAGS,
help='Extra compiling flags')

25
src/pyodide.js
View File

@ -159,6 +159,7 @@ var languagePluginLoader = new Promise((resolve, reject) => {
window.pyodide.loadedPackages[package] = toLoad[package];
}
delete window.pyodide._module.monitorRunDependencies;
window.removeEventListener('error', windowErrorHandler);
if (!isFirefox) {
preloadWasm().then(() => {resolve(`Loaded ${packageList}`)});
} else {
@ -167,6 +168,17 @@ var languagePluginLoader = new Promise((resolve, reject) => {
}
};
// Add a handler for any exceptions that are thrown in the process of
// loading a package
var windowErrorHandler = (err) => {
delete window.pyodide._module.monitorRunDependencies;
window.removeEventListener('error', windowErrorHandler);
// Set up a new Promise chain, since this one failed
loadPackagePromise = new Promise((resolve) => resolve());
reject(err.message);
};
window.addEventListener('error', windowErrorHandler);
for (let package in toLoad) {
let script = document.createElement('script');
let package_uri = toLoad[package];
@ -179,7 +191,7 @@ var languagePluginLoader = new Promise((resolve, reject) => {
// If the package_uri fails to load, call monitorRunDependencies twice
// (so packageCounter will still hit 0 and finish loading), and remove
// the package from toLoad so we don't mark it as loaded.
console.log(`Couldn't load package from URL ${script.src}`)
console.error(`Couldn't load package from URL ${script.src}`)
let index = toLoad.indexOf(package);
if (index !== -1) {
toLoad.splice(index, 1);
@ -247,6 +259,7 @@ var languagePluginLoader = new Promise((resolve, reject) => {
'repr',
'runPython',
'runPythonAsync',
'checkABI',
'version',
];
@ -277,6 +290,16 @@ var languagePluginLoader = new Promise((resolve, reject) => {
return {};
};
Module.checkABI = function(ABI_number) {
if (ABI_number !== parseInt('{{ABI}}')) {
var ABI_mismatch_exception =
`ABI numbers differ. Expected {{ABI}}, got ${ABI_number}`;
console.error(ABI_mismatch_exception);
throw ABI_mismatch_exception;
}
return true;
};
Module.locateFile = (path) => baseURL + path;
var postRunPromise = new Promise((resolve, reject) => {
Module.postRun = () => {

37
test/test_package_loading.py
View File

@ -1,7 +1,7 @@
import pytest
from pathlib import Path
import shutil
import re
from pathlib import Path
@pytest.mark.parametrize('active_server', ['main', 'secondary'])
@ -108,6 +108,39 @@ def test_load_packages_sequential(selenium_standalone, packages):
assert selenium.logs.count(f'Loading {packages[1]}') == 1
def test_different_ABI(selenium_standalone):
url = selenium_standalone.server_hostname
port = selenium_standalone.server_port
build_dir = Path(__file__).parent.parent / 'build'
original_file = open('build/numpy.js', 'r+')
original_contents = original_file.read()
original_file.close()
modified_contents = re.sub(r'checkABI\(\d+\)', 'checkABI(-1)',
original_contents)
modified_file = open('build/numpy-broken.js', 'w+')
modified_file.write(modified_contents)
modified_file.close()
try:
selenium_standalone.load_package(
f'http://{url}:{port}/numpy-broken.js'
)
assert 'ABI numbers differ.' in selenium_standalone.logs
finally:
(build_dir / 'numpy-broken.js').unlink()
selenium_standalone.load_package('kiwisolver')
selenium_standalone.run('import kiwisolver')
assert (
selenium_standalone.run('repr(kiwisolver)') ==
"<module 'kiwisolver' from "
"'/lib/python3.7/site-packages/kiwisolver.so'>"
)
def test_load_handle_failure(selenium_standalone):
selenium = selenium_standalone
selenium.load_package('pytz')

8
tools/file_packager.py
View File

@ -144,6 +144,10 @@ for arg in sys.argv[2:]:
leading = 'preload'
elif arg == '--embed':
leading = 'embed'
elif arg.startswith('--abi'):
if '=' in arg:
package_abi = arg.split('=', 1)[1]
leading = ''
elif arg == '--exclude':
leading = 'exclude'
elif arg == '--no-force':
@ -227,6 +231,10 @@ if not from_emcc:
var Module = typeof %(EXPORT_NAME)s !== 'undefined' ? %(EXPORT_NAME)s : {};
''' % {"EXPORT_NAME": export_name}
ret += '''
Module.checkABI({0});
'''.format(package_abi)
ret += '''
if (!Module.expectedDataFileDownloads) {
Module.expectedDataFileDownloads = 0;