From f7aec29209b248a005441de7c41fce86f7bc1bb9 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Tue, 18 Sep 2018 21:46:57 +0200 Subject: [PATCH 1/4] Add unit test to ensure that .data is loaded from the same base url as .js --- test/test_package_loading.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/test/test_package_loading.py b/test/test_package_loading.py index 6183ca078..d4e6e3edb 100644 --- a/test/test_package_loading.py +++ b/test/test_package_loading.py @@ -1,13 +1,29 @@ import pytest -def test_load_from_url(selenium_standalone): +def test_load_from_url(selenium_standalone, web_server_secondary): - url = selenium_standalone.server_hostname - port = selenium_standalone.server_port + url, port, log_secondary = web_server_secondary - selenium_standalone.load_package(f"http://{url}:{port}/pyparsing.js") - assert "Invalid package name or URI" not in selenium_standalone.logs + log_main = selenium_standalone.server_log + + with log_secondary.open('r') as fh_secondary, \ + log_main.open('r') as fh_main: + + # skip existing log lines + fh_main.read() + fh_secondary.read() + + selenium_standalone.load_package(f"http://{url}:{port}/pyparsing.js") + assert "Invalid package name or URI" not in selenium_standalone.logs + + # check that all ressources were loaded from the secondary server + txt = fh_secondary.read() + assert '"GET /pyparsing.js HTTP/1.1" 200' in txt + assert '"GET /pyparsing.data HTTP/1.1" 200' in txt + + # no additional ressources were loaded from the main server + assert len(fh_main.read()) == 0 selenium_standalone.run("from pyparsing import Word, alphas") selenium_standalone.run("Word(alphas).parseString('hello')") From 73e3898b72bd6e820f03f28db9c45f1e79c1730a Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Wed, 19 Sep 2018 18:12:05 +0200 Subject: [PATCH 2/4] Load .data files from the same base URL as .js --- src/pyodide.js | 20 ++++++++++++++++++-- test/conftest.py | 5 +++++ test/test_package_loading.py | 14 ++++++++++++-- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/src/pyodide.js b/src/pyodide.js index a0756e67f..06f4318d7 100644 --- a/src/pyodide.js +++ b/src/pyodide.js @@ -35,8 +35,10 @@ var languagePluginLoader = new Promise((resolve, reject) => { let _loadPackage = (names) => { // DFS to find all dependencies of the requested packages let packages = window.pyodide._module.packages.dependencies; + let loadedPackages = window.pyodide.loadedPackages; let queue = [].concat(names || []); let toLoad = new Array(); + window.pyodide._toLoadPackages = toLoad; while (queue.length) { let package_uri = queue.pop(); @@ -87,7 +89,7 @@ var languagePluginLoader = new Promise((resolve, reject) => { window.pyodide._module.monitorRunDependencies = (n) => { if (n === 0) { for (let package in toLoad) { - loadedPackages[package] = toLoad[package]; + window.pyodide.loadedPackages[package] = toLoad[package]; } delete window.pyodide._module.monitorRunDependencies; const packageList = Array.from(Object.keys(toLoad)).join(', '); @@ -195,7 +197,20 @@ var languagePluginLoader = new Promise((resolve, reject) => { return {}; }; - Module.locateFile = (path) => baseURL + path; + Module.locateFile = (path) => { + if ((window.hasOwnProperty('pyodide')) && + (window.pyodide.hasOwnProperty('_toLoadPackages'))) { + // handle packages loaded from custom URLs + let package = path.replace(/\.data$/, ""); + if (package in window.pyodide._toLoadPackages) { + let package_uri = window.pyodide._toLoadPackages[package]; + if (package_uri != 'default channel') { + return package_uri.replace(/\.js$/, ".data"); + }; + }; + }; + return baseURL + path; + }; var postRunPromise = new Promise((resolve, reject) => { Module.postRun = () => { delete window.Module; @@ -232,6 +247,7 @@ var languagePluginLoader = new Promise((resolve, reject) => { // filesystem to install itself. Once that's complete, it will be replaced // by the call to `makePublicAPI` with a more limited public API. window.pyodide = pyodide(Module); + window.pyodide.loadedPackages = new Array(); window.pyodide.loadPackage = loadPackage; }; document.head.appendChild(script); diff --git a/test/conftest.py b/test/conftest.py index 081e39c64..c1c340ed5 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -272,6 +272,11 @@ def run_web_server(q, log_filepath): *self.client_address, format_ % args)) + def end_headers(self): + # Enable Cross-Origin Resource Sharing (CORS) + self.send_header('Access-Control-Allow-Origin', '*') + super().end_headers() + Handler.extensions_map['.wasm'] = 'application/wasm' with socketserver.TCPServer(("", 0), Handler) as httpd: diff --git a/test/test_package_loading.py b/test/test_package_loading.py index d4e6e3edb..69a189dc1 100644 --- a/test/test_package_loading.py +++ b/test/test_package_loading.py @@ -11,8 +11,8 @@ def test_load_from_url(selenium_standalone, web_server_secondary): log_main.open('r') as fh_main: # skip existing log lines - fh_main.read() - fh_secondary.read() + fh_main.seek(0, 2) + fh_secondary.seek(0, 2) selenium_standalone.load_package(f"http://{url}:{port}/pyparsing.js") assert "Invalid package name or URI" not in selenium_standalone.logs @@ -32,6 +32,16 @@ def test_load_from_url(selenium_standalone, web_server_secondary): selenium_standalone.run("import numpy as np") +def test_list_loaded_urls(selenium_standalone): + selenium = selenium_standalone + + selenium.load_package('pyparsing') + assert selenium.run_js( + 'return Object.keys(pyodide.loadedPackages)') == ['pyparsing'] + assert selenium.run_js( + "return pyodide.loadedPackages['pyparsing']") == "default channel" + + def test_uri_mismatch(selenium_standalone): selenium_standalone.load_package('pyparsing') selenium_standalone.load_package('http://some_url/pyparsing.js') From 6de43c9e19a292e58305a98621379824bcc437aa Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Thu, 20 Sep 2018 11:07:45 +0200 Subject: [PATCH 3/4] Update Module.locateFile inside _loadPackage --- src/pyodide.js | 31 ++++++++++++++++--------------- test/conftest.py | 2 +- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/pyodide.js b/src/pyodide.js index 06f4318d7..99f76ee75 100644 --- a/src/pyodide.js +++ b/src/pyodide.js @@ -38,7 +38,6 @@ var languagePluginLoader = new Promise((resolve, reject) => { let loadedPackages = window.pyodide.loadedPackages; let queue = [].concat(names || []); let toLoad = new Array(); - window.pyodide._toLoadPackages = toLoad; while (queue.length) { let package_uri = queue.pop(); @@ -81,6 +80,20 @@ var languagePluginLoader = new Promise((resolve, reject) => { } } + console.log(`Trying to load ${Object.keys(toLoad)}`); + + window.pyodide._module.locateFile = (path) => { + // handle packages loaded from custom URLs + let package = path.replace(/\.data$/, ""); + if (package in toLoad) { + let package_uri = toLoad[package]; + if (package_uri != 'default channel') { + return package_uri.replace(/\.js$/, ".data"); + }; + }; + return baseURL + path; + }; + let promise = new Promise((resolve, reject) => { if (Object.keys(toLoad).length === 0) { resolve('No new packages to load'); @@ -162,6 +175,7 @@ var languagePluginLoader = new Promise((resolve, reject) => { // Rearrange namespace for public API let PUBLIC_API = [ 'loadPackage', + 'loadedPackages', 'pyimport', 'repr', 'runPython', @@ -197,20 +211,7 @@ var languagePluginLoader = new Promise((resolve, reject) => { return {}; }; - Module.locateFile = (path) => { - if ((window.hasOwnProperty('pyodide')) && - (window.pyodide.hasOwnProperty('_toLoadPackages'))) { - // handle packages loaded from custom URLs - let package = path.replace(/\.data$/, ""); - if (package in window.pyodide._toLoadPackages) { - let package_uri = window.pyodide._toLoadPackages[package]; - if (package_uri != 'default channel') { - return package_uri.replace(/\.js$/, ".data"); - }; - }; - }; - return baseURL + path; - }; + Module.locateFile = (path) => baseURL + path; var postRunPromise = new Promise((resolve, reject) => { Module.postRun = () => { delete window.Module; diff --git a/test/conftest.py b/test/conftest.py index c1c340ed5..090eec6b2 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -112,7 +112,7 @@ class SeleniumWrapper: except TimeoutException as exc: _display_driver_logs(self.browser, self.driver) print(self.logs) - raise TimeoutException() + raise TimeoutException('wait_until_packages_loaded timed out') @property def urls(self): From 227f748795e2fdb0527e6f300808e9700118074c Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Thu, 20 Sep 2018 13:05:21 +0200 Subject: [PATCH 4/4] Test loading from URLs using both the main and secondary web servers --- src/pyodide.js | 2 -- test/test_package_loading.py | 29 +++++++++++++++++++---------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/pyodide.js b/src/pyodide.js index 99f76ee75..20cc21ee3 100644 --- a/src/pyodide.js +++ b/src/pyodide.js @@ -80,8 +80,6 @@ var languagePluginLoader = new Promise((resolve, reject) => { } } - console.log(`Trying to load ${Object.keys(toLoad)}`); - window.pyodide._module.locateFile = (path) => { // handle packages loaded from custom URLs let package = path.replace(/\.data$/, ""); diff --git a/test/test_package_loading.py b/test/test_package_loading.py index 69a189dc1..40fe2bd40 100644 --- a/test/test_package_loading.py +++ b/test/test_package_loading.py @@ -1,29 +1,38 @@ import pytest -def test_load_from_url(selenium_standalone, web_server_secondary): +@pytest.mark.parametrize('active_server', ['main', 'secondary']) +def test_load_from_url(selenium_standalone, web_server_secondary, + active_server): - url, port, log_secondary = web_server_secondary + if active_server == 'secondary': + url, port, log_main = web_server_secondary + log_backup = selenium_standalone.server_log + elif active_server == 'main': + _, _, log_backup = web_server_secondary + log_main = selenium_standalone.server_log + url = selenium_standalone.server_hostname + port = selenium_standalone.server_port + else: + raise AssertionError() - log_main = selenium_standalone.server_log - - with log_secondary.open('r') as fh_secondary, \ + with log_backup.open('r') as fh_backup, \ log_main.open('r') as fh_main: # skip existing log lines fh_main.seek(0, 2) - fh_secondary.seek(0, 2) + fh_backup.seek(0, 2) selenium_standalone.load_package(f"http://{url}:{port}/pyparsing.js") assert "Invalid package name or URI" not in selenium_standalone.logs - # check that all ressources were loaded from the secondary server - txt = fh_secondary.read() + # check that all ressources were loaded from the active server + txt = fh_main.read() assert '"GET /pyparsing.js HTTP/1.1" 200' in txt assert '"GET /pyparsing.data HTTP/1.1" 200' in txt - # no additional ressources were loaded from the main server - assert len(fh_main.read()) == 0 + # no additional ressources were loaded from the other server + assert len(fh_backup.read()) == 0 selenium_standalone.run("from pyparsing import Word, alphas") selenium_standalone.run("Word(alphas).parseString('hello')")