Merge pull request #178 from rth/fix-custom-url-data-v2

Fix load .data files from the same base URL as .js

src/pyodide.js

@@ -35,6 +35,7 @@ var languagePluginLoader = new Promise((resolve, reject) => {
   let _loadPackage = (names) => {
     // DFS to find all dependencies of the requested packages
     let packages = window.pyodide._module.packages.dependencies;
+    let loadedPackages = window.pyodide.loadedPackages;
     let queue = [].concat(names || []);
     let toLoad = new Array();
     while (queue.length) {
@@ -79,6 +80,18 @@ var languagePluginLoader = new Promise((resolve, reject) => {
       }
     }
 
+    window.pyodide._module.locateFile = (path) => {
+      // handle packages loaded from custom URLs
+      let package = path.replace(/\.data$/, "");
+      if (package in toLoad) {
+        let package_uri = toLoad[package];
+        if (package_uri != 'default channel') {
+          return package_uri.replace(/\.js$/, ".data");
+        };
+      };
+      return baseURL + path;
+    };
+
     let promise = new Promise((resolve, reject) => {
       if (Object.keys(toLoad).length === 0) {
         resolve('No new packages to load');
@@ -87,7 +100,7 @@ var languagePluginLoader = new Promise((resolve, reject) => {
       window.pyodide._module.monitorRunDependencies = (n) => {
         if (n === 0) {
           for (let package in toLoad) {
-            loadedPackages[package] = toLoad[package];
+            window.pyodide.loadedPackages[package] = toLoad[package];
           }
           delete window.pyodide._module.monitorRunDependencies;
           const packageList = Array.from(Object.keys(toLoad)).join(', ');
@@ -160,6 +173,7 @@ var languagePluginLoader = new Promise((resolve, reject) => {
   // Rearrange namespace for public API
   let PUBLIC_API = [
     'loadPackage',
+    'loadedPackages',
     'pyimport',
     'repr',
     'runPython',
@@ -232,6 +246,7 @@ var languagePluginLoader = new Promise((resolve, reject) => {
       // filesystem to install itself. Once that's complete, it will be replaced
       // by the call to `makePublicAPI` with a more limited public API.
       window.pyodide = pyodide(Module);
+      window.pyodide.loadedPackages = new Array();
       window.pyodide.loadPackage = loadPackage;
     };
     document.head.appendChild(script);

test/conftest.py

@@ -274,6 +274,11 @@ def run_web_server(q, log_filepath):
                      *self.client_address,
                      format_ % args))
 
+        def end_headers(self):
+            # Enable Cross-Origin Resource Sharing (CORS)
+            self.send_header('Access-Control-Allow-Origin', '*')
+            super().end_headers()
+
     Handler.extensions_map['.wasm'] = 'application/wasm'
 
     with socketserver.TCPServer(("", 0), Handler) as httpd:

test/test_package_loading.py

@@ -1,13 +1,38 @@
 import pytest
 
 
-def test_load_from_url(selenium_standalone):
+@pytest.mark.parametrize('active_server', ['main', 'secondary'])
+def test_load_from_url(selenium_standalone, web_server_secondary,
+                       active_server):
 
-    url = selenium_standalone.server_hostname
-    port = selenium_standalone.server_port
+    if active_server == 'secondary':
+        url, port, log_main = web_server_secondary
+        log_backup = selenium_standalone.server_log
+    elif active_server == 'main':
+        _, _, log_backup = web_server_secondary
+        log_main = selenium_standalone.server_log
+        url = selenium_standalone.server_hostname
+        port = selenium_standalone.server_port
+    else:
+        raise AssertionError()
 
-    selenium_standalone.load_package(f"http://{url}:{port}/pyparsing.js")
-    assert "Invalid package name or URI" not in selenium_standalone.logs
+    with log_backup.open('r') as fh_backup, \
+            log_main.open('r') as fh_main:
+
+        # skip existing log lines
+        fh_main.seek(0, 2)
+        fh_backup.seek(0, 2)
+
+        selenium_standalone.load_package(f"http://{url}:{port}/pyparsing.js")
+        assert "Invalid package name or URI" not in selenium_standalone.logs
+
+        # check that all ressources were loaded from the active server
+        txt = fh_main.read()
+        assert '"GET /pyparsing.js HTTP/1.1" 200' in txt
+        assert '"GET /pyparsing.data HTTP/1.1" 200' in txt
+
+        # no additional ressources were loaded from the other server
+        assert len(fh_backup.read()) == 0
 
     selenium_standalone.run("from pyparsing import Word, alphas")
     selenium_standalone.run("Word(alphas).parseString('hello')")
@@ -16,6 +41,16 @@ def test_load_from_url(selenium_standalone):
     selenium_standalone.run("import numpy as np")
 
 
+def test_list_loaded_urls(selenium_standalone):
+    selenium = selenium_standalone
+
+    selenium.load_package('pyparsing')
+    assert selenium.run_js(
+            'return Object.keys(pyodide.loadedPackages)') == ['pyparsing']
+    assert selenium.run_js(
+            "return pyodide.loadedPackages['pyparsing']") == "default channel"
+
+
 def test_uri_mismatch(selenium_standalone):
     selenium_standalone.load_package('pyparsing')
     selenium_standalone.load_package('http://some_url/pyparsing.js')