Merge pull request #652 from rth/arbitrary-wheel-location

Support installing from URLs in micropip

conftest.py

@@ -97,7 +97,10 @@ class SeleniumWrapper:
     @property
     def logs(self):
         logs = self.driver.execute_script("return window.logs")
-        return '\n'.join(str(x) for x in logs)
+        if logs is not None:
+            return '\n'.join(str(x) for x in logs)
+        else:
+            return ""
 
     def clean_logs(self):
         self.driver.execute_script("window.logs = []")

docs/pypi.md

@@ -22,7 +22,7 @@ For use outside of Iodide (just Python), you can use the `then` method on the
 `Promise` that `micropip.install` returns to do work once the packages have
 finished loading:
 
-```
+```py
 def do_work(*args):
     import snowballstemmer
     stemmer = snowballstemmer.stemmer('english')
@@ -32,6 +32,34 @@ import micropip
 micropip.install('snowballstemmer').then(do_work)
 ```
 
+Micropip implements file integrity validation by checking the hash of the
+downloaded wheel against pre-recorded hash digests from the PyPi JSON API.
+
+## Installing wheels from arbitrary URLs
+
+Pure python wheels can also be installed from any URL with micropip,
+```py
+import micropip
+micropip.install(
+    'https://example.com/files/snowballstemmer-2.0.0-py2.py3-none-any.whl'
+)
+```
+
+The wheel name in the URL must follow [PEP 427 naming
+convention](https://www.python.org/dev/peps/pep-0427/#file-format), which will
+be the case if the wheels is made using standard python tools (`pip wheel`,
+`setup.py bdist_wheel`).
+
+All required dependencies need also to be previously installed with `micropip`
+or `pyodide.loadPackage`.
+
+The remote server must set Cross-Origin Resource Sharing (CORS) headers to
+allow access. Otherwise, you can prepend a CORS proxy to the URL. Note however
+that using third-party CORS proxies has security implications, particularly
+since we are not able to check the file integrity, unlike with installs from
+PyPi.
+
+
 ## Complete example
 
 Adapting the setup from the section on ["using pyodide from

packages/micropip/micropip/micropip.py

@@ -2,7 +2,17 @@ try:
     from js import Promise, XMLHttpRequest
 except ImportError:
     XMLHttpRequest = None
-from js import pyodide as js_pyodide
+
+try:
+    from js import pyodide as js_pyodide
+except ImportError:
+
+    class js_pyodide:  # type: ignore
+        """A mock object to allow import of this package outside pyodide"""
+        class _module:
+            class packages:
+                dependencies = []  # type: ignore
+
 
 import hashlib
 import importlib
@@ -10,6 +20,7 @@ import io
 import json
 from pathlib import Path
 import zipfile
+from typing import Dict, Any, Union, List, Tuple
 
 from distlib import markers, util, version
 
@@ -64,12 +75,43 @@ def _get_pypi_json(pkgname):
     return json.load(fd)
 
 
+def _parse_wheel_url(url: str) -> Tuple[str, Dict[str, Any], str]:
+    """Parse wheels url and extract available metadata
+
+    See https://www.python.org/dev/peps/pep-0427/#file-name-convention
+    """
+    file_name = Path(url).name
+    # also strip '.whl' extension.
+    wheel_name = Path(url).stem
+    tokens = wheel_name.split('-')
+    # TODO: support optional build tags in the filename (cf PEP 427)
+    if len(tokens) < 5:
+        raise ValueError(f'{file_name} is not a valid wheel file name.')
+    version, python_tag, abi_tag, platform = tokens[-4:]
+    name = '-'.join(tokens[:-4])
+    wheel = {
+        'digests': None,  # checksums not available
+        'filename': file_name,
+        'packagetype': 'bdist_wheel',
+        'python_version': python_tag,
+        'abi_tag': abi_tag,
+        'platform': platform,
+        'url': url,
+    }
+
+    return name, wheel, version
+
+
 class _WheelInstaller:
     def extract_wheel(self, fd):
         with zipfile.ZipFile(fd) as zf:
             zf.extractall(WHEEL_BASE)
 
     def validate_wheel(self, data, fileinfo):
+        if fileinfo.get('digests') is None:
+            # No checksums available, e.g. because installing
+            # from a different location than PyPi.
+            return
         sha256 = fileinfo['digests']['sha256']
         m = hashlib.sha256()
         m.update(data.getvalue())
@@ -108,7 +150,7 @@ class _PackageManager:
 
     def install(
             self,
-            requirements,
+            requirements: Union[str, List[str]],
             ctx=None,
             wheel_installer=None,
             resolve=_nullop,
@@ -127,7 +169,7 @@ class _PackageManager:
             if isinstance(requirements, str):
                 requirements = [requirements]
 
-            transaction = {
+            transaction: Dict[str, Any] = {
                 'wheels': [],
                 'pyodide_packages': set(),
                 'locked': dict(self.installed_packages)
@@ -162,7 +204,13 @@ class _PackageManager:
             wheel_installer(name, wheel, do_resolve, reject)
             self.installed_packages[name] = ver
 
-    def add_requirement(self, requirement, ctx, transaction):
+    def add_requirement(self, requirement: str, ctx, transaction):
+        if requirement.startswith(('http://', 'https://')):
+            # custom download location
+            name, wheel, version = _parse_wheel_url(requirement)
+            transaction['wheels'].append((name, wheel, version))
+            return
+
         req = util.parse_requirement(requirement)
 
         # If it's a Pyodide package, use that instead of the one on PyPI
@@ -223,7 +271,7 @@ PACKAGE_MANAGER = _PackageManager()
 del _PackageManager
 
 
-def install(requirements):
+def install(requirements: Union[str, List[str]]):
     """
     Install the given package and all of its dependencies.
 

packages/micropip/test_micropip.py

@@ -1,4 +1,10 @@
 import time
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.append(str(Path(__file__).resolve().parent / 'micropip'))
 
 
 def test_install_simple(selenium_standalone):
@@ -24,3 +30,41 @@ def test_install_simple(selenium_standalone):
         "stemmer.stemWords('go going goes gone'.split())") == [
             'go', 'go', 'goe', 'gone'
         ]
+
+
+def test_parse_wheel_url():
+    import micropip
+
+    url = "https://a/snowballstemmer-2.0.0-py2.py3-none-any.whl"
+    name, wheel, version = micropip._parse_wheel_url(url)
+    assert name == 'snowballstemmer'
+    assert version == '2.0.0'
+    assert wheel == {
+        'digests': None,
+        'filename': 'snowballstemmer-2.0.0-py2.py3-none-any.whl',
+        'packagetype': 'bdist_wheel',
+        'python_version': 'py2.py3',
+        'abi_tag': 'none',
+        'platform': 'any',
+        'url': url
+    }
+
+    msg = "not a valid wheel file name"
+    with pytest.raises(ValueError, match=msg):
+        url = "https://a/snowballstemmer-2.0.0-py2.whl"
+        name, params, version = micropip._parse_wheel_url(url)
+
+    url = "http://scikit_learn-0.22.2.post1-cp35-cp35m-macosx_10_9_intel.whl"
+    name, wheel, version = micropip._parse_wheel_url(url)
+    assert name == 'scikit_learn'
+    assert wheel['platform'] == 'macosx_10_9_intel'
+
+
+def test_install_custom_url(selenium_standalone, web_server_secondary):
+    server_hostname, server_port, server_log = web_server_secondary
+    selenium_standalone.load_package("micropip")
+    selenium_standalone.run("import micropip")
+    base_url = f'http://{server_hostname}:{server_port}/test/data/'
+    url = base_url + 'snowballstemmer-2.0.0-py2.py3-none-any.whl'
+    selenium_standalone.run(f"micropip.install('{url}')")
+    selenium_standalone.run("import snowballstemmer")