Support relative URLs in micropip (#872)

Co-authored-by: Dexter Chua <dec41@srcf.net>

docs/changelog.md

@@ -1,13 +1,17 @@
 (changelog)=
 # Release notes
 
-## Verison 0.17.0
-*Unreleased*
-
+## Verison [Unreleased]
 ### Breaking changes
 
 - Removed iodide-specific code in `pyodide.js`. This breaks compatibility with
   iodide.
+  [#878](https://github.com/iodide-project/pyodide/pull/878),
+  [#981](https://github.com/iodide-project/pyodide/pull/981)
+
+### Added
+- `micropip` now supports installing wheels from relative urls. [#872](https://github.com/iodide-project/pyodide/pull/872)
+
 
 ## Version 0.16.1
 *December 25, 2020*

docs/loading_packages.md

@@ -1,29 +1,28 @@
 (loading_packages)=
 # Loading packages
 
-Only the Python standard library and six are available after importing Pyodide.
-To use other libraries, you’ll need to load their package using either,
+Only the Python standard library and [six](https://pypi.org/project/six/) are 
+available after importing Pyodide.
+To use other packages, you’ll need to load them using either:
  - {ref}`pyodide.loadPackage <js_api_pyodide_loadPackage>` for packages built
-   with pyodide.
+   with pyodide, or 
  - `micropip.install` for pure Python packages with wheels available on PyPi or
-   on other URLs.
+   from other URLs.
 
 ```{note}
-Note that `micropip` can also be used to load packages built in pyodide (in
-which case it relies on {ref}`pyodide.loadPackage
-<js_api_pyodide_loadPackage>`).
+`micropip` can also be used to load packages built in pyodide (in
+which case it relies on {ref}`pyodide.loadPackage <js_api_pyodide_loadPackage>`).
 ```
 
 Alternatively you can run Python code without manually pre-loading packages.
-You can do this with {ref}`pyodide.runPythonAsync
-<api_pyodide_runPythonAsync>`) function, which will automatically download all
-packages that the code snippet imports. It only supports packages included in
-Pyodide (not on PyPi) at present.
+You can do this with {ref}`pyodide.runPythonAsync <js_api_pyodide_runPythonAsync>` 
+which will automatically download all packages that the code snippet imports. 
+It only supports packages included in Pyodide (not on PyPi) at present.
 
 ## Loading packages with pyodide.loadPackage
 
 Packages can be loaded by name, for those included in the official pyodide
-repository using,
+repository using e.g.,
 ```js
 pyodide.loadPackage('numpy')
 ```
@@ -83,16 +82,16 @@ micropip.install(
     'https://example.com/files/snowballstemmer-2.0.0-py2.py3-none-any.whl'
 )
 ```
-
+Micropip currently decides whether a file is a url based on whether it ends in ".whl" or not.
 The wheel name in the URL must follow [PEP 427 naming
 convention](https://www.python.org/dev/peps/pep-0427/#file-format), which will
 be the case if the wheels is made using standard python tools (`pip wheel`,
 `setup.py bdist_wheel`).
 
 All required dependencies need also to be previously installed with `micropip`
-or `pyodide.loadPackage`.
+or {ref}`pyodide.loadPackage <js_api_pyodide_loadPackage>`.
 
-The remote server must set Cross-Origin Resource Sharing (CORS) headers to
+If the file is on a remote server, it must set Cross-Origin Resource Sharing (CORS) headers to
 allow access. Otherwise, you can prepend a CORS proxy to the URL. Note however
 that using third-party CORS proxies has security implications, particularly
 since we are not able to check the file integrity, unlike with installs from

packages/micropip/micropip/micropip.py

@@ -198,7 +198,7 @@ class _PackageManager:
             self.installed_packages[name] = ver
 
     def add_requirement(self, requirement: str, ctx, transaction):
-        if requirement.startswith(("http://", "https://")):
+        if requirement.endswith(".whl"):
             # custom download location
             name, wheel, version = _parse_wheel_url(requirement)
             transaction["wheels"].append((name, wheel, version))
@@ -268,23 +268,28 @@ del _PackageManager
 def install(requirements: Union[str, List[str]]):
     """Install the given package and all of its dependencies.
 
-    This only works for pure Python wheels or for packages built
-    in pyodide. If a package is not found in the pyodide repository
-    it will be loaded from PyPi.
+    See :ref:`loading packages <loading_packages>` for more information.
+
+    This only works for packages that are either pure Python or for packages with
+    C extensions that are built in pyodide. If a pure Python package is not found
+    in the pyodide repository it will be loaded from PyPi.
 
     Parameters
     ----------
     requirements
-       a requirements or a list of requirements to install.
-       Can be composed either of
+       A requirement or list of requirements to install.
+       Each requirement is a string.
 
-         - package names, as defined in pyodide repository or on PyPi
-         - URLs pointing to pure Python wheels. The file name of such wheels
-           end with ``none-any.whl``.
+         - If the requirement ends in ".whl", the file will be interpreted as a url.
+           The file must be a wheel named in compliance with the
+           [PEP 427 naming convention](https://www.python.org/dev/peps/pep-0427/#file-format)
+
+         - A package name. A package by this name must either be present in the pyodide
+           repository at `languagePluginUrl` or on PyPi.
 
     Returns
     -------
-    a Promise that resolves when all packages have downloaded and installed.
+    A Promise that resolves when all packages have been downloaded and installed.
     """
 
     def do_install(resolve, reject):

packages/micropip/test_micropip.py

@@ -71,6 +71,43 @@ def test_install_custom_url(selenium_standalone, web_server_tst_data):
     selenium_standalone.run("import snowballstemmer")
 
 
+def test_add_requirement_relative_url():
+    pytest.importorskip("distlib")
+    import micropip
+
+    transaction = {"wheels": []}
+    micropip.PACKAGE_MANAGER.add_requirement(
+        "./snowballstemmer-2.0.0-py2.py3-none-any.whl", {}, transaction
+    )
+    [name, req, version] = transaction["wheels"][0]
+    assert name == "snowballstemmer"
+    assert version == "2.0.0"
+    assert req["filename"] == "snowballstemmer-2.0.0-py2.py3-none-any.whl"
+    assert req["packagetype"] == "bdist_wheel"
+    assert req["python_version"] == "py2.py3"
+    assert req["abi_tag"] == "none"
+    assert req["platform"] == "any"
+    assert req["url"] == "./snowballstemmer-2.0.0-py2.py3-none-any.whl"
+
+
+def test_install_custom_relative_url(selenium_standalone):
+    selenium_standalone.load_package("micropip")
+    selenium_standalone.run("import micropip")
+
+    root = Path(__file__).resolve().parents[2]
+    src = root / "src" / "tests" / "data"
+    target = root / "build" / "test_data"
+    target.symlink_to(src, True)
+    try:
+        url = "./test_data/snowballstemmer-2.0.0-py2.py3-none-any.whl"
+        selenium_standalone.run(f"micropip.install('{url}')")
+        # wait untill micropip is loaded
+        time.sleep(1)
+        selenium_standalone.run("import snowballstemmer")
+    finally:
+        target.unlink()
+
+
 def test_last_version_from_pypi():
     pytest.importorskip("distlib")
     import micropip