From ac68a30511f8255e4a73a0a8e583b39dcf8c5735 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Mon, 11 May 2020 01:32:59 +0200 Subject: [PATCH] Supporty custom wheel locations in micropip --- conftest.py | 5 ++- packages/micropip/micropip/micropip.py | 58 +++++++++++++++++++++++--- packages/micropip/test_micropip.py | 44 +++++++++++++++++++ 3 files changed, 101 insertions(+), 6 deletions(-) diff --git a/conftest.py b/conftest.py index 44069dc1b..5792e0a85 100644 --- a/conftest.py +++ b/conftest.py @@ -97,7 +97,10 @@ class SeleniumWrapper: @property def logs(self): logs = self.driver.execute_script("return window.logs") - return '\n'.join(str(x) for x in logs) + if logs is not None: + return '\n'.join(str(x) for x in logs) + else: + return "" def clean_logs(self): self.driver.execute_script("window.logs = []") diff --git a/packages/micropip/micropip/micropip.py b/packages/micropip/micropip/micropip.py index 696447451..220c15cd4 100644 --- a/packages/micropip/micropip/micropip.py +++ b/packages/micropip/micropip/micropip.py @@ -2,7 +2,17 @@ try: from js import Promise, XMLHttpRequest except ImportError: XMLHttpRequest = None -from js import pyodide as js_pyodide + +try: + from js import pyodide as js_pyodide +except ImportError: + + class js_pyodide: # type: ignore + """A mock object to allow import of this package outside pyodide""" + class _module: + class packages: + dependencies = [] # type: ignore + import hashlib import importlib @@ -10,6 +20,7 @@ import io import json from pathlib import Path import zipfile +from typing import Dict, Any, Union, List, Tuple from distlib import markers, util, version @@ -64,12 +75,43 @@ def _get_pypi_json(pkgname): return json.load(fd) +def _parse_wheel_url(url: str) -> Tuple[str, Dict[str, Any], str]: + """Parse wheels url and extract available metadata + + See https://www.python.org/dev/peps/pep-0427/#file-name-convention + """ + file_name = Path(url).name + # also strip '.whl' extension. + wheel_name = Path(url).stem + tokens = wheel_name.split('-') + # TODO: support optional build tags in the filename (cf PEP 427) + if len(tokens) < 5: + raise ValueError(f'{file_name} is not a valid wheel file name.') + version, python_tag, abi_tag, platform = tokens[-4:] + name = '-'.join(tokens[:-4]) + wheel = { + 'digests': None, # checksums not available + 'filename': file_name, + 'packagetype': 'bdist_wheel', + 'python_version': python_tag, + 'abi_tag': abi_tag, + 'platform': platform, + 'url': url, + } + + return name, wheel, version + + class _WheelInstaller: def extract_wheel(self, fd): with zipfile.ZipFile(fd) as zf: zf.extractall(WHEEL_BASE) def validate_wheel(self, data, fileinfo): + if fileinfo.get('digests') is None: + # No checksums available, e.g. because installing + # from a different location than PyPi. + return sha256 = fileinfo['digests']['sha256'] m = hashlib.sha256() m.update(data.getvalue()) @@ -108,7 +150,7 @@ class _PackageManager: def install( self, - requirements, + requirements: Union[str, List[str]], ctx=None, wheel_installer=None, resolve=_nullop, @@ -127,7 +169,7 @@ class _PackageManager: if isinstance(requirements, str): requirements = [requirements] - transaction = { + transaction: Dict[str, Any] = { 'wheels': [], 'pyodide_packages': set(), 'locked': dict(self.installed_packages) @@ -162,7 +204,13 @@ class _PackageManager: wheel_installer(name, wheel, do_resolve, reject) self.installed_packages[name] = ver - def add_requirement(self, requirement, ctx, transaction): + def add_requirement(self, requirement: str, ctx, transaction): + if requirement.startswith(('http://', 'https://')): + # custom download location + name, wheel, version = _parse_wheel_url(requirement) + transaction['wheels'].append((name, wheel, version)) + return + req = util.parse_requirement(requirement) # If it's a Pyodide package, use that instead of the one on PyPI @@ -223,7 +271,7 @@ PACKAGE_MANAGER = _PackageManager() del _PackageManager -def install(requirements): +def install(requirements: Union[str, List[str]]): """ Install the given package and all of its dependencies. diff --git a/packages/micropip/test_micropip.py b/packages/micropip/test_micropip.py index 612411df2..7b74369a5 100644 --- a/packages/micropip/test_micropip.py +++ b/packages/micropip/test_micropip.py @@ -1,4 +1,10 @@ import time +import sys +from pathlib import Path + +import pytest + +sys.path.append(str(Path(__file__).resolve().parent / 'micropip')) def test_install_simple(selenium_standalone): @@ -24,3 +30,41 @@ def test_install_simple(selenium_standalone): "stemmer.stemWords('go going goes gone'.split())") == [ 'go', 'go', 'goe', 'gone' ] + print(selenium_standalone.logs) + + +def test_parse_wheel_url(): + import micropip + + url = "https://a/snowballstemmer-2.0.0-py2.py3-none-any.whl" + name, wheel, version = micropip._parse_wheel_url(url) + assert name == 'snowballstemmer' + assert version == '2.0.0' + assert wheel == { + 'digests': None, + 'filename': 'snowballstemmer-2.0.0-py2.py3-none-any.whl', + 'packagetype': 'bdist_wheel', + 'python_version': 'py2.py3', + 'abi_tag': 'none', + 'platform': 'any', + 'url': url + } + + msg = "not a valid wheel file name" + with pytest.raises(ValueError, match=msg): + url = "https://a/snowballstemmer-2.0.0-py2.whl" + name, params, version = micropip._parse_wheel_url(url) + + url = "http://scikit_learn-0.22.2.post1-cp35-cp35m-macosx_10_9_intel.whl" + name, wheel, version = micropip._parse_wheel_url(url) + assert name == 'scikit_learn' + assert wheel['platform'] == 'macosx_10_9_intel' + + +def test_install_custom_url(selenium_standalone, web_server_secondary): + server_hostname, server_port, server_log = web_server_secondary + selenium_standalone.load_package("micropip") + selenium_standalone.run("import micropip") + base_url = f'http://{server_hostname}:{server_port}/test/' + url = base_url + 'snowballstemmer-2.0.0-py2.py3-none-any.whl' + selenium_standalone.run(f"micropip.install('{url}')")