diff --git a/pyodide_build/buildpkg.py b/pyodide_build/buildpkg.py index 7b2ead838..5c7a60ec2 100755 --- a/pyodide_build/buildpkg.py +++ b/pyodide_build/buildpkg.py @@ -5,11 +5,13 @@ Builds a Pyodide package. """ import argparse +import cgi import hashlib import os from pathlib import Path import shutil import subprocess +from urllib import request from datetime import datetime @@ -49,13 +51,21 @@ def download_and_extract(buildpath, packagedir, pkg, args): return srcpath if "url" in pkg["source"]: - tarballpath = buildpath / Path(pkg["source"]["url"]).name + response = request.urlopen(pkg["source"]["url"]) + _, parameters = cgi.parse_header( + response.headers.get("Content-Disposition", "") + ) + if "filename" in parameters: + tarballname = parameters["filename"] + else: + tarballname = Path(response.geturl()).name + + tarballpath = buildpath / tarballname if not tarballpath.is_file(): try: - subprocess.run( - ["wget", "-q", "-O", str(tarballpath), pkg["source"]["url"]], - check=True, - ) + os.makedirs(os.path.dirname(tarballpath), exist_ok=True) + with open(tarballpath, "wb") as f: + f.write(response.read()) check_checksum(tarballpath, pkg) except Exception: tarballpath.unlink() @@ -64,6 +74,21 @@ def download_and_extract(buildpath, packagedir, pkg, args): if not srcpath.is_dir(): shutil.unpack_archive(str(tarballpath), str(buildpath)) + for extension in [ + ".tar.gz", + ".tgz", + ".tar", + ".tar.bz2", + ".tbz2", + ".tar.xz", + ".txz", + ".zip", + ]: + if tarballname.endswith(extension): + tarballname = tarballname[: -len(extension)] + break + + return buildpath / tarballname elif "path" in pkg["source"]: srcdir = Path(pkg["source"]["path"]) @@ -72,11 +97,11 @@ def download_and_extract(buildpath, packagedir, pkg, args): if not srcpath.is_dir(): shutil.copytree(srcdir, srcpath) + + return srcpath else: raise ValueError("Incorrect source provided") - return srcpath - def patch(path, srcpath, pkg, args): if (srcpath / ".patched").is_file(): diff --git a/test/pyodide_build/test_buildpkg.py b/test/pyodide_build/test_buildpkg.py new file mode 100644 index 000000000..276ad75c4 --- /dev/null +++ b/test/pyodide_build/test_buildpkg.py @@ -0,0 +1,35 @@ +import shutil +import subprocess + +from pathlib import Path + +from pyodide_build import buildpkg, common + + +def test_download_and_extract(monkeypatch): + monkeypatch.setattr(subprocess, "run", lambda *args, **kwargs: True) + monkeypatch.setattr(buildpkg, "check_checksum", lambda *args, **kwargs: True) + monkeypatch.setattr(shutil, "unpack_archive", lambda *args, **kwargs: True) + + test_pkgs = [] + + # tarballname == version + test_pkgs.append(common.parse_package("./packages/scipy/meta.yaml")) + test_pkgs.append(common.parse_package("./packages/numpy/meta.yaml")) + + # tarballname != version + test_pkgs.append( + { + "package": {"name": "pyyaml", "version": "5.3.1"}, + "source": { + "url": "https://files.pythonhosted.org/packages/64/c2/b80047c7ac2478f9501676c988a5411ed5572f35d1beff9cae07d321512c/PyYAML-5.3.1.tar.gz" + }, + } + ) + + for pkg in test_pkgs: + packagedir = pkg["package"]["name"] + "-" + pkg["package"]["version"] + buildpath = Path(pkg["package"]["name"]) / "build" + srcpath = buildpkg.download_and_extract(buildpath, packagedir, pkg, args=None) + + assert srcpath.name.lower().endswith(packagedir.lower())