Infer package tarball directory from source url (#687)

Co-authored-by: Roman Yurchak <rth.yurchak@gmail.com>

pyodide_build/buildpkg.py

@@ -5,11 +5,13 @@ Builds a Pyodide package.
 """
 
 import argparse
+import cgi
 import hashlib
 import os
 from pathlib import Path
 import shutil
 import subprocess
+from urllib import request
 from datetime import datetime
 
 
@@ -49,13 +51,21 @@ def download_and_extract(buildpath, packagedir, pkg, args):
         return srcpath
 
     if "url" in pkg["source"]:
-        tarballpath = buildpath / Path(pkg["source"]["url"]).name
+        response = request.urlopen(pkg["source"]["url"])
+        _, parameters = cgi.parse_header(
+            response.headers.get("Content-Disposition", "")
+        )
+        if "filename" in parameters:
+            tarballname = parameters["filename"]
+        else:
+            tarballname = Path(response.geturl()).name
+
+        tarballpath = buildpath / tarballname
         if not tarballpath.is_file():
             try:
-                subprocess.run(
+                os.makedirs(os.path.dirname(tarballpath), exist_ok=True)
-                    ["wget", "-q", "-O", str(tarballpath), pkg["source"]["url"]],
+                with open(tarballpath, "wb") as f:
-                    check=True,
+                    f.write(response.read())
-                )
                 check_checksum(tarballpath, pkg)
             except Exception:
                 tarballpath.unlink()
@@ -64,6 +74,21 @@ def download_and_extract(buildpath, packagedir, pkg, args):
         if not srcpath.is_dir():
             shutil.unpack_archive(str(tarballpath), str(buildpath))
 
+        for extension in [
+            ".tar.gz",
+            ".tgz",
+            ".tar",
+            ".tar.bz2",
+            ".tbz2",
+            ".tar.xz",
+            ".txz",
+            ".zip",
+        ]:
+            if tarballname.endswith(extension):
+                tarballname = tarballname[: -len(extension)]
+                break
+
+        return buildpath / tarballname
     elif "path" in pkg["source"]:
         srcdir = Path(pkg["source"]["path"])
 
@@ -72,11 +97,11 @@ def download_and_extract(buildpath, packagedir, pkg, args):
 
         if not srcpath.is_dir():
             shutil.copytree(srcdir, srcpath)
+
+        return srcpath
     else:
         raise ValueError("Incorrect source provided")
 
-    return srcpath
-
 
 def patch(path, srcpath, pkg, args):
     if (srcpath / ".patched").is_file():

test/pyodide_build/test_buildpkg.py

@@ -0,0 +1,35 @@
+import shutil
+import subprocess
+
+from pathlib import Path
+
+from pyodide_build import buildpkg, common
+
+
+def test_download_and_extract(monkeypatch):
+    monkeypatch.setattr(subprocess, "run", lambda *args, **kwargs: True)
+    monkeypatch.setattr(buildpkg, "check_checksum", lambda *args, **kwargs: True)
+    monkeypatch.setattr(shutil, "unpack_archive", lambda *args, **kwargs: True)
+
+    test_pkgs = []
+
+    # tarballname == version
+    test_pkgs.append(common.parse_package("./packages/scipy/meta.yaml"))
+    test_pkgs.append(common.parse_package("./packages/numpy/meta.yaml"))
+
+    # tarballname != version
+    test_pkgs.append(
+        {
+            "package": {"name": "pyyaml", "version": "5.3.1"},
+            "source": {
+                "url": "https://files.pythonhosted.org/packages/64/c2/b80047c7ac2478f9501676c988a5411ed5572f35d1beff9cae07d321512c/PyYAML-5.3.1.tar.gz"
+            },
+        }
+    )
+
+    for pkg in test_pkgs:
+        packagedir = pkg["package"]["name"] + "-" + pkg["package"]["version"]
+        buildpath = Path(pkg["package"]["name"]) / "build"
+        srcpath = buildpkg.download_and_extract(buildpath, packagedir, pkg, args=None)
+
+        assert srcpath.name.lower().endswith(packagedir.lower())