MAINT Use target sysconfigdata rather than host sysconfigdata when cross compiling (#2516)

sysconfig.py uses the environment variable `_PYTHON_SYSCONFIGDATA_NAME`
to decide where to look for the sysconfig data file with info about the compile target.
We also need to separately insure that our sysconfig data file is on the path. We
don't want the rest of our target stdlib on the path, so I made an extra sysconfigdata
folder, copied the sysconfig data into it, and put it on the path.

Makefile.envs

@@ -4,6 +4,7 @@ export PYVERSION ?= 3.10.2
 export PYODIDE_EMSCRIPTEN_VERSION ?= 2.0.27
 
 export PLATFORM_TRIPLET=wasm32-emscripten
+export SYSCONFIG_NAME=_sysconfigdata__emscripten_$(PLATFORM_TRIPLET)
 
 # BASH_ENV tells bash to run pyodide_env.sh on startup, which sets various
 # environment variables. The next line instructs make to use bash to run each

cpython/Makefile

@@ -10,7 +10,6 @@ INSTALL=$(ROOT)/installs/python-$(PYVERSION)
 TARBALL=$(ROOT)/downloads/Python-$(PYVERSION).tgz
 URL=https://www.python.org/ftp/python/$(PYVERSION)/Python-$(PYVERSION).tgz
 LIB=libpython$(PYMAJOR).$(PYMINOR).a
-SYSCONFIG_NAME=_sysconfigdata__emscripten_$(PLATFORM_TRIPLET)
 
 ZLIBVERSION=1.2.12
 ZLIBTARBALL=$(ROOT)/downloads/zlib-$(ZLIBVERSION).tar.gz
@@ -43,10 +42,15 @@ $(INSTALL)/lib/$(LIB): $(BUILD)/$(LIB) remove_modules.txt
 	# Generate sysconfigdata. It outputs into a subfolder of build/, and
 	# the subfolder is written to pybuilddir.txt.
 	_PYTHON_SYSCONFIGDATA_NAME=$(SYSCONFIG_NAME) _PYTHON_PROJECT_BASE=$(BUILD) $(HOSTPYTHON) -m sysconfig --generate-posix-vars
-	mv `cat pybuilddir.txt`/$(SYSCONFIG_NAME).py $(INSTALL)/lib/python$(PYMAJOR).$(PYMINOR)/
-	rmdir `cat pybuilddir.txt`
-	rm pybuilddir.txt
+
+	$(eval PYBUILDDIR=`cat pybuilddir.txt`)
+	PYTHONPATH=$(PYBUILDDIR) python adjust_sysconfig.py
+	cp $(PYBUILDDIR)/$(SYSCONFIG_NAME).py $(INSTALL)/lib/python$(PYMAJOR).$(PYMINOR)/
+	mkdir -p $(INSTALL)/sysconfigdata/
+	cp $(PYBUILDDIR)/$(SYSCONFIG_NAME).py $(INSTALL)/sysconfigdata/
 	cd $(INSTALL)/lib/python$(PYMAJOR).$(PYMINOR)/ && rm -rf `cat $(ROOT)/remove_modules.txt`
+	rm -rf $(PYBUILDDIR)
+	rm pybuilddir.txt
 
 
 clean:

cpython/adjust_sysconfig.py

@@ -0,0 +1,36 @@
+import os
+import pprint
+
+
+def load_sysconfig(sysconfig_name: str):
+    _temp = __import__(sysconfig_name, globals(), locals(), ["build_time_vars"], 0)
+    config_vars = _temp.build_time_vars
+    return config_vars, _temp.__file__
+
+
+def write_sysconfig(destfile: str, config_vars: dict[str, str]):
+    with open(destfile, "w", encoding="utf8") as f:
+        f.write(
+            "# system configuration generated and used by" " the sysconfig module\n"
+        )
+        f.write("build_time_vars = ")
+        pprint.pprint(config_vars, stream=f)
+
+
+def adjust_sysconfig(config_vars: dict[str, str]):
+    config_vars.update(
+        CC="cc",
+        MAINCC="cc",
+        LDSHARED="cc",
+        LINKCC="cc",
+        BLDSHARED="cc",
+        CXX="c++",
+        LDCXXSHARED="c++",
+    )
+
+
+if __name__ == "__main__":
+    sysconfig_name = os.environ["SYSCONFIG_NAME"]
+    config_vars, file = load_sysconfig(sysconfig_name)
+    adjust_sysconfig(config_vars)
+    write_sysconfig(file, config_vars)

docs/project/changelog.md

@@ -109,9 +109,14 @@ substitutions:
   wheels are distributed outside of the main Pyodide distribution.
   {pr}`2610`
 
+- {{ Enhancement }} The build system now uses the sysconfigdata from the target
+  Python rather than the host Python.
+  {pr}`2516`
+
 ### REPL
 
-- {{ Enhancement }} Add a spinner while the REPL is loading {pr}`2635`
+- {{ Enhancement }} Add a spinner while the REPL is loading
+  {pr}`2635`
 
 ### micropip
 

packages/fpcast-test/fpcast-test/setup.py

@@ -1,5 +1,19 @@
+from sysconfig import get_config_var
+
 from setuptools import Extension, setup
 
+if get_config_var("SIZEOF_VOID_P") != 4:
+    # pyodide_build.pypabuild will run this three times, the first time it fails
+    # and the exception is caught but the second and third times it must
+    # succeed.
+    raise Exception(
+        """
+This should appear in the log exactly one time. If it appears more than once,
+the Pyodide build system has misconfigured sysconfigdata (and also the build
+will fail).
+"""
+    )
+
 setup(
     name="fpcast-test",
     version="0.1.1",

pyodide-build/pyodide_build/buildpkg.py

@@ -137,6 +137,7 @@ def get_bash_runner():
             "PYODIDE_PACKAGE_ABI",
             "HOSTINSTALLDIR",
             "TARGETINSTALLDIR",
+            "SYSCONFIG_NAME",
             "HOSTSITEPACKAGES",
             "PYMAJOR",
             "PYMINOR",
@@ -506,6 +507,7 @@ def package_wheel(
         raise Exception(
             f"Unexpected number of wheels {len(rest) + 1} when building {pkg_name}"
         )
+    print(f"Unpacking wheel to {str(wheel)}")
     unpack_wheel(wheel)
     wheel.unlink()
     name, ver, _ = wheel.name.split("-", 2)
@@ -518,7 +520,8 @@ def package_wheel(
 
     post = build_metadata.get("post")
     if post:
-        bash_runner.env.update({"PKGDIR": str(pkg_root)})
+        print("Running post script in ", str(Path.cwd().absolute()))
+        bash_runner.env.update({"PKGDIR": str(pkg_root), "WHEELDIR": str(wheel_dir)})
         result = bash_runner.run(post)
         if result.returncode != 0:
             print("ERROR: post failed")

pyodide-build/pyodide_build/pypabuild.py

@@ -1,5 +1,6 @@
 import contextlib
 import os
+import shutil
 import sys
 import traceback
 from itertools import chain
@@ -24,6 +25,11 @@ def symlink_unisolated_packages(env: IsolatedEnv) -> None:
     pyversion = get_pyversion()
     site_packages_path = f"lib/{pyversion}/site-packages"
     env_site_packages = Path(env.path) / site_packages_path  # type: ignore[attr-defined]
+    sysconfigdata_name = os.environ["SYSCONFIG_NAME"]
+    sysconfigdata_path = (
+        Path(os.environ["TARGETINSTALLDIR"]) / f"sysconfigdata/{sysconfigdata_name}.py"
+    )
+    shutil.copy(sysconfigdata_path, env_site_packages)
     host_site_packages = Path(get_hostsitepackages())
     for name in get_unisolated_packages():
         for path in chain(

pyodide-build/pyodide_build/pywasmcross.py

@@ -103,11 +103,13 @@ def compile(
     env = dict(env)
     SYMLINKDIR = symlink_dir()
     env["PATH"] = f"{SYMLINKDIR}:{env['PATH']}"
-    args["PYTHONPATH"] = sys.path
+    sysconfig_dir = Path(os.environ["TARGETINSTALLDIR"]) / "sysconfigdata"
+    args["PYTHONPATH"] = sys.path + [str(sysconfig_dir)]
     args["orig__name__"] = __name__
     make_command_wrapper_symlinks(env)
     env["PYWASMCROSS_ARGS"] = json.dumps(args)
     env["_PYTHON_HOST_PLATFORM"] = common.platform()
+    env["_PYTHON_SYSCONFIGDATA_NAME"] = os.environ["SYSCONFIG_NAME"]
 
     from pyodide_build.pypabuild import build