From 2f4f258bb5900f02ae1f81aca26a99a1809183f8 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Sun, 29 May 2022 18:59:10 -0700 Subject: [PATCH] MAINT Use target sysconfigdata rather than host sysconfigdata when cross compiling (#2516) sysconfig.py uses the environment variable `_PYTHON_SYSCONFIGDATA_NAME` to decide where to look for the sysconfig data file with info about the compile target. We also need to separately insure that our sysconfig data file is on the path. We don't want the rest of our target stdlib on the path, so I made an extra sysconfigdata folder, copied the sysconfig data into it, and put it on the path. --- Makefile.envs | 1 + cpython/Makefile | 12 +++++--- cpython/adjust_sysconfig.py | 36 ++++++++++++++++++++++ docs/project/changelog.md | 7 ++++- packages/fpcast-test/fpcast-test/setup.py | 14 +++++++++ pyodide-build/pyodide_build/buildpkg.py | 5 ++- pyodide-build/pyodide_build/pypabuild.py | 6 ++++ pyodide-build/pyodide_build/pywasmcross.py | 4 ++- 8 files changed, 78 insertions(+), 7 deletions(-) create mode 100644 cpython/adjust_sysconfig.py diff --git a/Makefile.envs b/Makefile.envs index ceb4ab25c..49e91dbb3 100644 --- a/Makefile.envs +++ b/Makefile.envs @@ -4,6 +4,7 @@ export PYVERSION ?= 3.10.2 export PYODIDE_EMSCRIPTEN_VERSION ?= 2.0.27 export PLATFORM_TRIPLET=wasm32-emscripten +export SYSCONFIG_NAME=_sysconfigdata__emscripten_$(PLATFORM_TRIPLET) # BASH_ENV tells bash to run pyodide_env.sh on startup, which sets various # environment variables. The next line instructs make to use bash to run each diff --git a/cpython/Makefile b/cpython/Makefile index 3e096282c..da8a5c6bc 100644 --- a/cpython/Makefile +++ b/cpython/Makefile @@ -10,7 +10,6 @@ INSTALL=$(ROOT)/installs/python-$(PYVERSION) TARBALL=$(ROOT)/downloads/Python-$(PYVERSION).tgz URL=https://www.python.org/ftp/python/$(PYVERSION)/Python-$(PYVERSION).tgz LIB=libpython$(PYMAJOR).$(PYMINOR).a -SYSCONFIG_NAME=_sysconfigdata__emscripten_$(PLATFORM_TRIPLET) ZLIBVERSION=1.2.12 ZLIBTARBALL=$(ROOT)/downloads/zlib-$(ZLIBVERSION).tar.gz @@ -43,10 +42,15 @@ $(INSTALL)/lib/$(LIB): $(BUILD)/$(LIB) remove_modules.txt # Generate sysconfigdata. It outputs into a subfolder of build/, and # the subfolder is written to pybuilddir.txt. _PYTHON_SYSCONFIGDATA_NAME=$(SYSCONFIG_NAME) _PYTHON_PROJECT_BASE=$(BUILD) $(HOSTPYTHON) -m sysconfig --generate-posix-vars - mv `cat pybuilddir.txt`/$(SYSCONFIG_NAME).py $(INSTALL)/lib/python$(PYMAJOR).$(PYMINOR)/ - rmdir `cat pybuilddir.txt` - rm pybuilddir.txt + + $(eval PYBUILDDIR=`cat pybuilddir.txt`) + PYTHONPATH=$(PYBUILDDIR) python adjust_sysconfig.py + cp $(PYBUILDDIR)/$(SYSCONFIG_NAME).py $(INSTALL)/lib/python$(PYMAJOR).$(PYMINOR)/ + mkdir -p $(INSTALL)/sysconfigdata/ + cp $(PYBUILDDIR)/$(SYSCONFIG_NAME).py $(INSTALL)/sysconfigdata/ cd $(INSTALL)/lib/python$(PYMAJOR).$(PYMINOR)/ && rm -rf `cat $(ROOT)/remove_modules.txt` + rm -rf $(PYBUILDDIR) + rm pybuilddir.txt clean: diff --git a/cpython/adjust_sysconfig.py b/cpython/adjust_sysconfig.py new file mode 100644 index 000000000..511fffa32 --- /dev/null +++ b/cpython/adjust_sysconfig.py @@ -0,0 +1,36 @@ +import os +import pprint + + +def load_sysconfig(sysconfig_name: str): + _temp = __import__(sysconfig_name, globals(), locals(), ["build_time_vars"], 0) + config_vars = _temp.build_time_vars + return config_vars, _temp.__file__ + + +def write_sysconfig(destfile: str, config_vars: dict[str, str]): + with open(destfile, "w", encoding="utf8") as f: + f.write( + "# system configuration generated and used by" " the sysconfig module\n" + ) + f.write("build_time_vars = ") + pprint.pprint(config_vars, stream=f) + + +def adjust_sysconfig(config_vars: dict[str, str]): + config_vars.update( + CC="cc", + MAINCC="cc", + LDSHARED="cc", + LINKCC="cc", + BLDSHARED="cc", + CXX="c++", + LDCXXSHARED="c++", + ) + + +if __name__ == "__main__": + sysconfig_name = os.environ["SYSCONFIG_NAME"] + config_vars, file = load_sysconfig(sysconfig_name) + adjust_sysconfig(config_vars) + write_sysconfig(file, config_vars) diff --git a/docs/project/changelog.md b/docs/project/changelog.md index 5ad61a2f4..cc367f17b 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -109,9 +109,14 @@ substitutions: wheels are distributed outside of the main Pyodide distribution. {pr}`2610` +- {{ Enhancement }} The build system now uses the sysconfigdata from the target + Python rather than the host Python. + {pr}`2516` + ### REPL -- {{ Enhancement }} Add a spinner while the REPL is loading {pr}`2635` +- {{ Enhancement }} Add a spinner while the REPL is loading + {pr}`2635` ### micropip diff --git a/packages/fpcast-test/fpcast-test/setup.py b/packages/fpcast-test/fpcast-test/setup.py index 537d10aab..02550939d 100644 --- a/packages/fpcast-test/fpcast-test/setup.py +++ b/packages/fpcast-test/fpcast-test/setup.py @@ -1,5 +1,19 @@ +from sysconfig import get_config_var + from setuptools import Extension, setup +if get_config_var("SIZEOF_VOID_P") != 4: + # pyodide_build.pypabuild will run this three times, the first time it fails + # and the exception is caught but the second and third times it must + # succeed. + raise Exception( + """ +This should appear in the log exactly one time. If it appears more than once, +the Pyodide build system has misconfigured sysconfigdata (and also the build +will fail). +""" + ) + setup( name="fpcast-test", version="0.1.1", diff --git a/pyodide-build/pyodide_build/buildpkg.py b/pyodide-build/pyodide_build/buildpkg.py index 52622d3e9..d9587825e 100755 --- a/pyodide-build/pyodide_build/buildpkg.py +++ b/pyodide-build/pyodide_build/buildpkg.py @@ -137,6 +137,7 @@ def get_bash_runner(): "PYODIDE_PACKAGE_ABI", "HOSTINSTALLDIR", "TARGETINSTALLDIR", + "SYSCONFIG_NAME", "HOSTSITEPACKAGES", "PYMAJOR", "PYMINOR", @@ -506,6 +507,7 @@ def package_wheel( raise Exception( f"Unexpected number of wheels {len(rest) + 1} when building {pkg_name}" ) + print(f"Unpacking wheel to {str(wheel)}") unpack_wheel(wheel) wheel.unlink() name, ver, _ = wheel.name.split("-", 2) @@ -518,7 +520,8 @@ def package_wheel( post = build_metadata.get("post") if post: - bash_runner.env.update({"PKGDIR": str(pkg_root)}) + print("Running post script in ", str(Path.cwd().absolute())) + bash_runner.env.update({"PKGDIR": str(pkg_root), "WHEELDIR": str(wheel_dir)}) result = bash_runner.run(post) if result.returncode != 0: print("ERROR: post failed") diff --git a/pyodide-build/pyodide_build/pypabuild.py b/pyodide-build/pyodide_build/pypabuild.py index a6fed86db..8572d56fc 100644 --- a/pyodide-build/pyodide_build/pypabuild.py +++ b/pyodide-build/pyodide_build/pypabuild.py @@ -1,5 +1,6 @@ import contextlib import os +import shutil import sys import traceback from itertools import chain @@ -24,6 +25,11 @@ def symlink_unisolated_packages(env: IsolatedEnv) -> None: pyversion = get_pyversion() site_packages_path = f"lib/{pyversion}/site-packages" env_site_packages = Path(env.path) / site_packages_path # type: ignore[attr-defined] + sysconfigdata_name = os.environ["SYSCONFIG_NAME"] + sysconfigdata_path = ( + Path(os.environ["TARGETINSTALLDIR"]) / f"sysconfigdata/{sysconfigdata_name}.py" + ) + shutil.copy(sysconfigdata_path, env_site_packages) host_site_packages = Path(get_hostsitepackages()) for name in get_unisolated_packages(): for path in chain( diff --git a/pyodide-build/pyodide_build/pywasmcross.py b/pyodide-build/pyodide_build/pywasmcross.py index 09c3e96d8..96ad0dafc 100755 --- a/pyodide-build/pyodide_build/pywasmcross.py +++ b/pyodide-build/pyodide_build/pywasmcross.py @@ -103,11 +103,13 @@ def compile( env = dict(env) SYMLINKDIR = symlink_dir() env["PATH"] = f"{SYMLINKDIR}:{env['PATH']}" - args["PYTHONPATH"] = sys.path + sysconfig_dir = Path(os.environ["TARGETINSTALLDIR"]) / "sysconfigdata" + args["PYTHONPATH"] = sys.path + [str(sysconfig_dir)] args["orig__name__"] = __name__ make_command_wrapper_symlinks(env) env["PYWASMCROSS_ARGS"] = json.dumps(args) env["_PYTHON_HOST_PLATFORM"] = common.platform() + env["_PYTHON_SYSCONFIGDATA_NAME"] = os.environ["SYSCONFIG_NAME"] from pyodide_build.pypabuild import build