diff --git a/Makefile.envs b/Makefile.envs index ceb4ab25c..49e91dbb3 100644 --- a/Makefile.envs +++ b/Makefile.envs @@ -4,6 +4,7 @@ export PYVERSION ?= 3.10.2 export PYODIDE_EMSCRIPTEN_VERSION ?= 2.0.27 export PLATFORM_TRIPLET=wasm32-emscripten +export SYSCONFIG_NAME=_sysconfigdata__emscripten_$(PLATFORM_TRIPLET) # BASH_ENV tells bash to run pyodide_env.sh on startup, which sets various # environment variables. The next line instructs make to use bash to run each diff --git a/cpython/Makefile b/cpython/Makefile index 3e096282c..da8a5c6bc 100644 --- a/cpython/Makefile +++ b/cpython/Makefile @@ -10,7 +10,6 @@ INSTALL=$(ROOT)/installs/python-$(PYVERSION) TARBALL=$(ROOT)/downloads/Python-$(PYVERSION).tgz URL=https://www.python.org/ftp/python/$(PYVERSION)/Python-$(PYVERSION).tgz LIB=libpython$(PYMAJOR).$(PYMINOR).a -SYSCONFIG_NAME=_sysconfigdata__emscripten_$(PLATFORM_TRIPLET) ZLIBVERSION=1.2.12 ZLIBTARBALL=$(ROOT)/downloads/zlib-$(ZLIBVERSION).tar.gz @@ -43,10 +42,15 @@ $(INSTALL)/lib/$(LIB): $(BUILD)/$(LIB) remove_modules.txt # Generate sysconfigdata. It outputs into a subfolder of build/, and # the subfolder is written to pybuilddir.txt. _PYTHON_SYSCONFIGDATA_NAME=$(SYSCONFIG_NAME) _PYTHON_PROJECT_BASE=$(BUILD) $(HOSTPYTHON) -m sysconfig --generate-posix-vars - mv `cat pybuilddir.txt`/$(SYSCONFIG_NAME).py $(INSTALL)/lib/python$(PYMAJOR).$(PYMINOR)/ - rmdir `cat pybuilddir.txt` - rm pybuilddir.txt + + $(eval PYBUILDDIR=`cat pybuilddir.txt`) + PYTHONPATH=$(PYBUILDDIR) python adjust_sysconfig.py + cp $(PYBUILDDIR)/$(SYSCONFIG_NAME).py $(INSTALL)/lib/python$(PYMAJOR).$(PYMINOR)/ + mkdir -p $(INSTALL)/sysconfigdata/ + cp $(PYBUILDDIR)/$(SYSCONFIG_NAME).py $(INSTALL)/sysconfigdata/ cd $(INSTALL)/lib/python$(PYMAJOR).$(PYMINOR)/ && rm -rf `cat $(ROOT)/remove_modules.txt` + rm -rf $(PYBUILDDIR) + rm pybuilddir.txt clean: diff --git a/cpython/adjust_sysconfig.py b/cpython/adjust_sysconfig.py new file mode 100644 index 000000000..511fffa32 --- /dev/null +++ b/cpython/adjust_sysconfig.py @@ -0,0 +1,36 @@ +import os +import pprint + + +def load_sysconfig(sysconfig_name: str): + _temp = __import__(sysconfig_name, globals(), locals(), ["build_time_vars"], 0) + config_vars = _temp.build_time_vars + return config_vars, _temp.__file__ + + +def write_sysconfig(destfile: str, config_vars: dict[str, str]): + with open(destfile, "w", encoding="utf8") as f: + f.write( + "# system configuration generated and used by" " the sysconfig module\n" + ) + f.write("build_time_vars = ") + pprint.pprint(config_vars, stream=f) + + +def adjust_sysconfig(config_vars: dict[str, str]): + config_vars.update( + CC="cc", + MAINCC="cc", + LDSHARED="cc", + LINKCC="cc", + BLDSHARED="cc", + CXX="c++", + LDCXXSHARED="c++", + ) + + +if __name__ == "__main__": + sysconfig_name = os.environ["SYSCONFIG_NAME"] + config_vars, file = load_sysconfig(sysconfig_name) + adjust_sysconfig(config_vars) + write_sysconfig(file, config_vars) diff --git a/docs/project/changelog.md b/docs/project/changelog.md index 5ad61a2f4..cc367f17b 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -109,9 +109,14 @@ substitutions: wheels are distributed outside of the main Pyodide distribution. {pr}`2610` +- {{ Enhancement }} The build system now uses the sysconfigdata from the target + Python rather than the host Python. + {pr}`2516` + ### REPL -- {{ Enhancement }} Add a spinner while the REPL is loading {pr}`2635` +- {{ Enhancement }} Add a spinner while the REPL is loading + {pr}`2635` ### micropip diff --git a/packages/fpcast-test/fpcast-test/setup.py b/packages/fpcast-test/fpcast-test/setup.py index 537d10aab..02550939d 100644 --- a/packages/fpcast-test/fpcast-test/setup.py +++ b/packages/fpcast-test/fpcast-test/setup.py @@ -1,5 +1,19 @@ +from sysconfig import get_config_var + from setuptools import Extension, setup +if get_config_var("SIZEOF_VOID_P") != 4: + # pyodide_build.pypabuild will run this three times, the first time it fails + # and the exception is caught but the second and third times it must + # succeed. + raise Exception( + """ +This should appear in the log exactly one time. If it appears more than once, +the Pyodide build system has misconfigured sysconfigdata (and also the build +will fail). +""" + ) + setup( name="fpcast-test", version="0.1.1", diff --git a/pyodide-build/pyodide_build/buildpkg.py b/pyodide-build/pyodide_build/buildpkg.py index 52622d3e9..d9587825e 100755 --- a/pyodide-build/pyodide_build/buildpkg.py +++ b/pyodide-build/pyodide_build/buildpkg.py @@ -137,6 +137,7 @@ def get_bash_runner(): "PYODIDE_PACKAGE_ABI", "HOSTINSTALLDIR", "TARGETINSTALLDIR", + "SYSCONFIG_NAME", "HOSTSITEPACKAGES", "PYMAJOR", "PYMINOR", @@ -506,6 +507,7 @@ def package_wheel( raise Exception( f"Unexpected number of wheels {len(rest) + 1} when building {pkg_name}" ) + print(f"Unpacking wheel to {str(wheel)}") unpack_wheel(wheel) wheel.unlink() name, ver, _ = wheel.name.split("-", 2) @@ -518,7 +520,8 @@ def package_wheel( post = build_metadata.get("post") if post: - bash_runner.env.update({"PKGDIR": str(pkg_root)}) + print("Running post script in ", str(Path.cwd().absolute())) + bash_runner.env.update({"PKGDIR": str(pkg_root), "WHEELDIR": str(wheel_dir)}) result = bash_runner.run(post) if result.returncode != 0: print("ERROR: post failed") diff --git a/pyodide-build/pyodide_build/pypabuild.py b/pyodide-build/pyodide_build/pypabuild.py index a6fed86db..8572d56fc 100644 --- a/pyodide-build/pyodide_build/pypabuild.py +++ b/pyodide-build/pyodide_build/pypabuild.py @@ -1,5 +1,6 @@ import contextlib import os +import shutil import sys import traceback from itertools import chain @@ -24,6 +25,11 @@ def symlink_unisolated_packages(env: IsolatedEnv) -> None: pyversion = get_pyversion() site_packages_path = f"lib/{pyversion}/site-packages" env_site_packages = Path(env.path) / site_packages_path # type: ignore[attr-defined] + sysconfigdata_name = os.environ["SYSCONFIG_NAME"] + sysconfigdata_path = ( + Path(os.environ["TARGETINSTALLDIR"]) / f"sysconfigdata/{sysconfigdata_name}.py" + ) + shutil.copy(sysconfigdata_path, env_site_packages) host_site_packages = Path(get_hostsitepackages()) for name in get_unisolated_packages(): for path in chain( diff --git a/pyodide-build/pyodide_build/pywasmcross.py b/pyodide-build/pyodide_build/pywasmcross.py index 09c3e96d8..96ad0dafc 100755 --- a/pyodide-build/pyodide_build/pywasmcross.py +++ b/pyodide-build/pyodide_build/pywasmcross.py @@ -103,11 +103,13 @@ def compile( env = dict(env) SYMLINKDIR = symlink_dir() env["PATH"] = f"{SYMLINKDIR}:{env['PATH']}" - args["PYTHONPATH"] = sys.path + sysconfig_dir = Path(os.environ["TARGETINSTALLDIR"]) / "sysconfigdata" + args["PYTHONPATH"] = sys.path + [str(sysconfig_dir)] args["orig__name__"] = __name__ make_command_wrapper_symlinks(env) env["PYWASMCROSS_ARGS"] = json.dumps(args) env["_PYTHON_HOST_PLATFORM"] = common.platform() + env["_PYTHON_SYSCONFIGDATA_NAME"] = os.environ["SYSCONFIG_NAME"] from pyodide_build.pypabuild import build