pyodide/pyodide-build/pyodide_build/config.py

import os
import subprocess
from collections.abc import Mapping
from pathlib import Path
from types import MappingProxyType

from .common import _environment_substitute_str, exit_with_stdio
from .logger import logger


class ConfigManager:
    """
    Configuration manager for Package build process.

    The configuration manager is responsible for loading configuration from various sources.
    The configuration can be loaded from the following sources (in order of precedence):

        1. Command line arguments (TODO)
        2. Environment variables
        3. Configuration file (TODO)
        4. Makefile.envs
        5. Default values
    """

    def __init__(self, pyodide_root: Path):
        self.pyodide_root = pyodide_root
        self._config = {
            **self._load_default_config(),
            **self._load_makefile_envs(),
            **self._load_config_file(),
            **self._load_config_from_env(os.environ),
        }

    def _load_default_config(self) -> Mapping[str, str]:
        return {
            k: _environment_substitute_str(
                v, env={"PYODIDE_ROOT": str(self.pyodide_root)}
            )
            for k, v in DEFAULT_CONFIG.items()
        }

    def _load_makefile_envs(self) -> Mapping[str, str]:
        makefile_vars = self._get_make_environment_vars()
        computed_vars = {
            k: _environment_substitute_str(v, env=makefile_vars)
            for k, v in DEFAULT_CONFIG_COMPUTED.items()
        }

        return {
            BUILD_VAR_TO_KEY[k]: v
            for k, v in makefile_vars.items()
            if k in BUILD_VAR_TO_KEY
        } | computed_vars

    def _get_make_environment_vars(self) -> Mapping[str, str]:
        """
        Load environment variables from Makefile.envs
        """
        environment = {}
        result = subprocess.run(
            ["make", "-f", str(self.pyodide_root / "Makefile.envs"), ".output_vars"],
            capture_output=True,
            text=True,
            env={"PYODIDE_ROOT": str(self.pyodide_root)},
        )

        if result.returncode != 0:
            logger.error(
                "ERROR: Failed to load environment variables from Makefile.envs"
            )
            exit_with_stdio(result)

        for line in result.stdout.splitlines():
            equalPos = line.find("=")
            if equalPos != -1:
                varname = line[0:equalPos]

                if varname not in BUILD_VAR_TO_KEY:
                    continue

                value = line[equalPos + 1 :]
                value = value.strip("'").strip()
                environment[varname] = value

        return environment

    def _load_config_from_env(self, env: Mapping[str, str]) -> Mapping[str, str]:
        return {
            BUILD_VAR_TO_KEY[key]: env[key] for key in env if key in BUILD_VAR_TO_KEY
        }

    def _load_config_file(self) -> Mapping[str, str]:
        # TODO: Implement this
        return {}

    @property
    def config(self) -> Mapping[str, str]:
        return MappingProxyType(self._config)

    def to_env(self) -> dict[str, str]:
        """
        Export the configuration to environment variables.
        """
        return {BUILD_KEY_TO_VAR[k]: v for k, v in self.config.items()}


# Configuration variables and corresponding environment variables.
# TODO: distinguish between variables that are overridable by the user and those that are not.
BUILD_KEY_TO_VAR: dict[str, str] = {
    "pyodide_version": "PYODIDE_VERSION",
    "cargo_build_target": "CARGO_BUILD_TARGET",
    "cargo_target_wasm32_unknown_emscripten_linker": "CARGO_TARGET_WASM32_UNKNOWN_EMSCRIPTEN_LINKER",
    "host_install_dir": "HOSTINSTALLDIR",
    "host_site_packages": "HOSTSITEPACKAGES",
    "numpy_lib": "NUMPY_LIB",
    "platform_triplet": "PLATFORM_TRIPLET",
    "pip_constraint": "PIP_CONSTRAINT",
    "pymajor": "PYMAJOR",
    "pymicro": "PYMICRO",
    "pyminor": "PYMINOR",
    "pyo3_cross_include_dir": "PYO3_CROSS_INCLUDE_DIR",
    "pyo3_cross_lib_dir": "PYO3_CROSS_LIB_DIR",
    "pyodide_emscripten_version": "PYODIDE_EMSCRIPTEN_VERSION",
    "pyodide_jobs": "PYODIDE_JOBS",
    "pyodide_root": "PYODIDE_ROOT",
    "python_archive_sha256": "PYTHON_ARCHIVE_SHA256",
    "python_archive_url": "PYTHON_ARCHIVE_URL",
    "pythoninclude": "PYTHONINCLUDE",
    "pyversion": "PYVERSION",
    "cpythoninstall": "CPYTHONINSTALL",
    "rustflags": "RUSTFLAGS",
    "rust_toolchain": "RUST_TOOLCHAIN",
    "side_module_cflags": "SIDE_MODULE_CFLAGS",
    "side_module_cxxflags": "SIDE_MODULE_CXXFLAGS",
    "side_module_ldflags": "SIDE_MODULE_LDFLAGS",
    "stdlib_module_cflags": "STDLIB_MODULE_CFLAGS",
    "sysconfigdata_dir": "SYSCONFIGDATA_DIR",
    "sysconfig_name": "SYSCONFIG_NAME",
    "targetinstalldir": "TARGETINSTALLDIR",
    "cmake_toolchain_file": "CMAKE_TOOLCHAIN_FILE",
    "pyo3_config_file": "PYO3_CONFIG_FILE",
    "meson_cross_file": "MESON_CROSS_FILE",
    "cflags_base": "CFLAGS_BASE",
    "cxxflags_base": "CXXFLAGS_BASE",
    "ldflags_base": "LDFLAGS_BASE",
    "home": "HOME",
    "path": "PATH",
    "zip_compression_level": "PYODIDE_ZIP_COMPRESSION_LEVEL",
}

BUILD_VAR_TO_KEY = {v: k for k, v in BUILD_KEY_TO_VAR.items()}

# Default configuration values.
TOOLS_DIR = Path(__file__).parent / "tools"
DEFAULT_CONFIG: dict[str, str] = {
    # Paths to toolchain configuration files
    "cmake_toolchain_file": str(TOOLS_DIR / "cmake/Modules/Platform/Emscripten.cmake"),
    "pyo3_config_file": str(TOOLS_DIR / "pyo3_config.ini"),
    "meson_cross_file": str(TOOLS_DIR / "emscripten.meson.cross"),
    # Rust-specific configuration
    "rustflags": "-C link-arg=-sSIDE_MODULE=2 -C link-arg=-sWASM_BIGINT -Z link-native-libraries=no",
    "cargo_build_target": "wasm32-unknown-emscripten",
    "cargo_target_wasm32_unknown_emscripten_linker": "emcc",
    "rust_toolchain": "nightly-2024-01-29",
    # Other configuration
    "pyodide_jobs": "1",
}

# Default configs that are computed from other values (often from Makefile.envs)
# TODO: Remove dependency on Makefile.envs
DEFAULT_CONFIG_COMPUTED: dict[str, str] = {
    # Compiler flags
    "side_module_cflags": "$(CFLAGS_BASE) -I$(PYTHONINCLUDE)",
    "side_module_cxxflags": "$(CXXFLAGS_BASE)",
    "side_module_ldflags": "$(LDFLAGS_BASE) -s SIDE_MODULE=1",
    # Rust-specific configuration
    "pyo3_cross_lib_dir": "$(CPYTHONINSTALL)/lib",
    "pyo3_cross_include_dir": "$(PYTHONINCLUDE)",
    # Misc
    "stdlib_module_cflags": "$(CFLAGS_BASE) -I$(PYTHONINCLUDE) -I Include/ -I. -IInclude/internal/",  # TODO: remove this
    # Paths to build dependencies
    "host_install_dir": "$(PYODIDE_ROOT)/packages/.artifacts",
    "host_site_packages": "$(PYODIDE_ROOT)/packages/.artifacts/lib/python$(PYMAJOR).$(PYMINOR)/site-packages",
    "numpy_lib": "$(PYODIDE_ROOT)/packages/.artifacts/lib/python$(PYMAJOR).$(PYMINOR)/site-packages/numpy/",
}