mirror of https://github.com/python/cpython.git
GH-118306: Update JIT to use LLVM 18 (GH-118307)
This commit is contained in:
Savannah Ostrowski
GitHub
parent
8e4fb5d260
commit
8b56d82c59
|
|
@ -41,7 +41,7 @@ jobs:
|
|||
- true
|
||||
- false
|
||||
llvm:
|
||||
- 16
|
||||
- 18
|
||||
include:
|
||||
- target: i686-pc-windows-msvc/msvc
|
||||
architecture: Win32
|
||||
|
|
@ -94,6 +94,7 @@ jobs:
|
|||
- name: Native Windows
|
||||
if: runner.os == 'Windows' && matrix.architecture != 'ARM64'
|
||||
run: |
|
||||
choco upgrade llvm -y
|
||||
choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}
|
||||
./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }}
|
||||
./PCbuild/rt.bat ${{ matrix.debug && '-d' }} -p ${{ matrix.architecture }} -q --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3
|
||||
|
|
@ -102,27 +103,31 @@ jobs:
|
|||
- name: Emulated Windows
|
||||
if: runner.os == 'Windows' && matrix.architecture == 'ARM64'
|
||||
run: |
|
||||
choco upgrade llvm -y
|
||||
choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}
|
||||
./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }}
|
||||
|
||||
- name: Native macOS
|
||||
if: runner.os == 'macOS'
|
||||
run: |
|
||||
brew update
|
||||
brew install llvm@${{ matrix.llvm }}
|
||||
SDKROOT="$(xcrun --show-sdk-path)" \
|
||||
./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
|
||||
make all --jobs 4
|
||||
./python.exe -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3
|
||||
|
||||
# --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553)
|
||||
- name: Native Linux
|
||||
if: runner.os == 'Linux' && matrix.architecture == 'x86_64'
|
||||
run: |
|
||||
sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
|
||||
export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
|
||||
./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
|
||||
./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations' }}
|
||||
make all --jobs 4
|
||||
./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3
|
||||
|
||||
# --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553)
|
||||
- name: Emulated Linux
|
||||
if: runner.os == 'Linux' && matrix.architecture != 'x86_64'
|
||||
run: |
|
||||
|
|
@ -139,6 +144,6 @@ jobs:
|
|||
CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" \
|
||||
CPP="$CC --preprocess" \
|
||||
HOSTRUNNER=qemu-${{ matrix.architecture }} \
|
||||
./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
|
||||
./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations ' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
|
||||
make all --jobs 4
|
||||
./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
Update JIT compilation to use LLVM 18
|
||||
|
|
@ -234,6 +234,7 @@ patch(unsigned char *base, const Stencil *stencil, uintptr_t patches[])
|
|||
assert((int64_t)value < (1LL << 31));
|
||||
*loc32 = (uint32_t)value;
|
||||
continue;
|
||||
case HoleKind_ARM64_RELOC_BRANCH26:
|
||||
case HoleKind_IMAGE_REL_ARM64_BRANCH26:
|
||||
case HoleKind_R_AARCH64_CALL26:
|
||||
case HoleKind_R_AARCH64_JUMP26:
|
||||
|
|
@ -278,6 +279,7 @@ patch(unsigned char *base, const Stencil *stencil, uintptr_t patches[])
|
|||
case HoleKind_ARM64_RELOC_GOT_LOAD_PAGE21:
|
||||
case HoleKind_IMAGE_REL_ARM64_PAGEBASE_REL21:
|
||||
case HoleKind_R_AARCH64_ADR_GOT_PAGE:
|
||||
case HoleKind_R_AARCH64_ADR_PREL_PG_HI21:
|
||||
// 21-bit count of pages between this page and an absolute address's
|
||||
// page... I know, I know, it's weird. Pairs nicely with
|
||||
// ARM64_RELOC_GOT_LOAD_PAGEOFF12 (below).
|
||||
|
|
@ -341,6 +343,7 @@ patch(unsigned char *base, const Stencil *stencil, uintptr_t patches[])
|
|||
case HoleKind_ARM64_RELOC_PAGEOFF12:
|
||||
case HoleKind_IMAGE_REL_ARM64_PAGEOFFSET_12A:
|
||||
case HoleKind_IMAGE_REL_ARM64_PAGEOFFSET_12L:
|
||||
case HoleKind_R_AARCH64_ADD_ABS_LO12_NC:
|
||||
case HoleKind_R_AARCH64_LD64_GOT_LO12_NC:
|
||||
// 12-bit low part of an absolute address. Pairs nicely with
|
||||
// ARM64_RELOC_GOT_LOAD_PAGE21 (above).
|
||||
|
|
|
|||
|
|
@ -23,17 +23,21 @@ ### Linux
|
|||
|
||||
### macOS
|
||||
|
||||
Install LLVM 16 with [Homebrew](https://brew.sh):
|
||||
Install LLVM 18 with [Homebrew](https://brew.sh):
|
||||
|
||||
```sh
|
||||
brew install llvm@16
|
||||
brew install llvm@18
|
||||
```
|
||||
|
||||
Homebrew won't add any of the tools to your `$PATH`. That's okay; the build script knows how to find them.
|
||||
|
||||
### Windows
|
||||
|
||||
Install LLVM 16 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=16), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".**
|
||||
Install LLVM 18 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=18), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".**
|
||||
|
||||
### Dev Containers
|
||||
|
||||
If you are working CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using-codespaces), there's no need to install LLVM as the Fedora 40 base image includes LLVM 18 out of the box.
|
||||
|
||||
## Building
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
import subprocess
|
||||
import typing
|
||||
|
||||
_LLVM_VERSION = 16
|
||||
_LLVM_VERSION = 18
|
||||
_LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\s+")
|
||||
|
||||
_P = typing.ParamSpec("_P")
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
import typing
|
||||
|
||||
HoleKind: typing.TypeAlias = typing.Literal[
|
||||
"ARM64_RELOC_BRANCH26",
|
||||
"ARM64_RELOC_GOT_LOAD_PAGE21",
|
||||
"ARM64_RELOC_GOT_LOAD_PAGEOFF12",
|
||||
"ARM64_RELOC_PAGE21",
|
||||
|
|
@ -16,8 +17,10 @@
|
|||
"IMAGE_REL_I386_REL32",
|
||||
"R_AARCH64_ABS64",
|
||||
"R_AARCH64_ADR_GOT_PAGE",
|
||||
"R_AARCH64_ADR_PREL_PG_HI21",
|
||||
"R_AARCH64_CALL26",
|
||||
"R_AARCH64_JUMP26",
|
||||
"R_AARCH64_ADD_ABS_LO12_NC",
|
||||
"R_AARCH64_LD64_GOT_LO12_NC",
|
||||
"R_AARCH64_MOVW_UABS_G0_NC",
|
||||
"R_AARCH64_MOVW_UABS_G1_NC",
|
||||
|
|
@ -68,12 +71,12 @@ class _COFFSymbol(typing.TypedDict):
|
|||
|
||||
|
||||
class _ELFSymbol(typing.TypedDict):
|
||||
Name: dict[typing.Literal["Value"], str]
|
||||
Name: dict[typing.Literal["Name"], str]
|
||||
Value: int
|
||||
|
||||
|
||||
class _MachOSymbol(typing.TypedDict):
|
||||
Name: dict[typing.Literal["Value"], str]
|
||||
Name: dict[typing.Literal["Name"], str]
|
||||
Value: int
|
||||
|
||||
|
||||
|
|
@ -99,7 +102,7 @@ class ELFSection(typing.TypedDict):
|
|||
Relocations: list[dict[typing.Literal["Relocation"], ELFRelocation]]
|
||||
SectionData: dict[typing.Literal["Bytes"], list[int]]
|
||||
Symbols: list[dict[typing.Literal["Symbol"], _ELFSymbol]]
|
||||
Type: dict[typing.Literal["Value"], str]
|
||||
Type: dict[typing.Literal["Name"], str]
|
||||
|
||||
|
||||
class MachOSection(typing.TypedDict):
|
||||
|
|
|
|||
|
|
@ -203,7 +203,9 @@ def process_relocations(self, *, alignment: int = 1) -> None:
|
|||
"""Fix up all GOT and internal relocations for this stencil group."""
|
||||
for hole in self.code.holes.copy():
|
||||
if (
|
||||
hole.kind in {"R_AARCH64_CALL26", "R_AARCH64_JUMP26"}
|
||||
hole.kind in {
|
||||
"R_AARCH64_CALL26", "R_AARCH64_JUMP26", "ARM64_RELOC_BRANCH26"
|
||||
}
|
||||
and hole.value is HoleValue.ZERO
|
||||
):
|
||||
self.code.pad(alignment)
|
||||
|
|
|
|||
|
|
@ -221,7 +221,7 @@ def _handle_relocation(
|
|||
case {
|
||||
"Offset": offset,
|
||||
"Symbol": s,
|
||||
"Type": {"Value": "IMAGE_REL_I386_DIR32" as kind},
|
||||
"Type": {"Name": "IMAGE_REL_I386_DIR32" as kind},
|
||||
}:
|
||||
offset += base
|
||||
value, symbol = self._unwrap_dllimport(s)
|
||||
|
|
@ -230,7 +230,7 @@ def _handle_relocation(
|
|||
"Offset": offset,
|
||||
"Symbol": s,
|
||||
"Type": {
|
||||
"Value": "IMAGE_REL_AMD64_REL32" | "IMAGE_REL_I386_REL32" as kind
|
||||
"Name": "IMAGE_REL_AMD64_REL32" | "IMAGE_REL_I386_REL32" as kind
|
||||
},
|
||||
}:
|
||||
offset += base
|
||||
|
|
@ -242,7 +242,7 @@ def _handle_relocation(
|
|||
"Offset": offset,
|
||||
"Symbol": s,
|
||||
"Type": {
|
||||
"Value": "IMAGE_REL_ARM64_BRANCH26"
|
||||
"Name": "IMAGE_REL_ARM64_BRANCH26"
|
||||
| "IMAGE_REL_ARM64_PAGEBASE_REL21"
|
||||
| "IMAGE_REL_ARM64_PAGEOFFSET_12A"
|
||||
| "IMAGE_REL_ARM64_PAGEOFFSET_12L" as kind
|
||||
|
|
@ -262,7 +262,7 @@ class _ELF(
|
|||
def _handle_section(
|
||||
self, section: _schema.ELFSection, group: _stencils.StencilGroup
|
||||
) -> None:
|
||||
section_type = section["Type"]["Value"]
|
||||
section_type = section["Type"]["Name"]
|
||||
flags = {flag["Name"] for flag in section["Flags"]["Flags"]}
|
||||
if section_type == "SHT_RELA":
|
||||
assert "SHF_INFO_LINK" in flags, flags
|
||||
|
|
@ -290,7 +290,7 @@ def _handle_section(
|
|||
for wrapped_symbol in section["Symbols"]:
|
||||
symbol = wrapped_symbol["Symbol"]
|
||||
offset = len(stencil.body) + symbol["Value"]
|
||||
name = symbol["Name"]["Value"]
|
||||
name = symbol["Name"]["Name"]
|
||||
name = name.removeprefix(self.prefix)
|
||||
group.symbols[name] = value, offset
|
||||
stencil.body.extend(section["SectionData"]["Bytes"])
|
||||
|
|
@ -312,9 +312,9 @@ def _handle_relocation(
|
|||
case {
|
||||
"Addend": addend,
|
||||
"Offset": offset,
|
||||
"Symbol": {"Value": s},
|
||||
"Symbol": {"Name": s},
|
||||
"Type": {
|
||||
"Value": "R_AARCH64_ADR_GOT_PAGE"
|
||||
"Name": "R_AARCH64_ADR_GOT_PAGE"
|
||||
| "R_AARCH64_LD64_GOT_LO12_NC"
|
||||
| "R_X86_64_GOTPCREL"
|
||||
| "R_X86_64_GOTPCRELX"
|
||||
|
|
@ -327,8 +327,8 @@ def _handle_relocation(
|
|||
case {
|
||||
"Addend": addend,
|
||||
"Offset": offset,
|
||||
"Symbol": {"Value": s},
|
||||
"Type": {"Value": kind},
|
||||
"Symbol": {"Name": s},
|
||||
"Type": {"Name": kind},
|
||||
}:
|
||||
offset += base
|
||||
s = s.removeprefix(self.prefix)
|
||||
|
|
@ -371,7 +371,7 @@ def _handle_section(
|
|||
for wrapped_symbol in section["Symbols"]:
|
||||
symbol = wrapped_symbol["Symbol"]
|
||||
offset = symbol["Value"] - start_address
|
||||
name = symbol["Name"]["Value"]
|
||||
name = symbol["Name"]["Name"]
|
||||
name = name.removeprefix(self.prefix)
|
||||
group.symbols[name] = value, offset
|
||||
assert "Relocations" in section
|
||||
|
|
@ -387,9 +387,9 @@ def _handle_relocation(
|
|||
match relocation:
|
||||
case {
|
||||
"Offset": offset,
|
||||
"Symbol": {"Value": s},
|
||||
"Symbol": {"Name": s},
|
||||
"Type": {
|
||||
"Value": "ARM64_RELOC_GOT_LOAD_PAGE21"
|
||||
"Name": "ARM64_RELOC_GOT_LOAD_PAGE21"
|
||||
| "ARM64_RELOC_GOT_LOAD_PAGEOFF12" as kind
|
||||
},
|
||||
}:
|
||||
|
|
@ -399,8 +399,8 @@ def _handle_relocation(
|
|||
addend = 0
|
||||
case {
|
||||
"Offset": offset,
|
||||
"Symbol": {"Value": s},
|
||||
"Type": {"Value": "X86_64_RELOC_GOT" | "X86_64_RELOC_GOT_LOAD" as kind},
|
||||
"Symbol": {"Name": s},
|
||||
"Type": {"Name": "X86_64_RELOC_GOT" | "X86_64_RELOC_GOT_LOAD" as kind},
|
||||
}:
|
||||
offset += base
|
||||
s = s.removeprefix(self.prefix)
|
||||
|
|
@ -410,13 +410,13 @@ def _handle_relocation(
|
|||
)
|
||||
case {
|
||||
"Offset": offset,
|
||||
"Section": {"Value": s},
|
||||
"Type": {"Value": "X86_64_RELOC_SIGNED" as kind},
|
||||
"Section": {"Name": s},
|
||||
"Type": {"Name": "X86_64_RELOC_SIGNED" as kind},
|
||||
} | {
|
||||
"Offset": offset,
|
||||
"Symbol": {"Value": s},
|
||||
"Symbol": {"Name": s},
|
||||
"Type": {
|
||||
"Value": "X86_64_RELOC_BRANCH" | "X86_64_RELOC_SIGNED" as kind
|
||||
"Name": "X86_64_RELOC_BRANCH" | "X86_64_RELOC_SIGNED" as kind
|
||||
},
|
||||
}:
|
||||
offset += base
|
||||
|
|
@ -427,12 +427,12 @@ def _handle_relocation(
|
|||
)
|
||||
case {
|
||||
"Offset": offset,
|
||||
"Section": {"Value": s},
|
||||
"Type": {"Value": kind},
|
||||
"Section": {"Name": s},
|
||||
"Type": {"Name": kind},
|
||||
} | {
|
||||
"Offset": offset,
|
||||
"Symbol": {"Value": s},
|
||||
"Type": {"Value": kind},
|
||||
"Symbol": {"Name": s},
|
||||
"Type": {"Name": kind},
|
||||
}:
|
||||
offset += base
|
||||
s = s.removeprefix(self.prefix)
|
||||
|
|
@ -446,13 +446,12 @@ def _handle_relocation(
|
|||
def get_target(host: str) -> _COFF | _ELF | _MachO:
|
||||
"""Build a _Target for the given host "triple" and options."""
|
||||
if re.fullmatch(r"aarch64-apple-darwin.*", host):
|
||||
args = ["-mcmodel=large"]
|
||||
return _MachO(host, alignment=8, args=args, prefix="_")
|
||||
return _MachO(host, alignment=8, prefix="_")
|
||||
if re.fullmatch(r"aarch64-pc-windows-msvc", host):
|
||||
args = ["-fms-runtime-lib=dll"]
|
||||
return _COFF(host, alignment=8, args=args)
|
||||
if re.fullmatch(r"aarch64-.*-linux-gnu", host):
|
||||
args = ["-mcmodel=large"]
|
||||
args = ["-fpic"]
|
||||
return _ELF(host, alignment=8, args=args)
|
||||
if re.fullmatch(r"i686-pc-windows-msvc", host):
|
||||
args = ["-DPy_NO_ENABLE_SHARED"]
|
||||
|
|
@ -463,5 +462,6 @@ def get_target(host: str) -> _COFF | _ELF | _MachO:
|
|||
args = ["-fms-runtime-lib=dll"]
|
||||
return _COFF(host, args=args)
|
||||
if re.fullmatch(r"x86_64-.*-linux-gnu", host):
|
||||
return _ELF(host)
|
||||
args = ["-fpic"]
|
||||
return _ELF(host, args=args)
|
||||
raise ValueError(host)
|
||||
|
|
|
|||
Loading…
Reference in New Issue