Rooba
/
cpython
mirror of https://github.com/python/cpython.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

gh-115999: Specialize `STORE_ATTR` in free-threaded builds. (gh-127838) 

* Add `_PyDictKeys_StringLookupSplit` which does locking on dict keys and
  use in place of `_PyDictKeys_StringLookup`.

* Change `_PyObject_TryGetInstanceAttribute` to use that function
  in the case of split keys.

* Add `unicodekeys_lookup_split` helper which allows code sharing
  between `_Py_dict_lookup` and `_PyDictKeys_StringLookupSplit`.

* Fix locking for `STORE_ATTR_INSTANCE_VALUE`.  Create
  `_GUARD_TYPE_VERSION_AND_LOCK` uop so that object stays locked and
  `tp_version_tag` cannot change.

* Pass `tp_version_tag` to `specialize_dict_access()`, ensuring
  the version we store on the cache is the correct one (in case of
  it changing during the specalize analysis).

* Split `analyze_descriptor` into `analyze_descriptor_load` and
  `analyze_descriptor_store` since those don't share much logic.
  Add `descriptor_is_class` helper function.

* In `specialize_dict_access`, double check `_PyObject_GetManagedDict()`
  in case we race and dict was materialized before the lock.

* Avoid borrowed references in `_Py_Specialize_StoreAttr()`.

* Use `specialize()` and `unspecialize()` helpers.

* Add unit tests to ensure specializing happens as expected in FT builds.

* Add unit tests to attempt to trigger data races (useful for running under TSAN).

* Add `has_split_table` function to `_testinternalcapi`.
This commit is contained in:
Neil Schemenauer 2024-12-19 10:21:17 -08:00 committed by GitHub
parent d2f1d917e8
commit 1b15c89a17
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 716 additions and 297 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Include/internal/pycore_dict.h
View File

@ -114,6 +114,7 @@ extern Py_ssize_t _Py_dict_lookup_threadsafe_stackref(PyDictObject *mp, PyObject
extern Py_ssize_t _PyDict_LookupIndex(PyDictObject *, PyObject *);
extern Py_ssize_t _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyObject *key);
extern Py_ssize_t _PyDictKeys_StringLookupSplit(PyDictKeysObject* dictkeys, PyObject *key);
PyAPI_FUNC(PyObject *)_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *);
PyAPI_FUNC(void) _PyDict_LoadGlobalStackRef(PyDictObject *, PyDictObject *, PyObject *, _PyStackRef *);

4
Include/internal/pycore_opcode_metadata.h generated
View File

@ -2141,7 +2141,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = {
[SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_EXIT_FLAG },
[STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_EXIT_FLAG },
[STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG },
[STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG },
[STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
@ -2340,7 +2340,7 @@ _PyOpcode_macro_expansion[256] = {
[SET_FUNCTION_ATTRIBUTE] = { .nuops = 1, .uops = { { _SET_FUNCTION_ATTRIBUTE, 0, 0 } } },
[SET_UPDATE] = { .nuops = 1, .uops = { { _SET_UPDATE, 0, 0 } } },
[STORE_ATTR] = { .nuops = 1, .uops = { { _STORE_ATTR, 0, 0 } } },
[STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_NO_DICT, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } },
[STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION_AND_LOCK, 2, 1 }, { _GUARD_DORV_NO_DICT, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } },
[STORE_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 } } },
[STORE_ATTR_WITH_HINT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_WITH_HINT, 1, 3 } } },
[STORE_DEREF] = { .nuops = 1, .uops = { { _STORE_DEREF, 0, 0 } } },

201
Include/internal/pycore_uop_ids.h generated
View File

@ -139,15 +139,16 @@ extern "C" {
#define _GUARD_TOS_FLOAT 386
#define _GUARD_TOS_INT 387
#define _GUARD_TYPE_VERSION 388
#define _GUARD_TYPE_VERSION_AND_LOCK 389
#define _IMPORT_FROM IMPORT_FROM
#define _IMPORT_NAME IMPORT_NAME
#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 389
#define _INIT_CALL_PY_EXACT_ARGS 390
#define _INIT_CALL_PY_EXACT_ARGS_0 391
#define _INIT_CALL_PY_EXACT_ARGS_1 392
#define _INIT_CALL_PY_EXACT_ARGS_2 393
#define _INIT_CALL_PY_EXACT_ARGS_3 394
#define _INIT_CALL_PY_EXACT_ARGS_4 395
#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 390
#define _INIT_CALL_PY_EXACT_ARGS 391
#define _INIT_CALL_PY_EXACT_ARGS_0 392
#define _INIT_CALL_PY_EXACT_ARGS_1 393
#define _INIT_CALL_PY_EXACT_ARGS_2 394
#define _INIT_CALL_PY_EXACT_ARGS_3 395
#define _INIT_CALL_PY_EXACT_ARGS_4 396
#define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX
#define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW
#define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER
@ -160,143 +161,143 @@ extern "C" {
#define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE
#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE
#define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE
#define _INTERNAL_INCREMENT_OPT_COUNTER 396
#define _IS_NONE 397
#define _INTERNAL_INCREMENT_OPT_COUNTER 397
#define _IS_NONE 398
#define _IS_OP IS_OP
#define _ITER_CHECK_LIST 398
#define _ITER_CHECK_RANGE 399
#define _ITER_CHECK_TUPLE 400
#define _ITER_JUMP_LIST 401
#define _ITER_JUMP_RANGE 402
#define _ITER_JUMP_TUPLE 403
#define _ITER_NEXT_LIST 404
#define _ITER_NEXT_RANGE 405
#define _ITER_NEXT_TUPLE 406
#define _JUMP_TO_TOP 407
#define _ITER_CHECK_LIST 399
#define _ITER_CHECK_RANGE 400
#define _ITER_CHECK_TUPLE 401
#define _ITER_JUMP_LIST 402
#define _ITER_JUMP_RANGE 403
#define _ITER_JUMP_TUPLE 404
#define _ITER_NEXT_LIST 405
#define _ITER_NEXT_RANGE 406
#define _ITER_NEXT_TUPLE 407
#define _JUMP_TO_TOP 408
#define _LIST_APPEND LIST_APPEND
#define _LIST_EXTEND LIST_EXTEND
#define _LOAD_ATTR 408
#define _LOAD_ATTR_CLASS 409
#define _LOAD_ATTR_CLASS_0 410
#define _LOAD_ATTR_CLASS_1 411
#define _LOAD_ATTR 409
#define _LOAD_ATTR_CLASS 410
#define _LOAD_ATTR_CLASS_0 411
#define _LOAD_ATTR_CLASS_1 412
#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN
#define _LOAD_ATTR_INSTANCE_VALUE 412
#define _LOAD_ATTR_INSTANCE_VALUE_0 413
#define _LOAD_ATTR_INSTANCE_VALUE_1 414
#define _LOAD_ATTR_METHOD_LAZY_DICT 415
#define _LOAD_ATTR_METHOD_NO_DICT 416
#define _LOAD_ATTR_METHOD_WITH_VALUES 417
#define _LOAD_ATTR_MODULE 418
#define _LOAD_ATTR_MODULE_FROM_KEYS 419
#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 420
#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 421
#define _LOAD_ATTR_PROPERTY_FRAME 422
#define _LOAD_ATTR_SLOT 423
#define _LOAD_ATTR_SLOT_0 424
#define _LOAD_ATTR_SLOT_1 425
#define _LOAD_ATTR_WITH_HINT 426
#define _LOAD_ATTR_INSTANCE_VALUE 413
#define _LOAD_ATTR_INSTANCE_VALUE_0 414
#define _LOAD_ATTR_INSTANCE_VALUE_1 415
#define _LOAD_ATTR_METHOD_LAZY_DICT 416
#define _LOAD_ATTR_METHOD_NO_DICT 417
#define _LOAD_ATTR_METHOD_WITH_VALUES 418
#define _LOAD_ATTR_MODULE 419
#define _LOAD_ATTR_MODULE_FROM_KEYS 420
#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 421
#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 422
#define _LOAD_ATTR_PROPERTY_FRAME 423
#define _LOAD_ATTR_SLOT 424
#define _LOAD_ATTR_SLOT_0 425
#define _LOAD_ATTR_SLOT_1 426
#define _LOAD_ATTR_WITH_HINT 427
#define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS
#define _LOAD_BYTECODE 427
#define _LOAD_BYTECODE 428
#define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT
#define _LOAD_CONST LOAD_CONST
#define _LOAD_CONST_IMMORTAL LOAD_CONST_IMMORTAL
#define _LOAD_CONST_INLINE 428
#define _LOAD_CONST_INLINE_BORROW 429
#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 430
#define _LOAD_CONST_INLINE_WITH_NULL 431
#define _LOAD_CONST_INLINE 429
#define _LOAD_CONST_INLINE_BORROW 430
#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 431
#define _LOAD_CONST_INLINE_WITH_NULL 432
#define _LOAD_DEREF LOAD_DEREF
#define _LOAD_FAST 432
#define _LOAD_FAST_0 433
#define _LOAD_FAST_1 434
#define _LOAD_FAST_2 435
#define _LOAD_FAST_3 436
#define _LOAD_FAST_4 437
#define _LOAD_FAST_5 438
#define _LOAD_FAST_6 439
#define _LOAD_FAST_7 440
#define _LOAD_FAST 433
#define _LOAD_FAST_0 434
#define _LOAD_FAST_1 435
#define _LOAD_FAST_2 436
#define _LOAD_FAST_3 437
#define _LOAD_FAST_4 438
#define _LOAD_FAST_5 439
#define _LOAD_FAST_6 440
#define _LOAD_FAST_7 441
#define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR
#define _LOAD_FAST_CHECK LOAD_FAST_CHECK
#define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST
#define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF
#define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS
#define _LOAD_GLOBAL 441
#define _LOAD_GLOBAL_BUILTINS 442
#define _LOAD_GLOBAL_BUILTINS_FROM_KEYS 443
#define _LOAD_GLOBAL_MODULE 444
#define _LOAD_GLOBAL_MODULE_FROM_KEYS 445
#define _LOAD_GLOBAL 442
#define _LOAD_GLOBAL_BUILTINS 443
#define _LOAD_GLOBAL_BUILTINS_FROM_KEYS 444
#define _LOAD_GLOBAL_MODULE 445
#define _LOAD_GLOBAL_MODULE_FROM_KEYS 446
#define _LOAD_LOCALS LOAD_LOCALS
#define _LOAD_NAME LOAD_NAME
#define _LOAD_SMALL_INT 446
#define _LOAD_SMALL_INT_0 447
#define _LOAD_SMALL_INT_1 448
#define _LOAD_SMALL_INT_2 449
#define _LOAD_SMALL_INT_3 450
#define _LOAD_SMALL_INT 447
#define _LOAD_SMALL_INT_0 448
#define _LOAD_SMALL_INT_1 449
#define _LOAD_SMALL_INT_2 450
#define _LOAD_SMALL_INT_3 451
#define _LOAD_SPECIAL LOAD_SPECIAL
#define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR
#define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD
#define _MAKE_CALLARGS_A_TUPLE 451
#define _MAKE_CALLARGS_A_TUPLE 452
#define _MAKE_CELL MAKE_CELL
#define _MAKE_FUNCTION MAKE_FUNCTION
#define _MAKE_WARM 452
#define _MAKE_WARM 453
#define _MAP_ADD MAP_ADD
#define _MATCH_CLASS MATCH_CLASS
#define _MATCH_KEYS MATCH_KEYS
#define _MATCH_MAPPING MATCH_MAPPING
#define _MATCH_SEQUENCE MATCH_SEQUENCE
#define _MAYBE_EXPAND_METHOD 453
#define _MAYBE_EXPAND_METHOD_KW 454
#define _MONITOR_CALL 455
#define _MONITOR_JUMP_BACKWARD 456
#define _MONITOR_RESUME 457
#define _MAYBE_EXPAND_METHOD 454
#define _MAYBE_EXPAND_METHOD_KW 455
#define _MONITOR_CALL 456
#define _MONITOR_JUMP_BACKWARD 457
#define _MONITOR_RESUME 458
#define _NOP NOP
#define _POP_EXCEPT POP_EXCEPT
#define _POP_JUMP_IF_FALSE 458
#define _POP_JUMP_IF_TRUE 459
#define _POP_JUMP_IF_FALSE 459
#define _POP_JUMP_IF_TRUE 460
#define _POP_TOP POP_TOP
#define _POP_TOP_LOAD_CONST_INLINE_BORROW 460
#define _POP_TOP_LOAD_CONST_INLINE_BORROW 461
#define _PUSH_EXC_INFO PUSH_EXC_INFO
#define _PUSH_FRAME 461
#define _PUSH_FRAME 462
#define _PUSH_NULL PUSH_NULL
#define _PY_FRAME_GENERAL 462
#define _PY_FRAME_KW 463
#define _QUICKEN_RESUME 464
#define _REPLACE_WITH_TRUE 465
#define _PY_FRAME_GENERAL 463
#define _PY_FRAME_KW 464
#define _QUICKEN_RESUME 465
#define _REPLACE_WITH_TRUE 466
#define _RESUME_CHECK RESUME_CHECK
#define _RETURN_GENERATOR RETURN_GENERATOR
#define _RETURN_VALUE RETURN_VALUE
#define _SAVE_RETURN_OFFSET 466
#define _SEND 467
#define _SEND_GEN_FRAME 468
#define _SAVE_RETURN_OFFSET 467
#define _SEND 468
#define _SEND_GEN_FRAME 469
#define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS
#define _SET_ADD SET_ADD
#define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE
#define _SET_UPDATE SET_UPDATE
#define _START_EXECUTOR 469
#define _STORE_ATTR 470
#define _STORE_ATTR_INSTANCE_VALUE 471
#define _STORE_ATTR_SLOT 472
#define _STORE_ATTR_WITH_HINT 473
#define _START_EXECUTOR 470
#define _STORE_ATTR 471
#define _STORE_ATTR_INSTANCE_VALUE 472
#define _STORE_ATTR_SLOT 473
#define _STORE_ATTR_WITH_HINT 474
#define _STORE_DEREF STORE_DEREF
#define _STORE_FAST 474
#define _STORE_FAST_0 475
#define _STORE_FAST_1 476
#define _STORE_FAST_2 477
#define _STORE_FAST_3 478
#define _STORE_FAST_4 479
#define _STORE_FAST_5 480
#define _STORE_FAST_6 481
#define _STORE_FAST_7 482
#define _STORE_FAST 475
#define _STORE_FAST_0 476
#define _STORE_FAST_1 477
#define _STORE_FAST_2 478
#define _STORE_FAST_3 479
#define _STORE_FAST_4 480
#define _STORE_FAST_5 481
#define _STORE_FAST_6 482
#define _STORE_FAST_7 483
#define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST
#define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST
#define _STORE_GLOBAL STORE_GLOBAL
#define _STORE_NAME STORE_NAME
#define _STORE_SLICE 483
#define _STORE_SUBSCR 484
#define _STORE_SLICE 484
#define _STORE_SUBSCR 485
#define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT
#define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT
#define _SWAP SWAP
#define _TIER2_RESUME_CHECK 485
#define _TO_BOOL 486
#define _TIER2_RESUME_CHECK 486
#define _TO_BOOL 487
#define _TO_BOOL_BOOL TO_BOOL_BOOL
#define _TO_BOOL_INT TO_BOOL_INT
#define _TO_BOOL_LIST TO_BOOL_LIST
@ -306,13 +307,13 @@ extern "C" {
#define _UNARY_NEGATIVE UNARY_NEGATIVE
#define _UNARY_NOT UNARY_NOT
#define _UNPACK_EX UNPACK_EX
#define _UNPACK_SEQUENCE 487
#define _UNPACK_SEQUENCE 488
#define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST
#define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE
#define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE
#define _WITH_EXCEPT_START WITH_EXCEPT_START
#define _YIELD_VALUE YIELD_VALUE
#define MAX_UOP_ID 487
#define MAX_UOP_ID 488
#ifdef __cplusplus
}

6
Include/internal/pycore_uop_metadata.h generated
View File

@ -148,6 +148,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
[_LOAD_SUPER_ATTR_METHOD] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_LOAD_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_GUARD_TYPE_VERSION] = HAS_EXIT_FLAG,
[_GUARD_TYPE_VERSION_AND_LOCK] = HAS_EXIT_FLAG,
[_CHECK_MANAGED_OBJECT_HAS_VALUES] = HAS_DEOPT_FLAG,
[_LOAD_ATTR_INSTANCE_VALUE_0] = HAS_DEOPT_FLAG,
[_LOAD_ATTR_INSTANCE_VALUE_1] = HAS_DEOPT_FLAG,
@ -167,7 +168,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
[_GUARD_DORV_NO_DICT] = HAS_EXIT_FLAG,
[_STORE_ATTR_INSTANCE_VALUE] = 0,
[_STORE_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG,
[_STORE_ATTR_SLOT] = 0,
[_STORE_ATTR_SLOT] = HAS_DEOPT_FLAG,
[_COMPARE_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_COMPARE_OP_FLOAT] = HAS_ARG_FLAG,
[_COMPARE_OP_INT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG,
@ -428,6 +429,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {
[_GUARD_TOS_FLOAT] = "_GUARD_TOS_FLOAT",
[_GUARD_TOS_INT] = "_GUARD_TOS_INT",
[_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION",
[_GUARD_TYPE_VERSION_AND_LOCK] = "_GUARD_TYPE_VERSION_AND_LOCK",
[_IMPORT_FROM] = "_IMPORT_FROM",
[_IMPORT_NAME] = "_IMPORT_NAME",
[_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS",
@ -839,6 +841,8 @@ int _PyUop_num_popped(int opcode, int oparg)
return 1;
case _GUARD_TYPE_VERSION:
return 0;
case _GUARD_TYPE_VERSION_AND_LOCK:
return 0;
case _CHECK_MANAGED_OBJECT_HAS_VALUES:
return 0;
case _LOAD_ATTR_INSTANCE_VALUE_0:

141
Lib/test/test_free_threading/test_races.py
View File

@ -4,6 +4,7 @@
import threading
import time
import unittest
import _testinternalcapi
from test.support import threading_helper
@ -129,6 +130,146 @@ def mutate():
# with the cell binding being changed).
do_race(access, mutate)
def test_racing_to_bool(self):
seq = [1]
class C:
def __bool__(self):
return False
def access():
if seq:
return 1
else:
return 2
def mutate():
nonlocal seq
seq = [1]
time.sleep(0)
seq = C()
time.sleep(0)
do_race(access, mutate)
def test_racing_store_attr_slot(self):
class C:
__slots__ = ['x', '__dict__']
c = C()
def set_slot():
for i in range(10):
c.x = i
time.sleep(0)
def change_type():
def set_x(self, x):
pass
def get_x(self):
pass
C.x = property(get_x, set_x)
time.sleep(0)
del C.x
time.sleep(0)
do_race(set_slot, change_type)
def set_getattribute():
C.__getattribute__ = lambda self, x: x
time.sleep(0)
del C.__getattribute__
time.sleep(0)
do_race(set_slot, set_getattribute)
def test_racing_store_attr_instance_value(self):
class C:
pass
c = C()
def set_value():
for i in range(100):
c.x = i
set_value()
def read():
x = c.x
def mutate():
# Adding a property for 'x' should unspecialize it.
C.x = property(lambda self: None, lambda self, x: None)
time.sleep(0)
del C.x
time.sleep(0)
do_race(read, mutate)
def test_racing_store_attr_with_hint(self):
class C:
pass
c = C()
for i in range(29):
setattr(c, f"_{i}", None)
def set_value():
for i in range(100):
c.x = i
set_value()
def read():
x = c.x
def mutate():
# Adding a property for 'x' should unspecialize it.
C.x = property(lambda self: None, lambda self, x: None)
time.sleep(0)
del C.x
time.sleep(0)
do_race(read, mutate)
def make_shared_key_dict(self):
class C:
pass
a = C()
a.x = 1
return a.__dict__
def test_racing_store_attr_dict(self):
"""Test STORE_ATTR with various dictionary types."""
class C:
pass
c = C()
def set_value():
for i in range(20):
c.x = i
def mutate():
nonlocal c
c.x = 1
self.assertTrue(_testinternalcapi.has_inline_values(c))
for i in range(30):
setattr(c, f"_{i}", None)
self.assertFalse(_testinternalcapi.has_inline_values(c.__dict__))
c.__dict__ = self.make_shared_key_dict()
self.assertTrue(_testinternalcapi.has_split_table(c.__dict__))
c.__dict__[1] = None
self.assertFalse(_testinternalcapi.has_split_table(c.__dict__))
c = C()
do_race(set_value, mutate)
if __name__ == "__main__":
unittest.main()

66
Lib/test/test_opcache.py
View File

@ -1383,6 +1383,72 @@ def send_yield_from():
self.assert_specialized(send_yield_from, "SEND_GEN")
self.assert_no_opcode(send_yield_from, "SEND")
@cpython_only
@requires_specialization_ft
def test_store_attr_slot(self):
class C:
__slots__ = ['x']
def set_slot():
c = C()
for i in range(100):
c.x = i
set_slot()
self.assert_specialized(set_slot, "STORE_ATTR_SLOT")
self.assert_no_opcode(set_slot, "STORE_ATTR")
# Adding a property for 'x' should unspecialize it.
C.x = property(lambda self: None, lambda self, x: None)
set_slot()
self.assert_no_opcode(set_slot, "STORE_ATTR_SLOT")
@cpython_only
@requires_specialization_ft
def test_store_attr_instance_value(self):
class C:
pass
def set_value():
c = C()
for i in range(100):
c.x = i
set_value()
self.assert_specialized(set_value, "STORE_ATTR_INSTANCE_VALUE")
self.assert_no_opcode(set_value, "STORE_ATTR")
# Adding a property for 'x' should unspecialize it.
C.x = property(lambda self: None, lambda self, x: None)
set_value()
self.assert_no_opcode(set_value, "STORE_ATTR_INSTANCE_VALUE")
@cpython_only
@requires_specialization_ft
def test_store_attr_with_hint(self):
class C:
pass
c = C()
for i in range(29):
setattr(c, f"_{i}", None)
def set_value():
for i in range(100):
c.x = i
set_value()
self.assert_specialized(set_value, "STORE_ATTR_WITH_HINT")
self.assert_no_opcode(set_value, "STORE_ATTR")
# Adding a property for 'x' should unspecialize it.
C.x = property(lambda self: None, lambda self, x: None)
set_value()
self.assert_no_opcode(set_value, "STORE_ATTR_WITH_HINT")
@cpython_only
@requires_specialization_ft
def test_to_bool(self):

9
Modules/_testinternalcapi.c
View File

@ -1989,6 +1989,14 @@ has_inline_values(PyObject *self, PyObject *obj)
Py_RETURN_FALSE;
}
static PyObject *
has_split_table(PyObject *self, PyObject *obj)
{
if (PyDict_Check(obj) && _PyDict_HasSplitTable((PyDictObject *)obj)) {
Py_RETURN_TRUE;
}
Py_RETURN_FALSE;
}
// Circumvents standard version assignment machinery - use with caution and only on
// short-lived heap types
@ -2139,6 +2147,7 @@ static PyMethodDef module_functions[] = {
{"get_rare_event_counters", get_rare_event_counters, METH_NOARGS},
{"reset_rare_event_counters", reset_rare_event_counters, METH_NOARGS},
{"has_inline_values", has_inline_values, METH_O},
{"has_split_table", has_split_table, METH_O},
{"type_assign_specific_version_unsafe", type_assign_specific_version_unsafe, METH_VARARGS,
PyDoc_STR("forcefully assign type->tp_version_tag")},

66
Objects/dictobject.c
View File

@ -1129,6 +1129,35 @@ dictkeys_generic_lookup(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, P
return do_lookup(mp, dk, key, hash, compare_generic);
}
#ifdef Py_GIL_DISABLED
static Py_ssize_t
unicodekeys_lookup_unicode_threadsafe(PyDictKeysObject* dk, PyObject *key,
Py_hash_t hash);
#endif
static Py_ssize_t
unicodekeys_lookup_split(PyDictKeysObject* dk, PyObject *key, Py_hash_t hash)
{
Py_ssize_t ix;
assert(dk->dk_kind == DICT_KEYS_SPLIT);
assert(PyUnicode_CheckExact(key));
#ifdef Py_GIL_DISABLED
// A split dictionaries keys can be mutated by other dictionaries
// but if we have a unicode key we can avoid locking the shared
// keys.
ix = unicodekeys_lookup_unicode_threadsafe(dk, key, hash);
if (ix == DKIX_KEY_CHANGED) {
LOCK_KEYS(dk);
ix = unicodekeys_lookup_unicode(dk, key, hash);
UNLOCK_KEYS(dk);
}
#else
ix = unicodekeys_lookup_unicode(dk, key, hash);
#endif
return ix;
}
/* Lookup a string in a (all unicode) dict keys.
* Returns DKIX_ERROR if key is not a string,
* or if the dict keys is not all strings.
@ -1153,13 +1182,24 @@ _PyDictKeys_StringLookup(PyDictKeysObject* dk, PyObject *key)
return unicodekeys_lookup_unicode(dk, key, hash);
}
#ifdef Py_GIL_DISABLED
static Py_ssize_t
unicodekeys_lookup_unicode_threadsafe(PyDictKeysObject* dk, PyObject *key,
Py_hash_t hash);
#endif
/* Like _PyDictKeys_StringLookup() but only works on split keys. Note
* that in free-threaded builds this locks the keys object as required.
*/
Py_ssize_t
_PyDictKeys_StringLookupSplit(PyDictKeysObject* dk, PyObject *key)
{
assert(dk->dk_kind == DICT_KEYS_SPLIT);
assert(PyUnicode_CheckExact(key));
Py_hash_t hash = unicode_get_hash(key);
if (hash == -1) {
hash = PyUnicode_Type.tp_hash(key);
if (hash == -1) {
PyErr_Clear();
return DKIX_ERROR;
}
}
return unicodekeys_lookup_split(dk, key, hash);
}
/*
The basic lookup function used by all operations.
@ -1192,15 +1232,7 @@ _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **valu
if (PyUnicode_CheckExact(key)) {
#ifdef Py_GIL_DISABLED
if (kind == DICT_KEYS_SPLIT) {
// A split dictionaries keys can be mutated by other
// dictionaries but if we have a unicode key we can avoid
// locking the shared keys.
ix = unicodekeys_lookup_unicode_threadsafe(dk, key, hash);
if (ix == DKIX_KEY_CHANGED) {
LOCK_KEYS(dk);
ix = unicodekeys_lookup_unicode(dk, key, hash);
UNLOCK_KEYS(dk);
}
ix = unicodekeys_lookup_split(dk, key, hash);
}
else {
ix = unicodekeys_lookup_unicode(dk, key, hash);
@ -6967,7 +6999,7 @@ _PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name, PyObject **attr
PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj));
assert(keys != NULL);
Py_ssize_t ix = _PyDictKeys_StringLookup(keys, name);
Py_ssize_t ix = _PyDictKeys_StringLookupSplit(keys, name);
if (ix == DKIX_EMPTY) {
*attr = NULL;
return true;

65
Python/bytecodes.c
View File

@ -1467,7 +1467,7 @@ dummy_func(
};
specializing op(_SPECIALIZE_STORE_ATTR, (counter/1, owner -- owner)) {
#if ENABLE_SPECIALIZATION
#if ENABLE_SPECIALIZATION_FT
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
next_instr = this_instr;
@ -1476,7 +1476,7 @@ dummy_func(
}
OPCODE_DEFERRED_INC(STORE_ATTR);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
#endif /* ENABLE_SPECIALIZATION_FT */
}
op(_STORE_ATTR, (v, owner --)) {
@ -2129,7 +2129,18 @@ dummy_func(
op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) {
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
EXIT_IF(tp->tp_version_tag != type_version);
EXIT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version);
}
op(_GUARD_TYPE_VERSION_AND_LOCK, (type_version/2, owner -- owner)) {
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(type_version != 0);
EXIT_IF(!LOCK_OBJECT(owner_o));
PyTypeObject *tp = Py_TYPE(owner_o);
if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
UNLOCK_OBJECT(owner_o);
EXIT_IF(true);
}
}
op(_CHECK_MANAGED_OBJECT_HAS_VALUES, (owner -- owner)) {
@ -2336,8 +2347,11 @@ dummy_func(
assert(Py_TYPE(owner_o)->tp_dictoffset < 0);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
EXIT_IF(_PyObject_GetManagedDict(owner_o));
EXIT_IF(_PyObject_InlineValues(owner_o)->valid == 0);
if (_PyObject_GetManagedDict(owner_o) ||
!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) {
UNLOCK_OBJECT(owner_o);
EXIT_IF(true);
}
}
op(_STORE_ATTR_INSTANCE_VALUE, (offset/1, value, owner --)) {
@ -2347,21 +2361,20 @@ dummy_func(
assert(_PyObject_GetManagedDict(owner_o) == NULL);
PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset);
PyObject *old_value = *value_ptr;
*value_ptr = PyStackRef_AsPyObjectSteal(value);
FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value));
if (old_value == NULL) {
PyDictValues *values = _PyObject_InlineValues(owner_o);
Py_ssize_t index = value_ptr - values->values;
_PyDictValues_AddToInsertionOrder(values, index);
}
else {
Py_DECREF(old_value);
}
UNLOCK_OBJECT(owner_o);
Py_XDECREF(old_value);
PyStackRef_CLOSE(owner);
}
macro(STORE_ATTR_INSTANCE_VALUE) =
unused/1 +
_GUARD_TYPE_VERSION +
_GUARD_TYPE_VERSION_AND_LOCK +
_GUARD_DORV_NO_DICT +
_STORE_ATTR_INSTANCE_VALUE;
@ -2370,16 +2383,34 @@ dummy_func(
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
DEOPT_IF(dict == NULL);
DEOPT_IF(!LOCK_OBJECT(dict));
#ifdef Py_GIL_DISABLED
if (dict != _PyObject_GetManagedDict(owner_o)) {
UNLOCK_OBJECT(dict);
DEOPT_IF(true);
}
#endif
assert(PyDict_CheckExact((PyObject *)dict));
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries);
DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys));
if (hint >= (size_t)dict->ma_keys->dk_nentries ||
!DK_IS_UNICODE(dict->ma_keys)) {
UNLOCK_OBJECT(dict);
DEOPT_IF(true);
}
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
DEOPT_IF(ep->me_key != name);
if (ep->me_key != name) {
UNLOCK_OBJECT(dict);
DEOPT_IF(true);
}
PyObject *old_value = ep->me_value;
DEOPT_IF(old_value == NULL);
if (old_value == NULL) {
UNLOCK_OBJECT(dict);
DEOPT_IF(true);
}
_PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value));
ep->me_value = PyStackRef_AsPyObjectSteal(value);
FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value));
UNLOCK_OBJECT(dict);
// old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault,
// when dict only holds the strong reference to value in ep->me_value.
Py_XDECREF(old_value);
@ -2395,10 +2426,12 @@ dummy_func(
op(_STORE_ATTR_SLOT, (index/1, value, owner --)) {
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
DEOPT_IF(!LOCK_OBJECT(owner_o));
char *addr = (char *)owner_o + index;
STAT_INC(STORE_ATTR, hit);
PyObject *old_value = *(PyObject **)addr;
*(PyObject **)addr = PyStackRef_AsPyObjectSteal(value);
FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value));
UNLOCK_OBJECT(owner_o);
Py_XDECREF(old_value);
PyStackRef_CLOSE(owner);
}

93
Python/executor_cases.c.h generated
View File

@ -2574,13 +2574,34 @@
uint32_t type_version = (uint32_t)CURRENT_OPERAND0();
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
if (tp->tp_version_tag != type_version) {
if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _GUARD_TYPE_VERSION_AND_LOCK: {
_PyStackRef owner;
owner = stack_pointer[-1];
uint32_t type_version = (uint32_t)CURRENT_OPERAND0();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(type_version != 0);
if (!LOCK_OBJECT(owner_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
PyTypeObject *tp = Py_TYPE(owner_o);
if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
UNLOCK_OBJECT(owner_o);
if (true) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
break;
}
case _CHECK_MANAGED_OBJECT_HAS_VALUES: {
_PyStackRef owner;
owner = stack_pointer[-1];
@ -2910,13 +2931,13 @@
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_dictoffset < 0);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
if (_PyObject_GetManagedDict(owner_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (_PyObject_InlineValues(owner_o)->valid == 0) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
if (_PyObject_GetManagedDict(owner_o) ||
!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) {
UNLOCK_OBJECT(owner_o);
if (true) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
break;
}
@ -2932,15 +2953,14 @@
assert(_PyObject_GetManagedDict(owner_o) == NULL);
PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset);
PyObject *old_value = *value_ptr;
*value_ptr = PyStackRef_AsPyObjectSteal(value);
FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value));
if (old_value == NULL) {
PyDictValues *values = _PyObject_InlineValues(owner_o);
Py_ssize_t index = value_ptr - values->values;
_PyDictValues_AddToInsertionOrder(values, index);
}
else {
Py_DECREF(old_value);
}
UNLOCK_OBJECT(owner_o);
Py_XDECREF(old_value);
PyStackRef_CLOSE(owner);
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
@ -2961,30 +2981,50 @@
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
assert(PyDict_CheckExact((PyObject *)dict));
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
if (hint >= (size_t)dict->ma_keys->dk_nentries) {
if (!LOCK_OBJECT(dict)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (!DK_IS_UNICODE(dict->ma_keys)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
#ifdef Py_GIL_DISABLED
if (dict != _PyObject_GetManagedDict(owner_o)) {
UNLOCK_OBJECT(dict);
if (true) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
#endif
assert(PyDict_CheckExact((PyObject *)dict));
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
if (hint >= (size_t)dict->ma_keys->dk_nentries ||
!DK_IS_UNICODE(dict->ma_keys)) {
UNLOCK_OBJECT(dict);
if (true) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
if (ep->me_key != name) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
UNLOCK_OBJECT(dict);
if (true) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
PyObject *old_value = ep->me_value;
if (old_value == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
UNLOCK_OBJECT(dict);
if (true) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value));
stack_pointer = _PyFrame_GetStackPointer(frame);
ep->me_value = PyStackRef_AsPyObjectSteal(value);
FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value));
UNLOCK_OBJECT(dict);
// old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault,
// when dict only holds the strong reference to value in ep->me_value.
Py_XDECREF(old_value);
@ -3002,10 +3042,15 @@
value = stack_pointer[-2];
uint16_t index = (uint16_t)CURRENT_OPERAND0();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
if (!LOCK_OBJECT(owner_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
char *addr = (char *)owner_o + index;
STAT_INC(STORE_ATTR, hit);
PyObject *old_value = *(PyObject **)addr;
*(PyObject **)addr = PyStackRef_AsPyObjectSteal(value);
FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value));
UNLOCK_OBJECT(owner_o);
Py_XDECREF(old_value);
PyStackRef_CLOSE(owner);
stack_pointer += -2;

86
Python/generated_cases.c.h generated
View File

@ -5319,7 +5319,7 @@
uint32_t type_version = read_u32(&this_instr[4].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
}
// _LOAD_ATTR_CLASS
{
@ -5388,7 +5388,7 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
}
// _CHECK_MANAGED_OBJECT_HAS_VALUES
{
@ -5433,7 +5433,7 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
}
// _CHECK_ATTR_METHOD_LAZY_DICT
{
@ -5476,7 +5476,7 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
}
/* Skip 2 cache entries */
// _LOAD_ATTR_METHOD_NO_DICT
@ -5512,7 +5512,7 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
}
// _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT
{
@ -5611,7 +5611,7 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
}
/* Skip 2 cache entries */
// _LOAD_ATTR_NONDESCRIPTOR_NO_DICT
@ -5642,7 +5642,7 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
}
// _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT
{
@ -5688,7 +5688,7 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
}
/* Skip 2 cache entries */
// _LOAD_ATTR_PROPERTY_FRAME
@ -5750,7 +5750,7 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
}
// _LOAD_ATTR_SLOT
{
@ -5787,7 +5787,7 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
}
// _CHECK_ATTR_WITH_HINT
{
@ -7314,7 +7314,7 @@
owner = stack_pointer[-1];
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
#if ENABLE_SPECIALIZATION_FT
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
next_instr = this_instr;
@ -7325,7 +7325,7 @@
}
OPCODE_DEFERRED_INC(STORE_ATTR);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
#endif /* ENABLE_SPECIALIZATION_FT */
}
/* Skip 3 cache entries */
// _STORE_ATTR
@ -7353,21 +7353,29 @@
_PyStackRef owner;
_PyStackRef value;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
// _GUARD_TYPE_VERSION_AND_LOCK
{
owner = stack_pointer[-1];
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR);
PyTypeObject *tp = Py_TYPE(owner_o);
if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
UNLOCK_OBJECT(owner_o);
DEOPT_IF(true, STORE_ATTR);
}
}
// _GUARD_DORV_NO_DICT
{
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_dictoffset < 0);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
DEOPT_IF(_PyObject_GetManagedDict(owner_o), STORE_ATTR);
DEOPT_IF(_PyObject_InlineValues(owner_o)->valid == 0, STORE_ATTR);
if (_PyObject_GetManagedDict(owner_o) ||
!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) {
UNLOCK_OBJECT(owner_o);
DEOPT_IF(true, STORE_ATTR);
}
}
// _STORE_ATTR_INSTANCE_VALUE
{
@ -7378,15 +7386,14 @@
assert(_PyObject_GetManagedDict(owner_o) == NULL);
PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset);
PyObject *old_value = *value_ptr;
*value_ptr = PyStackRef_AsPyObjectSteal(value);
FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value));
if (old_value == NULL) {
PyDictValues *values = _PyObject_InlineValues(owner_o);
Py_ssize_t index = value_ptr - values->values;
_PyDictValues_AddToInsertionOrder(values, index);
}
else {
Py_DECREF(old_value);
}
UNLOCK_OBJECT(owner_o);
Py_XDECREF(old_value);
PyStackRef_CLOSE(owner);
}
stack_pointer += -2;
@ -7408,17 +7415,19 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR);
}
// _STORE_ATTR_SLOT
{
value = stack_pointer[-2];
uint16_t index = read_u16(&this_instr[4].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR);
char *addr = (char *)owner_o + index;
STAT_INC(STORE_ATTR, hit);
PyObject *old_value = *(PyObject **)addr;
*(PyObject **)addr = PyStackRef_AsPyObjectSteal(value);
FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value));
UNLOCK_OBJECT(owner_o);
Py_XDECREF(old_value);
PyStackRef_CLOSE(owner);
}
@ -7441,7 +7450,7 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR);
}
// _STORE_ATTR_WITH_HINT
{
@ -7451,18 +7460,35 @@
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
DEOPT_IF(dict == NULL, STORE_ATTR);
DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR);
#ifdef Py_GIL_DISABLED
if (dict != _PyObject_GetManagedDict(owner_o)) {
UNLOCK_OBJECT(dict);
DEOPT_IF(true, STORE_ATTR);
}
#endif
assert(PyDict_CheckExact((PyObject *)dict));
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR);
DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys), STORE_ATTR);
if (hint >= (size_t)dict->ma_keys->dk_nentries ||
!DK_IS_UNICODE(dict->ma_keys)) {
UNLOCK_OBJECT(dict);
DEOPT_IF(true, STORE_ATTR);
}
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
DEOPT_IF(ep->me_key != name, STORE_ATTR);
if (ep->me_key != name) {
UNLOCK_OBJECT(dict);
DEOPT_IF(true, STORE_ATTR);
}
PyObject *old_value = ep->me_value;
DEOPT_IF(old_value == NULL, STORE_ATTR);
if (old_value == NULL) {
UNLOCK_OBJECT(dict);
DEOPT_IF(true, STORE_ATTR);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value));
stack_pointer = _PyFrame_GetStackPointer(frame);
ep->me_value = PyStackRef_AsPyObjectSteal(value);
FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value));
UNLOCK_OBJECT(dict);
// old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault,
// when dict only holds the strong reference to value in ep->me_value.
Py_XDECREF(old_value);
@ -7815,7 +7841,7 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, TO_BOOL);
DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL);
}
// _REPLACE_WITH_TRUE
{

4
Python/optimizer_cases.c.h generated
View File

@ -1121,6 +1121,10 @@
break;
}
case _GUARD_TYPE_VERSION_AND_LOCK: {
break;
}
case _CHECK_MANAGED_OBJECT_HAS_VALUES: {
break;
}

271
Python/specialize.c
View File

@ -741,8 +741,8 @@ static int function_kind(PyCodeObject *code);
#ifndef Py_GIL_DISABLED
static bool function_check_args(PyObject *o, int expected_argcount, int opcode);
static uint32_t function_get_version(PyObject *o, int opcode);
#endif
static uint32_t type_get_version(PyTypeObject *t, int opcode);
#endif
static int
specialize_module_load_attr_lock_held(PyDictObject *dict, _Py_CODEUNIT *instr, PyObject *name)
@ -881,71 +881,142 @@ classify_descriptor(PyObject *descriptor, bool has_getattr)
return NON_DESCRIPTOR;
}
static DescriptorClassification
analyze_descriptor(PyTypeObject *type, PyObject *name, PyObject **descr, int store)
static bool
descriptor_is_class(PyObject *descriptor, PyObject *name)
{
return ((PyUnicode_CompareWithASCIIString(name, "__class__") == 0) &&
(descriptor == _PyType_Lookup(&PyBaseObject_Type, name)));
}
#ifndef Py_GIL_DISABLED
static DescriptorClassification
analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr) {
bool has_getattr = false;
if (store) {
if (type->tp_setattro != PyObject_GenericSetAttr) {
getattrofunc getattro_slot = type->tp_getattro;
if (getattro_slot == PyObject_GenericGetAttr) {
/* Normal attribute lookup; */
has_getattr = false;
}
else if (getattro_slot == _Py_slot_tp_getattr_hook ||
getattro_slot == _Py_slot_tp_getattro) {
/* One or both of __getattribute__ or __getattr__ may have been
overridden See typeobject.c for why these functions are special. */
PyObject *getattribute = _PyType_LookupRef(type, &_Py_ID(__getattribute__));
PyInterpreterState *interp = _PyInterpreterState_GET();
bool has_custom_getattribute = getattribute != NULL &&
getattribute != interp->callable_cache.object__getattribute__;
PyObject *getattr = _PyType_LookupRef(type, &_Py_ID(__getattr__));
has_getattr = getattr != NULL;
Py_XDECREF(getattr);
if (has_custom_getattribute) {
if (getattro_slot == _Py_slot_tp_getattro &&
!has_getattr &&
Py_IS_TYPE(getattribute, &PyFunction_Type)) {
*descr = getattribute;
return GETATTRIBUTE_IS_PYTHON_FUNCTION;
}
/* Potentially both __getattr__ and __getattribute__ are set.
Too complicated */
Py_DECREF(getattribute);
*descr = NULL;
return GETSET_OVERRIDDEN;
}
/* Potentially has __getattr__ but no custom __getattribute__.
Fall through to usual descriptor analysis.
Usual attribute lookup should only be allowed at runtime
if we can guarantee that there is no way an exception can be
raised. This means some specializations, e.g. specializing
for property() isn't safe.
*/
Py_XDECREF(getattribute);
}
else {
getattrofunc getattro_slot = type->tp_getattro;
if (getattro_slot == PyObject_GenericGetAttr) {
/* Normal attribute lookup; */
has_getattr = false;
}
else if (getattro_slot == _Py_slot_tp_getattr_hook ||
getattro_slot == _Py_slot_tp_getattro) {
/* One or both of __getattribute__ or __getattr__ may have been
overridden See typeobject.c for why these functions are special. */
PyObject *getattribute = _PyType_Lookup(type,
&_Py_ID(__getattribute__));
PyInterpreterState *interp = _PyInterpreterState_GET();
bool has_custom_getattribute = getattribute != NULL &&
getattribute != interp->callable_cache.object__getattribute__;
has_getattr = _PyType_Lookup(type, &_Py_ID(__getattr__)) != NULL;
if (has_custom_getattribute) {
if (getattro_slot == _Py_slot_tp_getattro &&
!has_getattr &&
Py_IS_TYPE(getattribute, &PyFunction_Type)) {
*descr = getattribute;
return GETATTRIBUTE_IS_PYTHON_FUNCTION;
}
/* Potentially both __getattr__ and __getattribute__ are set.
Too complicated */
*descr = NULL;
return GETSET_OVERRIDDEN;
}
/* Potentially has __getattr__ but no custom __getattribute__.
Fall through to usual descriptor analysis.
Usual attribute lookup should only be allowed at runtime
if we can guarantee that there is no way an exception can be
raised. This means some specializations, e.g. specializing
for property() isn't safe.
*/
}
else {
*descr = NULL;
return GETSET_OVERRIDDEN;
}
*descr = NULL;
return GETSET_OVERRIDDEN;
}
PyObject *descriptor = _PyType_Lookup(type, name);
PyObject *descriptor = _PyType_LookupRef(type, name);
*descr = descriptor;
if (PyUnicode_CompareWithASCIIString(name, "__class__") == 0) {
if (descriptor == _PyType_Lookup(&PyBaseObject_Type, name)) {
return DUNDER_CLASS;
}
if (descriptor_is_class(descriptor, name)) {
return DUNDER_CLASS;
}
return classify_descriptor(descriptor, has_getattr);
}
#endif //!Py_GIL_DISABLED
static DescriptorClassification
analyze_descriptor_store(PyTypeObject *type, PyObject *name, PyObject **descr, unsigned int *tp_version)
{
if (type->tp_setattro != PyObject_GenericSetAttr) {
*descr = NULL;
return GETSET_OVERRIDDEN;
}
PyObject *descriptor = _PyType_LookupRefAndVersion(type, name, tp_version);
*descr = descriptor;
if (descriptor_is_class(descriptor, name)) {
return DUNDER_CLASS;
}
return classify_descriptor(descriptor, false);
}
static int
specialize_dict_access_inline(
PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type,
DescriptorClassification kind, PyObject *name, unsigned int tp_version,
int base_op, int values_op)
{
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys;
assert(PyUnicode_CheckExact(name));
Py_ssize_t index = _PyDictKeys_StringLookupSplit(keys, name);
assert (index != DKIX_ERROR);
if (index == DKIX_EMPTY) {
SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_IN_KEYS);
return 0;
}
assert(index >= 0);
char *value_addr = (char *)&_PyObject_InlineValues(owner)->values[index];
Py_ssize_t offset = value_addr - (char *)owner;
if (offset != (uint16_t)offset) {
SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE);
return 0;
}
cache->index = (uint16_t)offset;
write_u32(cache->version, tp_version);
specialize(instr, values_op);
return 1;
}
static int
specialize_dict_access_hint(
PyDictObject *dict, _Py_CODEUNIT *instr, PyTypeObject *type,
DescriptorClassification kind, PyObject *name, unsigned int tp_version,
int base_op, int hint_op)
{
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
// We found an instance with a __dict__.
if (_PyDict_HasSplitTable(dict)) {
SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_SPLIT_DICT);
return 0;
}
Py_ssize_t index = _PyDict_LookupIndex(dict, name);
if (index != (uint16_t)index) {
SPECIALIZATION_FAIL(base_op,
index == DKIX_EMPTY ?
SPEC_FAIL_ATTR_NOT_IN_DICT :
SPEC_FAIL_OUT_OF_RANGE);
return 0;
}
cache->index = (uint16_t)index;
write_u32(cache->version, tp_version);
specialize(instr, hint_op);
return 1;
}
static int
specialize_dict_access(
PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type,
DescriptorClassification kind, PyObject *name,
DescriptorClassification kind, PyObject *name, unsigned int tp_version,
int base_op, int values_op, int hint_op)
{
assert(kind == NON_OVERRIDING || kind == NON_DESCRIPTOR || kind == ABSENT ||
@ -956,29 +1027,25 @@ specialize_dict_access(
SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_MANAGED_DICT);
return 0;
}
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES &&
_PyObject_InlineValues(owner)->valid &&
FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner)->valid) &&
!(base_op == STORE_ATTR && _PyObject_GetManagedDict(owner) != NULL))
{
PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys;
assert(PyUnicode_CheckExact(name));
Py_ssize_t index = _PyDictKeys_StringLookup(keys, name);
assert (index != DKIX_ERROR);
if (index == DKIX_EMPTY) {
SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_IN_KEYS);
return 0;
int res;
Py_BEGIN_CRITICAL_SECTION(owner);
PyDictObject *dict = _PyObject_GetManagedDict(owner);
if (dict == NULL) {
// managed dict, not materialized, inline values valid
res = specialize_dict_access_inline(owner, instr, type, kind, name,
tp_version, base_op, values_op);
}
assert(index >= 0);
char *value_addr = (char *)&_PyObject_InlineValues(owner)->values[index];
Py_ssize_t offset = value_addr - (char *)owner;
if (offset != (uint16_t)offset) {
SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE);
return 0;
else {
// lost race and dict was created, fail specialization
SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OTHER);
res = 0;
}
write_u32(cache->version, type->tp_version_tag);
cache->index = (uint16_t)offset;
specialize(instr, values_op);
Py_END_CRITICAL_SECTION();
return res;
}
else {
PyDictObject *dict = _PyObject_GetManagedDict(owner);
@ -986,25 +1053,14 @@ specialize_dict_access(
SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT);
return 0;
}
// We found an instance with a __dict__.
if (dict->ma_values) {
SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_SPLIT_DICT);
return 0;
}
Py_ssize_t index =
_PyDict_LookupIndex(dict, name);
if (index != (uint16_t)index) {
SPECIALIZATION_FAIL(base_op,
index == DKIX_EMPTY ?
SPEC_FAIL_ATTR_NOT_IN_DICT :
SPEC_FAIL_OUT_OF_RANGE);
return 0;
}
cache->index = (uint16_t)index;
write_u32(cache->version, type->tp_version_tag);
specialize(instr, hint_op);
int res;
Py_BEGIN_CRITICAL_SECTION(dict);
// materialized managed dict
res = specialize_dict_access_hint(dict, instr, type, kind, name,
tp_version, base_op, hint_op);
Py_END_CRITICAL_SECTION();
return res;
}
return 1;
}
#ifndef Py_GIL_DISABLED
@ -1050,7 +1106,8 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na
PyTypeObject *type = Py_TYPE(owner);
bool shadow = instance_has_key(owner, name);
PyObject *descr = NULL;
DescriptorClassification kind = analyze_descriptor(type, name, &descr, 0);
DescriptorClassification kind = analyze_descriptor_load(type, name, &descr);
Py_XDECREF(descr); // turn strong ref into a borrowed ref
assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN);
if (type_get_version(type, LOAD_ATTR) == 0) {
return -1;
@ -1204,8 +1261,8 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na
}
Py_UNREACHABLE();
try_instance:
if (specialize_dict_access(owner, instr, type, kind, name, LOAD_ATTR,
LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT))
if (specialize_dict_access(owner, instr, type, kind, name, type->tp_version_tag,
LOAD_ATTR, LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT))
{
return 0;
}
@ -1259,8 +1316,9 @@ _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *na
{
PyObject *owner = PyStackRef_AsPyObjectBorrow(owner_st);
assert(ENABLE_SPECIALIZATION);
assert(ENABLE_SPECIALIZATION_FT);
assert(_PyOpcode_Caches[STORE_ATTR] == INLINE_CACHE_ENTRIES_STORE_ATTR);
PyObject *descr = NULL;
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
PyTypeObject *type = Py_TYPE(owner);
if (!_PyType_IsReady(type)) {
@ -1274,11 +1332,12 @@ _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *na
SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OVERRIDDEN);
goto fail;
}
PyObject *descr;
DescriptorClassification kind = analyze_descriptor(type, name, &descr, 1);
if (type_get_version(type, STORE_ATTR) == 0) {
unsigned int tp_version = 0;
DescriptorClassification kind = analyze_descriptor_store(type, name, &descr, &tp_version);
if (tp_version == 0) {
goto fail;
}
assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN);
switch(kind) {
case OVERRIDING:
SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR);
@ -1309,8 +1368,8 @@ _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *na
assert(dmem->type == Py_T_OBJECT_EX || dmem->type == _Py_T_OBJECT);
assert(offset > 0);
cache->index = (uint16_t)offset;
write_u32(cache->version, type->tp_version_tag);
instr->op.code = STORE_ATTR_SLOT;
write_u32(cache->version, tp_version);
specialize(instr, STORE_ATTR_SLOT);
goto success;
}
case DUNDER_CLASS:
@ -1337,22 +1396,19 @@ _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *na
SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE);
goto fail;
case ABSENT:
if (specialize_dict_access(owner, instr, type, kind, name, STORE_ATTR,
STORE_ATTR_INSTANCE_VALUE, STORE_ATTR_WITH_HINT))
{
if (specialize_dict_access(owner, instr, type, kind, name, tp_version,
STORE_ATTR, STORE_ATTR_INSTANCE_VALUE,
STORE_ATTR_WITH_HINT)) {
goto success;
}
}
fail:
STAT_INC(STORE_ATTR, failure);
assert(!PyErr_Occurred());
instr->op.code = STORE_ATTR;
cache->counter = adaptive_counter_backoff(cache->counter);
Py_XDECREF(descr);
unspecialize(instr);
return;
success:
STAT_INC(STORE_ATTR, success);
assert(!PyErr_Occurred());
cache->counter = adaptive_counter_cooldown();
Py_XDECREF(descr);
return;
}
#ifndef Py_GIL_DISABLED
@ -1421,7 +1477,8 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr,
}
PyObject *descr = NULL;
DescriptorClassification kind = 0;
kind = analyze_descriptor(cls, name, &descr, 0);
kind = analyze_descriptor_load(cls, name, &descr);
Py_XDECREF(descr); // turn strong ref into a borrowed ref
if (type_get_version(cls, LOAD_ATTR) == 0) {
return -1;
}
@ -1714,7 +1771,6 @@ function_get_version(PyObject *o, int opcode)
}
return version;
}
#endif // Py_GIL_DISABLED
/* Returning 0 indicates a failure. */
static uint32_t
@ -1727,6 +1783,7 @@ type_get_version(PyTypeObject *t, int opcode)
}
return version;
}
#endif // Py_GIL_DISABLED
void
_Py_Specialize_BinarySubscr(