Rooba
/
cpython
mirror of https://github.com/python/cpython.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

gh-115999: Specialize `STORE_ATTR` in free-threaded builds. (gh-127838) 

* Add `_PyDictKeys_StringLookupSplit` which does locking on dict keys and
  use in place of `_PyDictKeys_StringLookup`.

* Change `_PyObject_TryGetInstanceAttribute` to use that function
  in the case of split keys.

* Add `unicodekeys_lookup_split` helper which allows code sharing
  between `_Py_dict_lookup` and `_PyDictKeys_StringLookupSplit`.

* Fix locking for `STORE_ATTR_INSTANCE_VALUE`.  Create
  `_GUARD_TYPE_VERSION_AND_LOCK` uop so that object stays locked and
  `tp_version_tag` cannot change.

* Pass `tp_version_tag` to `specialize_dict_access()`, ensuring
  the version we store on the cache is the correct one (in case of
  it changing during the specalize analysis).

* Split `analyze_descriptor` into `analyze_descriptor_load` and
  `analyze_descriptor_store` since those don't share much logic.
  Add `descriptor_is_class` helper function.

* In `specialize_dict_access`, double check `_PyObject_GetManagedDict()`
  in case we race and dict was materialized before the lock.

* Avoid borrowed references in `_Py_Specialize_StoreAttr()`.

* Use `specialize()` and `unspecialize()` helpers.

* Add unit tests to ensure specializing happens as expected in FT builds.

* Add unit tests to attempt to trigger data races (useful for running under TSAN).

* Add `has_split_table` function to `_testinternalcapi`.
This commit is contained in:
Neil Schemenauer 2024-12-19 10:21:17 -08:00 committed by GitHub
parent d2f1d917e8
commit 1b15c89a17
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 716 additions and 297 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Include/internal/pycore_dict.h
View File

@ -114,6 +114,7 @@ extern Py_ssize_t _Py_dict_lookup_threadsafe_stackref(PyDictObject *mp, PyObject
extern Py_ssize_t _PyDict_LookupIndex(PyDictObject *, PyObject *);
extern Py_ssize_t _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyObject *key);
extern Py_ssize_t _PyDictKeys_StringLookupSplit(PyDictKeysObject* dictkeys, PyObject *key);
PyAPI_FUNC(PyObject *)_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *);
PyAPI_FUNC(void) _PyDict_LoadGlobalStackRef(PyDictObject *, PyDictObject *, PyObject *, _PyStackRef *);

4
Include/internal/pycore_opcode_metadata.h generated
View File

@ -2141,7 +2141,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = {
[SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_EXIT_FLAG },
[STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_EXIT_FLAG },
[STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG },
[STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG },
[STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
@ -2340,7 +2340,7 @@ _PyOpcode_macro_expansion[256] = {
[SET_FUNCTION_ATTRIBUTE] = { .nuops = 1, .uops = { { _SET_FUNCTION_ATTRIBUTE, 0, 0 } } },
[SET_UPDATE] = { .nuops = 1, .uops = { { _SET_UPDATE, 0, 0 } } },
[STORE_ATTR] = { .nuops = 1, .uops = { { _STORE_ATTR, 0, 0 } } },
[STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_NO_DICT, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } },
[STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION_AND_LOCK, 2, 1 }, { _GUARD_DORV_NO_DICT, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } },
[STORE_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 } } },
[STORE_ATTR_WITH_HINT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_WITH_HINT, 1, 3 } } },
[STORE_DEREF] = { .nuops = 1, .uops = { { _STORE_DEREF, 0, 0 } } },

201
Include/internal/pycore_uop_ids.h generated
View File

@ -139,15 +139,16 @@ extern "C" {
#define _GUARD_TOS_FLOAT 386
#define _GUARD_TOS_INT 387
#define _GUARD_TYPE_VERSION 388
#define _GUARD_TYPE_VERSION_AND_LOCK 389
#define _IMPORT_FROM IMPORT_FROM
#define _IMPORT_NAME IMPORT_NAME
#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 389
#define _INIT_CALL_PY_EXACT_ARGS 390
#define _INIT_CALL_PY_EXACT_ARGS_0 391
#define _INIT_CALL_PY_EXACT_ARGS_1 392
#define _INIT_CALL_PY_EXACT_ARGS_2 393
#define _INIT_CALL_PY_EXACT_ARGS_3 394
#define _INIT_CALL_PY_EXACT_ARGS_4 395
#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 390
#define _INIT_CALL_PY_EXACT_ARGS 391
#define _INIT_CALL_PY_EXACT_ARGS_0 392
#define _INIT_CALL_PY_EXACT_ARGS_1 393
#define _INIT_CALL_PY_EXACT_ARGS_2 394
#define _INIT_CALL_PY_EXACT_ARGS_3 395
#define _INIT_CALL_PY_EXACT_ARGS_4 396
#define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX
#define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW
#define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER
@ -160,143 +161,143 @@ extern "C" {
#define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE
#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE
#define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE
#define _INTERNAL_INCREMENT_OPT_COUNTER 396
#define _IS_NONE 397
#define _INTERNAL_INCREMENT_OPT_COUNTER 397
#define _IS_NONE 398
#define _IS_OP IS_OP
#define _ITER_CHECK_LIST 398
#define _ITER_CHECK_RANGE 399
#define _ITER_CHECK_TUPLE 400
#define _ITER_JUMP_LIST 401
#define _ITER_JUMP_RANGE 402
#define _ITER_JUMP_TUPLE 403
#define _ITER_NEXT_LIST 404
#define _ITER_NEXT_RANGE 405
#define _ITER_NEXT_TUPLE 406
#define _JUMP_TO_TOP 407
#define _ITER_CHECK_LIST 399
#define _ITER_CHECK_RANGE 400
#define _ITER_CHECK_TUPLE 401
#define _ITER_JUMP_LIST 402
#define _ITER_JUMP_RANGE 403
#define _ITER_JUMP_TUPLE 404
#define _ITER_NEXT_LIST 405
#define _ITER_NEXT_RANGE 406
#define _ITER_NEXT_TUPLE 407
#define _JUMP_TO_TOP 408
#define _LIST_APPEND LIST_APPEND
#define _LIST_EXTEND LIST_EXTEND
#define _LOAD_ATTR 408
#define _LOAD_ATTR_CLASS 409
#define _LOAD_ATTR_CLASS_0 410
#define _LOAD_ATTR_CLASS_1 411
#define _LOAD_ATTR 409
#define _LOAD_ATTR_CLASS 410
#define _LOAD_ATTR_CLASS_0 411
#define _LOAD_ATTR_CLASS_1 412
#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN
#define _LOAD_ATTR_INSTANCE_VALUE 412
#define _LOAD_ATTR_INSTANCE_VALUE_0 413
#define _LOAD_ATTR_INSTANCE_VALUE_1 414
#define _LOAD_ATTR_METHOD_LAZY_DICT 415
#define _LOAD_ATTR_METHOD_NO_DICT 416
#define _LOAD_ATTR_METHOD_WITH_VALUES 417
#define _LOAD_ATTR_MODULE 418
#define _LOAD_ATTR_MODULE_FROM_KEYS 419
#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 420
#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 421
#define _LOAD_ATTR_PROPERTY_FRAME 422
#define _LOAD_ATTR_SLOT 423
#define _LOAD_ATTR_SLOT_0 424
#define _LOAD_ATTR_SLOT_1 425
#define _LOAD_ATTR_WITH_HINT 426
#define _LOAD_ATTR_INSTANCE_VALUE 413
#define _LOAD_ATTR_INSTANCE_VALUE_0 414
#define _LOAD_ATTR_INSTANCE_VALUE_1 415
#define _LOAD_ATTR_METHOD_LAZY_DICT 416
#define _LOAD_ATTR_METHOD_NO_DICT 417
#define _LOAD_ATTR_METHOD_WITH_VALUES 418
#define _LOAD_ATTR_MODULE 419
#define _LOAD_ATTR_MODULE_FROM_KEYS 420
#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 421
#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 422
#define _LOAD_ATTR_PROPERTY_FRAME 423
#define _LOAD_ATTR_SLOT 424
#define _LOAD_ATTR_SLOT_0 425
#define _LOAD_ATTR_SLOT_1 426
#define _LOAD_ATTR_WITH_HINT 427
#define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS
#define _LOAD_BYTECODE 427
#define _LOAD_BYTECODE 428
#define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT
#define _LOAD_CONST LOAD_CONST
#define _LOAD_CONST_IMMORTAL LOAD_CONST_IMMORTAL
#define _LOAD_CONST_INLINE 428
#define _LOAD_CONST_INLINE_BORROW 429
#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 430
#define _LOAD_CONST_INLINE_WITH_NULL 431
#define _LOAD_CONST_INLINE 429
#define _LOAD_CONST_INLINE_BORROW 430
#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 431
#define _LOAD_CONST_INLINE_WITH_NULL 432
#define _LOAD_DEREF LOAD_DEREF
#define _LOAD_FAST 432
#define _LOAD_FAST_0 433
#define _LOAD_FAST_1 434
#define _LOAD_FAST_2 435
#define _LOAD_FAST_3 436
#define _LOAD_FAST_4 437
#define _LOAD_FAST_5 438
#define _LOAD_FAST_6 439
#define _LOAD_FAST_7 440
#define _LOAD_FAST 433
#define _LOAD_FAST_0 434
#define _LOAD_FAST_1 435
#define _LOAD_FAST_2 436
#define _LOAD_FAST_3 437
#define _LOAD_FAST_4 438
#define _LOAD_FAST_5 439
#define _LOAD_FAST_6 440
#define _LOAD_FAST_7 441
#define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR
#define _LOAD_FAST_CHECK LOAD_FAST_CHECK
#define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST
#define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF
#define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS
#define _LOAD_GLOBAL 441
#define _LOAD_GLOBAL_BUILTINS 442
#define _LOAD_GLOBAL_BUILTINS_FROM_KEYS 443
#define _LOAD_GLOBAL_MODULE 444
#define _LOAD_GLOBAL_MODULE_FROM_KEYS 445
#define _LOAD_GLOBAL 442
#define _LOAD_GLOBAL_BUILTINS 443
#define _LOAD_GLOBAL_BUILTINS_FROM_KEYS 444
#define _LOAD_GLOBAL_MODULE 445
#define _LOAD_GLOBAL_MODULE_FROM_KEYS 446
#define _LOAD_LOCALS LOAD_LOCALS
#define _LOAD_NAME LOAD_NAME
#define _LOAD_SMALL_INT 446
#define _LOAD_SMALL_INT_0 447
#define _LOAD_SMALL_INT_1 448
#define _LOAD_SMALL_INT_2 449
#define _LOAD_SMALL_INT_3 450
#define _LOAD_SMALL_INT 447
#define _LOAD_SMALL_INT_0 448
#define _LOAD_SMALL_INT_1 449
#define _LOAD_SMALL_INT_2 450
#define _LOAD_SMALL_INT_3 451
#define _LOAD_SPECIAL LOAD_SPECIAL
#define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR
#define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD
#define _MAKE_CALLARGS_A_TUPLE 451
#define _MAKE_CALLARGS_A_TUPLE 452
#define _MAKE_CELL MAKE_CELL
#define _MAKE_FUNCTION MAKE_FUNCTION
#define _MAKE_WARM 452
#define _MAKE_WARM 453
#define _MAP_ADD MAP_ADD
#define _MATCH_CLASS MATCH_CLASS
#define _MATCH_KEYS MATCH_KEYS
#define _MATCH_MAPPING MATCH_MAPPING
#define _MATCH_SEQUENCE MATCH_SEQUENCE
#define _MAYBE_EXPAND_METHOD 453
#define _MAYBE_EXPAND_METHOD_KW 454
#define _MONITOR_CALL 455
#define _MONITOR_JUMP_BACKWARD 456
#define _MONITOR_RESUME 457
#define _MAYBE_EXPAND_METHOD 454
#define _MAYBE_EXPAND_METHOD_KW 455
#define _MONITOR_CALL 456
#define _MONITOR_JUMP_BACKWARD 457
#define _MONITOR_RESUME 458
#define _NOP NOP
#define _POP_EXCEPT POP_EXCEPT
#define _POP_JUMP_IF_FALSE 458
#define _POP_JUMP_IF_TRUE 459
#define _POP_JUMP_IF_FALSE 459
#define _POP_JUMP_IF_TRUE 460
#define _POP_TOP POP_TOP
#define _POP_TOP_LOAD_CONST_INLINE_BORROW 460
#define _POP_TOP_LOAD_CONST_INLINE_BORROW 461
#define _PUSH_EXC_INFO PUSH_EXC_INFO
#define _PUSH_FRAME 461
#define _PUSH_FRAME 462
#define _PUSH_NULL PUSH_NULL
#define _PY_FRAME_GENERAL 462
#define _PY_FRAME_KW 463
#define _QUICKEN_RESUME 464
#define _REPLACE_WITH_TRUE 465
#define _PY_FRAME_GENERAL 463
#define _PY_FRAME_KW 464
#define _QUICKEN_RESUME 465
#define _REPLACE_WITH_TRUE 466
#define _RESUME_CHECK RESUME_CHECK
#define _RETURN_GENERATOR RETURN_GENERATOR
#define _RETURN_VALUE RETURN_VALUE
#define _SAVE_RETURN_OFFSET 466
#define _SEND 467
#define _SEND_GEN_FRAME 468
#define _SAVE_RETURN_OFFSET 467
#define _SEND 468
#define _SEND_GEN_FRAME 469
#define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS
#define _SET_ADD SET_ADD
#define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE
#define _SET_UPDATE SET_UPDATE
#define _START_EXECUTOR 469
#define _STORE_ATTR 470
#define _STORE_ATTR_INSTANCE_VALUE 471
#define _STORE_ATTR_SLOT 472
#define _STORE_ATTR_WITH_HINT 473
#define _START_EXECUTOR 470
#define _STORE_ATTR 471
#define _STORE_ATTR_INSTANCE_VALUE 472
#define _STORE_ATTR_SLOT 473
#define _STORE_ATTR_WITH_HINT 474
#define _STORE_DEREF STORE_DEREF
#define _STORE_FAST 474
#define _STORE_FAST_0 475
#define _STORE_FAST_1 476
#define _STORE_FAST_2 477
#define _STORE_FAST_3 478
#define _STORE_FAST_4 479
#define _STORE_FAST_5 480
#define _STORE_FAST_6 481
#define _STORE_FAST_7 482
#define _STORE_FAST 475
#define _STORE_FAST_0 476
#define _STORE_FAST_1 477
#define _STORE_FAST_2 478
#define _STORE_FAST_3 479
#define _STORE_FAST_4 480
#define _STORE_FAST_5 481
#define _STORE_FAST_6 482
#define _STORE_FAST_7 483
#define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST
#define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST
#define _STORE_GLOBAL STORE_GLOBAL
#define _STORE_NAME STORE_NAME
#define _STORE_SLICE 483
#define _STORE_SUBSCR 484
#define _STORE_SLICE 484
#define _STORE_SUBSCR 485
#define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT
#define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT
#define _SWAP SWAP
#define _TIER2_RESUME_CHECK 485
#define _TO_BOOL 486
#define _TIER2_RESUME_CHECK 486
#define _TO_BOOL 487
#define _TO_BOOL_BOOL TO_BOOL_BOOL
#define _TO_BOOL_INT TO_BOOL_INT
#define _TO_BOOL_LIST TO_BOOL_LIST
@ -306,13 +307,13 @@ extern "C" {
#define _UNARY_NEGATIVE UNARY_NEGATIVE
#define _UNARY_NOT UNARY_NOT
#define _UNPACK_EX UNPACK_EX
#define _UNPACK_SEQUENCE 487
#define _UNPACK_SEQUENCE 488
#define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST
#define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE
#define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE
#define _WITH_EXCEPT_START WITH_EXCEPT_START
#define _YIELD_VALUE YIELD_VALUE
#define MAX_UOP_ID 487
#define MAX_UOP_ID 488
#ifdef __cplusplus
}

6
Include/internal/pycore_uop_metadata.h generated
View File

@ -148,6 +148,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
[_LOAD_SUPER_ATTR_METHOD] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_LOAD_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_GUARD_TYPE_VERSION] = HAS_EXIT_FLAG,
[_GUARD_TYPE_VERSION_AND_LOCK] = HAS_EXIT_FLAG,
[_CHECK_MANAGED_OBJECT_HAS_VALUES] = HAS_DEOPT_FLAG,
[_LOAD_ATTR_INSTANCE_VALUE_0] = HAS_DEOPT_FLAG,
[_LOAD_ATTR_INSTANCE_VALUE_1] = HAS_DEOPT_FLAG,
@ -167,7 +168,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
[_GUARD_DORV_NO_DICT] = HAS_EXIT_FLAG,
[_STORE_ATTR_INSTANCE_VALUE] = 0,
[_STORE_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG,
[_STORE_ATTR_SLOT] = 0,
[_STORE_ATTR_SLOT] = HAS_DEOPT_FLAG,
[_COMPARE_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_COMPARE_OP_FLOAT] = HAS_ARG_FLAG,
[_COMPARE_OP_INT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG,
@ -428,6 +429,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {
[_GUARD_TOS_FLOAT] = "_GUARD_TOS_FLOAT",
[_GUARD_TOS_INT] = "_GUARD_TOS_INT",
[_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION",
[_GUARD_TYPE_VERSION_AND_LOCK] = "_GUARD_TYPE_VERSION_AND_LOCK",
[_IMPORT_FROM] = "_IMPORT_FROM",
[_IMPORT_NAME] = "_IMPORT_NAME",
[_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS",
@ -839,6 +841,8 @@ int _PyUop_num_popped(int opcode, int oparg)
return 1;
case _GUARD_TYPE_VERSION:
return 0;
case _GUARD_TYPE_VERSION_AND_LOCK:
return 0;
case _CHECK_MANAGED_OBJECT_HAS_VALUES:
return 0;
case _LOAD_ATTR_INSTANCE_VALUE_0:

141
Lib/test/test_free_threading/test_races.py
View File

@ -4,6 +4,7 @@
import threading
import time
import unittest
import _testinternalcapi
from test.support import threading_helper
@ -129,6 +130,146 @@ def mutate():
# with the cell binding being changed).
do_race(access, mutate)
def test_racing_to_bool(self):
seq = [1]
class C:
def __bool__(self):
return False
def access():
if seq:
return 1
else:
return 2
def mutate():
nonlocal seq
seq = [1]
time.sleep(0)
seq = C()
time.sleep(0)
do_race(access, mutate)
def test_racing_store_attr_slot(self):
class C:
__slots__ = ['x', '__dict__']
c = C()
def set_slot():
for i in range(10):
c.x = i
time.sleep(0)
def change_type():
def set_x(self, x):
pass
def get_x(self):
pass
C.x = property(get_x, set_x)
time.sleep(0)
del C.x
time.sleep(0)
do_race(set_slot, change_type)
def set_getattribute():
C.__getattribute__ = lambda self, x: x
time.sleep(0)
del C.__getattribute__
time.sleep(0)
do_race(set_slot, set_getattribute)
def test_racing_store_attr_instance_value(self):
class C:
pass
c = C()
def set_value():
for i in range(100):
c.x = i
set_value()
def read():
x = c.x
def mutate():
# Adding a property for 'x' should unspecialize it.
C.x = property(lambda self: None, lambda self, x: None)
time.sleep(0)
del C.x
time.sleep(0)
do_race(read, mutate)
def test_racing_store_attr_with_hint(self):
class C:
pass
c = C()
for i in range(29):
setattr(c, f"_{i}", None)
def set_value():
for i in range(100):
c.x = i
set_value()
def read():
x = c.x
def mutate():
# Adding a property for 'x' should unspecialize it.
C.x = property(lambda self: None, lambda self, x: None)
time.sleep(0)
del C.x
time.sleep(0)
do_race(read, mutate)
def make_shared_key_dict(self):
class C:
pass
a = C()
a.x = 1
return a.__dict__
def test_racing_store_attr_dict(self):
"""Test STORE_ATTR with various dictionary types."""
class C:
pass
c = C()
def set_value():
for i in range(20):
c.x = i
def mutate():
nonlocal c
c.x = 1
self.assertTrue(_testinternalcapi.has_inline_values(c))
for i in range(30):
setattr(c, f"_{i}", None)
self.assertFalse(_testinternalcapi.has_inline_values(c.__dict__))
c.__dict__ = self.make_shared_key_dict()
self.assertTrue(_testinternalcapi.has_split_table(c.__dict__))
c.__dict__[1] = None
self.assertFalse(_testinternalcapi.has_split_table(c.__dict__))
c = C()
do_race(set_value, mutate)
if __name__ == "__main__":
unittest.main()

66
Lib/test/test_opcache.py
View File

@ -1383,6 +1383,72 @@ def send_yield_from():
self.assert_specialized(send_yield_from, "SEND_GEN")
self.assert_no_opcode(send_yield_from, "SEND")
@cpython_only
@requires_specialization_ft
def test_store_attr_slot(self):
class C:
__slots__ = ['x']
def set_slot():
c = C()
for i in range(100):
c.x = i
set_slot()
self.assert_specialized(set_slot, "STORE_ATTR_SLOT")
self.assert_no_opcode(set_slot, "STORE_ATTR")
# Adding a property for 'x' should unspecialize it.
C.x = property(lambda self: None, lambda self, x: None)
set_slot()
self.assert_no_opcode(set_slot, "STORE_ATTR_SLOT")
@cpython_only
@requires_specialization_ft
def test_store_attr_instance_value(self):
class C:
pass
def set_value():
c = C()
for i in range(100):
c.x = i
set_value()
self.assert_specialized(set_value, "STORE_ATTR_INSTANCE_VALUE")
self.assert_no_opcode(set_value, "STORE_ATTR")
# Adding a property for 'x' should unspecialize it.
C.x = property(lambda self: None, lambda self, x: None)
set_value()
self.assert_no_opcode(set_value, "STORE_ATTR_INSTANCE_VALUE")
@cpython_only
@requires_specialization_ft
def test_store_attr_with_hint(self):
class C:
pass
c = C()
for i in range(29):
setattr(c, f"_{i}", None)
def set_value():
for i in range(100):
c.x = i
set_value()
self.assert_specialized(set_value, "STORE_ATTR_WITH_HINT")
self.assert_no_opcode(set_value, "STORE_ATTR")
# Adding a property for 'x' should unspecialize it.
C.x = property(lambda self: None, lambda self, x: None)
set_value()
self.assert_no_opcode(set_value, "STORE_ATTR_WITH_HINT")
@cpython_only
@requires_specialization_ft
def test_to_bool(self):

9
Modules/_testinternalcapi.c
View File

@ -1989,6 +1989,14 @@ has_inline_values(PyObject *self, PyObject *obj)
Py_RETURN_FALSE;
}
static PyObject *
has_split_table(PyObject *self, PyObject *obj)
{
if (PyDict_Check(obj) && _PyDict_HasSplitTable((PyDictObject *)obj)) {
Py_RETURN_TRUE;
}
Py_RETURN_FALSE;
}
// Circumvents standard version assignment machinery - use with caution and only on
// short-lived heap types
@ -2139,6 +2147,7 @@ static PyMethodDef module_functions[] = {
{"get_rare_event_counters", get_rare_event_counters, METH_NOARGS},
{"reset_rare_event_counters", reset_rare_event_counters, METH_NOARGS},
{"has_inline_values", has_inline_values, METH_O},
{"has_split_table", has_split_table, METH_O},
{"type_assign_specific_version_unsafe", type_assign_specific_version_unsafe, METH_VARARGS,
PyDoc_STR("forcefully assign type->tp_version_tag")},

66
Objects/dictobject.c
View File

@ -1129,6 +1129,35 @@ dictkeys_generic_lookup(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, P
return do_lookup(mp, dk, key, hash, compare_generic);
}
#ifdef Py_GIL_DISABLED
static Py_ssize_t
unicodekeys_lookup_unicode_threadsafe(PyDictKeysObject* dk, PyObject *key,
Py_hash_t hash);
#endif
static Py_ssize_t
unicodekeys_lookup_split(PyDictKeysObject* dk, PyObject *key, Py_hash_t hash)
{
Py_ssize_t ix;
assert(dk->dk_kind == DICT_KEYS_SPLIT);
assert(PyUnicode_CheckExact(key));
#ifdef Py_GIL_DISABLED
// A split dictionaries keys can be mutated by other dictionaries
// but if we have a unicode key we can avoid locking the shared
// keys.
ix = unicodekeys_lookup_unicode_threadsafe(dk, key, hash);
if (ix == DKIX_KEY_CHANGED) {
LOCK_KEYS(dk);
ix = unicodekeys_lookup_unicode(dk, key, hash);
UNLOCK_KEYS(dk);
}
#else
ix = unicodekeys_lookup_unicode(dk, key, hash);
#endif
return ix;
}
/* Lookup a string in a (all unicode) dict keys.
* Returns DKIX_ERROR if key is not a string,
* or if the dict keys is not all strings.
@ -1153,13 +1182,24 @@ _PyDictKeys_StringLookup(PyDictKeysObject* dk, PyObject *key)
return unicodekeys_lookup_unicode(dk, key, hash);
}
#ifdef Py_GIL_DISABLED
static Py_ssize_t
unicodekeys_lookup_unicode_threadsafe(PyDictKeysObject* dk, PyObject *key,
Py_hash_t hash);
#endif
/* Like _PyDictKeys_StringLookup() but only works on split keys. Note
* that in free-threaded builds this locks the keys object as required.
*/
Py_ssize_t
_PyDictKeys_StringLookupSplit(PyDictKeysObject* dk, PyObject *key)
{
assert(dk->dk_kind == DICT_KEYS_SPLIT);
assert(PyUnicode_CheckExact(key));
Py_hash_t hash = unicode_get_hash(key);
if (hash == -1) {
hash = PyUnicode_Type.tp_hash(key);
if (hash == -1) {
PyErr_Clear();
return DKIX_ERROR;
}
}
return unicodekeys_lookup_split(dk, key, hash);
}
/*
The basic lookup function used by all operations.
@ -1192,15 +1232,7 @@ _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **valu
if (PyUnicode_CheckExact(key)) {
#ifdef Py_GIL_DISABLED
if (kind == DICT_KEYS_SPLIT) {
// A split dictionaries keys can be mutated by other
// dictionaries but if we have a unicode key we can avoid
// locking the shared keys.
ix = unicodekeys_lookup_unicode_threadsafe(dk, key, hash);
if (ix == DKIX_KEY_CHANGED) {
LOCK_KEYS(dk);
ix = unicodekeys_lookup_unicode(dk, key, hash);
UNLOCK_KEYS(dk);
}
ix = unicodekeys_lookup_split(dk, key, hash);
}
else {
ix = unicodekeys_lookup_unicode(dk, key, hash);
@ -6967,7 +6999,7 @@ _PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name, PyObject **attr
PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj));
assert(keys != NULL);
Py_ssize_t ix = _PyDictKeys_StringLookup(keys, name);
Py_ssize_t ix = _PyDictKeys_StringLookupSplit(keys, name);
if (ix == DKIX_EMPTY) {
*attr = NULL;
return true;

65
Python/bytecodes.c
View File

@ -1467,7 +1467,7 @@ dummy_func(
};
specializing op(_SPECIALIZE_STORE_ATTR, (counter/1, owner -- owner)) {
#if ENABLE_SPECIALIZATION
#if ENABLE_SPECIALIZATION_FT
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
next_instr = this_instr;
@ -1476,7 +1476,7 @@ dummy_func(
}
OPCODE_DEFERRED_INC(STORE_ATTR);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
#endif /* ENABLE_SPECIALIZATION_FT */
}
op(_STORE_ATTR, (v, owner --)) {
@ -2129,7 +2129,18 @@ dummy_func(
op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) {
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
EXIT_IF(tp->tp_version_tag != type_version);
EXIT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version);
}
op(_GUARD_TYPE_VERSION_AND_LOCK, (type_version/2, owner -- owner)) {
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(type_version != 0);
EXIT_IF(!LOCK_OBJECT(owner_o));
PyTypeObject *tp = Py_TYPE(owner_o);
if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
UNLOCK_OBJECT(owner_o);
EXIT_IF(true);
}
}
op(_CHECK_MANAGED_OBJECT_HAS_VALUES, (owner -- owner)) {
@ -2336,8 +2347,11 @@ dummy_func(
assert(Py_TYPE(owner_o)->tp_dictoffset < 0);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
EXIT_IF(_PyObject_GetManagedDict(owner_o));
EXIT_IF(_PyObject_InlineValues(owner_o)->valid == 0);
if (_PyObject_GetManagedDict(owner_o) ||
!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) {
UNLOCK_OBJECT(owner_o);
EXIT_IF(true);
}
}
op(_STORE_ATTR_INSTANCE_VALUE, (offset/1, value, owner --)) {
@ -2347,21 +2361,20 @@ dummy_func(
assert(_PyObject_GetManagedDict(owner_o) == NULL);
PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset);
PyObject *old_value = *value_ptr;
*value_ptr = PyStackRef_AsPyObjectSteal(value);
FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value));
if (old_value == NULL) {
PyDictValues *values = _PyObject_InlineValues(owner_o);
Py_ssize_t index = value_ptr - values->values;
_PyDictValues_AddToInsertionOrder(values, index);
}
else {
Py_DECREF(old_value);
}
UNLOCK_OBJECT(owner_o);
Py_XDECREF(old_value);
PyStackRef_CLOSE(owner);
}
macro(STORE_ATTR_INSTANCE_VALUE) =
unused/1 +
_GUARD_TYPE_VERSION +
_GUARD_TYPE_VERSION_AND_LOCK +
_GUARD_DORV_NO_DICT +
_STORE_ATTR_INSTANCE_VALUE;
@ -2370,16 +2383,34 @@ dummy_func(
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
DEOPT_IF(dict == NULL);
DEOPT_IF(!LOCK_OBJECT(dict));
#ifdef Py_GIL_DISABLED
if (dict != _PyObject_GetManagedDict(owner_o)) {
UNLOCK_OBJECT(dict);
DEOPT_IF(true);
}
#endif
assert(PyDict_CheckExact((PyObject *)dict));
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries);
DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys));
if (hint >= (size_t)dict->ma_keys->dk_nentries ||
!DK_IS_UNICODE(dict->ma_keys)) {
UNLOCK_OBJECT(dict);
DEOPT_IF(true);
}
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
DEOPT_IF(ep->me_key != name);
if (ep->me_key != name) {
UNLOCK_OBJECT(dict);
DEOPT_IF(true);
}
PyObject *old_value = ep->me_value;
DEOPT_IF(old_value == NULL);
if (old_value == NULL) {
UNLOCK_OBJECT(dict);
DEOPT_IF(true);
}
_PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value));
ep->me_value = PyStackRef_AsPyObjectSteal(value);
FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value));
UNLOCK_OBJECT(dict);
// old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault,
// when dict only holds the strong reference to value in ep->me_value.
Py_XDECREF(old_value);
@ -2395,10 +2426,12 @@ dummy_func(
op(_STORE_ATTR_SLOT, (index/1, value, owner --)) {
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
DEOPT_IF(!LOCK_OBJECT(owner_o));
char *addr = (char *)owner_o + index;
STAT_INC(STORE_ATTR, hit);
PyObject *old_value = *(PyObject **)addr;
*(PyObject **)addr = PyStackRef_AsPyObjectSteal(value);
FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value));
UNLOCK_OBJECT(owner_o);
Py_XDECREF(old_value);
PyStackRef_CLOSE(owner);
}

75
Python/executor_cases.c.h generated
View File

@ -2574,13 +2574,34 @@
uint32_t type_version = (uint32_t)CURRENT_OPERAND0();
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
if (tp->tp_version_tag != type_version) {
if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _GUARD_TYPE_VERSION_AND_LOCK: {
_PyStackRef owner;
owner = stack_pointer[-1];
uint32_t type_version = (uint32_t)CURRENT_OPERAND0();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(type_version != 0);
if (!LOCK_OBJECT(owner_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
PyTypeObject *tp = Py_TYPE(owner_o);
if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
UNLOCK_OBJECT(owner_o);
if (true) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
break;
}
case _CHECK_MANAGED_OBJECT_HAS_VALUES: {
_PyStackRef owner;
owner = stack_pointer[-1];
@ -2910,13 +2931,13 @@
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_dictoffset < 0);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
if (_PyObject_GetManagedDict(owner_o)) {
if (_PyObject_GetManagedDict(owner_o) ||
!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) {
UNLOCK_OBJECT(owner_o);
if (true) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (_PyObject_InlineValues(owner_o)->valid == 0) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
@ -2932,15 +2953,14 @@
assert(_PyObject_GetManagedDict(owner_o) == NULL);
PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset);
PyObject *old_value = *value_ptr;
*value_ptr = PyStackRef_AsPyObjectSteal(value);
FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value));
if (old_value == NULL) {
PyDictValues *values = _PyObject_InlineValues(owner_o);
Py_ssize_t index = value_ptr - values->values;
_PyDictValues_AddToInsertionOrder(values, index);
}
else {
Py_DECREF(old_value);
}
UNLOCK_OBJECT(owner_o);
Py_XDECREF(old_value);
PyStackRef_CLOSE(owner);
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
@ -2961,30 +2981,50 @@
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
assert(PyDict_CheckExact((PyObject *)dict));
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
if (hint >= (size_t)dict->ma_keys->dk_nentries) {
if (!LOCK_OBJECT(dict)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (!DK_IS_UNICODE(dict->ma_keys)) {
#ifdef Py_GIL_DISABLED
if (dict != _PyObject_GetManagedDict(owner_o)) {
UNLOCK_OBJECT(dict);
if (true) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
#endif
assert(PyDict_CheckExact((PyObject *)dict));
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
if (hint >= (size_t)dict->ma_keys->dk_nentries ||
!DK_IS_UNICODE(dict->ma_keys)) {
UNLOCK_OBJECT(dict);
if (true) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
if (ep->me_key != name) {
UNLOCK_OBJECT(dict);
if (true) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
PyObject *old_value = ep->me_value;
if (old_value == NULL) {
UNLOCK_OBJECT(dict);
if (true) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value));
stack_pointer = _PyFrame_GetStackPointer(frame);
ep->me_value = PyStackRef_AsPyObjectSteal(value);
FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value));
UNLOCK_OBJECT(dict);
// old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault,
// when dict only holds the strong reference to value in ep->me_value.
Py_XDECREF(old_value);
@ -3002,10 +3042,15 @@
value = stack_pointer[-2];
uint16_t index = (uint16_t)CURRENT_OPERAND0();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
if (!LOCK_OBJECT(owner_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
char *addr = (char *)owner_o + index;
STAT_INC(STORE_ATTR, hit);
PyObject *old_value = *(PyObject **)addr;
*(PyObject **)addr = PyStackRef_AsPyObjectSteal(value);
FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value));
UNLOCK_OBJECT(owner_o);
Py_XDECREF(old_value);
PyStackRef_CLOSE(owner);
stack_pointer += -2;

86
Python/generated_cases.c.h generated
View File

@ -5319,7 +5319,7 @@
uint32_t type_version = read_u32(&this_instr[4].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
}
// _LOAD_ATTR_CLASS
{
@ -5388,7 +5388,7 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
}
// _CHECK_MANAGED_OBJECT_HAS_VALUES
{
@ -5433,7 +5433,7 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
}
// _CHECK_ATTR_METHOD_LAZY_DICT
{
@ -5476,7 +5476,7 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
}
/* Skip 2 cache entries */
// _LOAD_ATTR_METHOD_NO_DICT
@ -5512,7 +5512,7 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
}
// _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT
{
@ -5611,7 +5611,7 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
}
/* Skip 2 cache entries */
// _LOAD_ATTR_NONDESCRIPTOR_NO_DICT
@ -5642,7 +5642,7 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
}
// _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT
{
@ -5688,7 +5688,7 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
}
/* Skip 2 cache entries */
// _LOAD_ATTR_PROPERTY_FRAME
@ -5750,7 +5750,7 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
}
// _LOAD_ATTR_SLOT
{
@ -5787,7 +5787,7 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
}
// _CHECK_ATTR_WITH_HINT
{
@ -7314,7 +7314,7 @@
owner = stack_pointer[-1];
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
#if ENABLE_SPECIALIZATION_FT
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
next_instr = this_instr;
@ -7325,7 +7325,7 @@
}
OPCODE_DEFERRED_INC(STORE_ATTR);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
#endif /* ENABLE_SPECIALIZATION_FT */
}
/* Skip 3 cache entries */
// _STORE_ATTR
@ -7353,21 +7353,29 @@
_PyStackRef owner;
_PyStackRef value;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
// _GUARD_TYPE_VERSION_AND_LOCK
{
owner = stack_pointer[-1];
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR);
PyTypeObject *tp = Py_TYPE(owner_o);
if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
UNLOCK_OBJECT(owner_o);
DEOPT_IF(true, STORE_ATTR);
}
}
// _GUARD_DORV_NO_DICT
{
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_dictoffset < 0);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
DEOPT_IF(_PyObject_GetManagedDict(owner_o), STORE_ATTR);
DEOPT_IF(_PyObject_InlineValues(owner_o)->valid == 0, STORE_ATTR);
if (_PyObject_GetManagedDict(owner_o) ||
!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) {
UNLOCK_OBJECT(owner_o);
DEOPT_IF(true, STORE_ATTR);
}
}
// _STORE_ATTR_INSTANCE_VALUE
{
@ -7378,15 +7386,14 @@
assert(_PyObject_GetManagedDict(owner_o) == NULL);
PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset);
PyObject *old_value = *value_ptr;
*value_ptr = PyStackRef_AsPyObjectSteal(value);
FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value));
if (old_value == NULL) {
PyDictValues *values = _PyObject_InlineValues(owner_o);
Py_ssize_t index = value_ptr - values->values;
_PyDictValues_AddToInsertionOrder(values, index);
}
else {
Py_DECREF(old_value);
}
UNLOCK_OBJECT(owner_o);
Py_XDECREF(old_value);
PyStackRef_CLOSE(owner);
}
stack_pointer += -2;
@ -7408,17 +7415,19 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR);
}
// _STORE_ATTR_SLOT
{
value = stack_pointer[-2];
uint16_t index = read_u16(&this_instr[4].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR);
char *addr = (char *)owner_o + index;
STAT_INC(STORE_ATTR, hit);
PyObject *old_value = *(PyObject **)addr;
*(PyObject **)addr = PyStackRef_AsPyObjectSteal(value);
FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value));
UNLOCK_OBJECT(owner_o);
Py_XDECREF(old_value);
PyStackRef_CLOSE(owner);
}
@ -7441,7 +7450,7 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR);
}
// _STORE_ATTR_WITH_HINT
{
@ -7451,18 +7460,35 @@
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
DEOPT_IF(dict == NULL, STORE_ATTR);
DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR);
#ifdef Py_GIL_DISABLED
if (dict != _PyObject_GetManagedDict(owner_o)) {
UNLOCK_OBJECT(dict);
DEOPT_IF(true, STORE_ATTR);
}
#endif
assert(PyDict_CheckExact((PyObject *)dict));
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR);
DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys), STORE_ATTR);
if (hint >= (size_t)dict->ma_keys->dk_nentries ||
!DK_IS_UNICODE(dict->ma_keys)) {
UNLOCK_OBJECT(dict);
DEOPT_IF(true, STORE_ATTR);
}
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
DEOPT_IF(ep->me_key != name, STORE_ATTR);
if (ep->me_key != name) {
UNLOCK_OBJECT(dict);
DEOPT_IF(true, STORE_ATTR);
}
PyObject *old_value = ep->me_value;
DEOPT_IF(old_value == NULL, STORE_ATTR);
if (old_value == NULL) {
UNLOCK_OBJECT(dict);
DEOPT_IF(true, STORE_ATTR);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value));
stack_pointer = _PyFrame_GetStackPointer(frame);
ep->me_value = PyStackRef_AsPyObjectSteal(value);
FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value));
UNLOCK_OBJECT(dict);
// old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault,
// when dict only holds the strong reference to value in ep->me_value.
Py_XDECREF(old_value);
@ -7815,7 +7841,7 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, TO_BOOL);
DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL);
}
// _REPLACE_WITH_TRUE
{

4
Python/optimizer_cases.c.h generated
View File

@ -1121,6 +1121,10 @@
break;
}
case _GUARD_TYPE_VERSION_AND_LOCK: {
break;
}
case _CHECK_MANAGED_OBJECT_HAS_VALUES: {
break;
}

193
Python/specialize.c
View File

@ -741,8 +741,8 @@ static int function_kind(PyCodeObject *code);
#ifndef Py_GIL_DISABLED
static bool function_check_args(PyObject *o, int expected_argcount, int opcode);
static uint32_t function_get_version(PyObject *o, int opcode);
#endif
static uint32_t type_get_version(PyTypeObject *t, int opcode);
#endif
static int
specialize_module_load_attr_lock_held(PyDictObject *dict, _Py_CODEUNIT *instr, PyObject *name)
@ -881,17 +881,17 @@ classify_descriptor(PyObject *descriptor, bool has_getattr)
return NON_DESCRIPTOR;
}
static DescriptorClassification
analyze_descriptor(PyTypeObject *type, PyObject *name, PyObject **descr, int store)
static bool
descriptor_is_class(PyObject *descriptor, PyObject *name)
{
return ((PyUnicode_CompareWithASCIIString(name, "__class__") == 0) &&
(descriptor == _PyType_Lookup(&PyBaseObject_Type, name)));
}
#ifndef Py_GIL_DISABLED
static DescriptorClassification
analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr) {
bool has_getattr = false;
if (store) {
if (type->tp_setattro != PyObject_GenericSetAttr) {
*descr = NULL;
return GETSET_OVERRIDDEN;
}
}
else {
getattrofunc getattro_slot = type->tp_getattro;
if (getattro_slot == PyObject_GenericGetAttr) {
/* Normal attribute lookup; */
@ -901,12 +901,13 @@ analyze_descriptor(PyTypeObject *type, PyObject *name, PyObject **descr, int sto
getattro_slot == _Py_slot_tp_getattro) {
/* One or both of __getattribute__ or __getattr__ may have been
overridden See typeobject.c for why these functions are special. */
PyObject *getattribute = _PyType_Lookup(type,
&_Py_ID(__getattribute__));
PyObject *getattribute = _PyType_LookupRef(type, &_Py_ID(__getattribute__));
PyInterpreterState *interp = _PyInterpreterState_GET();
bool has_custom_getattribute = getattribute != NULL &&
getattribute != interp->callable_cache.object__getattribute__;
has_getattr = _PyType_Lookup(type, &_Py_ID(__getattr__)) != NULL;
PyObject *getattr = _PyType_LookupRef(type, &_Py_ID(__getattr__));
has_getattr = getattr != NULL;
Py_XDECREF(getattr);
if (has_custom_getattribute) {
if (getattro_slot == _Py_slot_tp_getattro &&
!has_getattr &&
@ -916,6 +917,7 @@ analyze_descriptor(PyTypeObject *type, PyObject *name, PyObject **descr, int sto
}
/* Potentially both __getattr__ and __getattribute__ are set.
Too complicated */
Py_DECREF(getattribute);
*descr = NULL;
return GETSET_OVERRIDDEN;
}
@ -926,44 +928,46 @@ analyze_descriptor(PyTypeObject *type, PyObject *name, PyObject **descr, int sto
raised. This means some specializations, e.g. specializing
for property() isn't safe.
*/
Py_XDECREF(getattribute);
}
else {
*descr = NULL;
return GETSET_OVERRIDDEN;
}
}
PyObject *descriptor = _PyType_Lookup(type, name);
PyObject *descriptor = _PyType_LookupRef(type, name);
*descr = descriptor;
if (PyUnicode_CompareWithASCIIString(name, "__class__") == 0) {
if (descriptor == _PyType_Lookup(&PyBaseObject_Type, name)) {
if (descriptor_is_class(descriptor, name)) {
return DUNDER_CLASS;
}
}
return classify_descriptor(descriptor, has_getattr);
}
#endif //!Py_GIL_DISABLED
static DescriptorClassification
analyze_descriptor_store(PyTypeObject *type, PyObject *name, PyObject **descr, unsigned int *tp_version)
{
if (type->tp_setattro != PyObject_GenericSetAttr) {
*descr = NULL;
return GETSET_OVERRIDDEN;
}
PyObject *descriptor = _PyType_LookupRefAndVersion(type, name, tp_version);
*descr = descriptor;
if (descriptor_is_class(descriptor, name)) {
return DUNDER_CLASS;
}
return classify_descriptor(descriptor, false);
}
static int
specialize_dict_access(
specialize_dict_access_inline(
PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type,
DescriptorClassification kind, PyObject *name,
int base_op, int values_op, int hint_op)
DescriptorClassification kind, PyObject *name, unsigned int tp_version,
int base_op, int values_op)
{
assert(kind == NON_OVERRIDING || kind == NON_DESCRIPTOR || kind == ABSENT ||
kind == BUILTIN_CLASSMETHOD || kind == PYTHON_CLASSMETHOD ||
kind == METHOD);
// No descriptor, or non overriding.
if ((type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_MANAGED_DICT);
return 0;
}
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES &&
_PyObject_InlineValues(owner)->valid &&
!(base_op == STORE_ATTR && _PyObject_GetManagedDict(owner) != NULL))
{
PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys;
assert(PyUnicode_CheckExact(name));
Py_ssize_t index = _PyDictKeys_StringLookup(keys, name);
Py_ssize_t index = _PyDictKeys_StringLookupSplit(keys, name);
assert (index != DKIX_ERROR);
if (index == DKIX_EMPTY) {
SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_IN_KEYS);
@ -976,23 +980,25 @@ specialize_dict_access(
SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE);
return 0;
}
write_u32(cache->version, type->tp_version_tag);
cache->index = (uint16_t)offset;
write_u32(cache->version, tp_version);
specialize(instr, values_op);
}
else {
PyDictObject *dict = _PyObject_GetManagedDict(owner);
if (dict == NULL || !PyDict_CheckExact(dict)) {
SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT);
return 0;
}
return 1;
}
static int
specialize_dict_access_hint(
PyDictObject *dict, _Py_CODEUNIT *instr, PyTypeObject *type,
DescriptorClassification kind, PyObject *name, unsigned int tp_version,
int base_op, int hint_op)
{
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
// We found an instance with a __dict__.
if (dict->ma_values) {
if (_PyDict_HasSplitTable(dict)) {
SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_SPLIT_DICT);
return 0;
}
Py_ssize_t index =
_PyDict_LookupIndex(dict, name);
Py_ssize_t index = _PyDict_LookupIndex(dict, name);
if (index != (uint16_t)index) {
SPECIALIZATION_FAIL(base_op,
index == DKIX_EMPTY ?
@ -1001,12 +1007,62 @@ specialize_dict_access(
return 0;
}
cache->index = (uint16_t)index;
write_u32(cache->version, type->tp_version_tag);
write_u32(cache->version, tp_version);
specialize(instr, hint_op);
}
return 1;
}
static int
specialize_dict_access(
PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type,
DescriptorClassification kind, PyObject *name, unsigned int tp_version,
int base_op, int values_op, int hint_op)
{
assert(kind == NON_OVERRIDING || kind == NON_DESCRIPTOR || kind == ABSENT ||
kind == BUILTIN_CLASSMETHOD || kind == PYTHON_CLASSMETHOD ||
kind == METHOD);
// No descriptor, or non overriding.
if ((type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_MANAGED_DICT);
return 0;
}
if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES &&
FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner)->valid) &&
!(base_op == STORE_ATTR && _PyObject_GetManagedDict(owner) != NULL))
{
int res;
Py_BEGIN_CRITICAL_SECTION(owner);
PyDictObject *dict = _PyObject_GetManagedDict(owner);
if (dict == NULL) {
// managed dict, not materialized, inline values valid
res = specialize_dict_access_inline(owner, instr, type, kind, name,
tp_version, base_op, values_op);
}
else {
// lost race and dict was created, fail specialization
SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OTHER);
res = 0;
}
Py_END_CRITICAL_SECTION();
return res;
}
else {
PyDictObject *dict = _PyObject_GetManagedDict(owner);
if (dict == NULL || !PyDict_CheckExact(dict)) {
SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT);
return 0;
}
int res;
Py_BEGIN_CRITICAL_SECTION(dict);
// materialized managed dict
res = specialize_dict_access_hint(dict, instr, type, kind, name,
tp_version, base_op, hint_op);
Py_END_CRITICAL_SECTION();
return res;
}
}
#ifndef Py_GIL_DISABLED
static int specialize_attr_loadclassattr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name,
PyObject* descr, DescriptorClassification kind, bool is_method);
@ -1050,7 +1106,8 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na
PyTypeObject *type = Py_TYPE(owner);
bool shadow = instance_has_key(owner, name);
PyObject *descr = NULL;
DescriptorClassification kind = analyze_descriptor(type, name, &descr, 0);
DescriptorClassification kind = analyze_descriptor_load(type, name, &descr);
Py_XDECREF(descr); // turn strong ref into a borrowed ref
assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN);
if (type_get_version(type, LOAD_ATTR) == 0) {
return -1;
@ -1204,8 +1261,8 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na
}
Py_UNREACHABLE();
try_instance:
if (specialize_dict_access(owner, instr, type, kind, name, LOAD_ATTR,
LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT))
if (specialize_dict_access(owner, instr, type, kind, name, type->tp_version_tag,
LOAD_ATTR, LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT))
{
return 0;
}
@ -1259,8 +1316,9 @@ _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *na
{
PyObject *owner = PyStackRef_AsPyObjectBorrow(owner_st);
assert(ENABLE_SPECIALIZATION);
assert(ENABLE_SPECIALIZATION_FT);
assert(_PyOpcode_Caches[STORE_ATTR] == INLINE_CACHE_ENTRIES_STORE_ATTR);
PyObject *descr = NULL;
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
PyTypeObject *type = Py_TYPE(owner);
if (!_PyType_IsReady(type)) {
@ -1274,11 +1332,12 @@ _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *na
SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OVERRIDDEN);
goto fail;
}
PyObject *descr;
DescriptorClassification kind = analyze_descriptor(type, name, &descr, 1);
if (type_get_version(type, STORE_ATTR) == 0) {
unsigned int tp_version = 0;
DescriptorClassification kind = analyze_descriptor_store(type, name, &descr, &tp_version);
if (tp_version == 0) {
goto fail;
}
assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN);
switch(kind) {
case OVERRIDING:
SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR);
@ -1309,8 +1368,8 @@ _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *na
assert(dmem->type == Py_T_OBJECT_EX || dmem->type == _Py_T_OBJECT);
assert(offset > 0);
cache->index = (uint16_t)offset;
write_u32(cache->version, type->tp_version_tag);
instr->op.code = STORE_ATTR_SLOT;
write_u32(cache->version, tp_version);
specialize(instr, STORE_ATTR_SLOT);
goto success;
}
case DUNDER_CLASS:
@ -1337,22 +1396,19 @@ _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *na
SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE);
goto fail;
case ABSENT:
if (specialize_dict_access(owner, instr, type, kind, name, STORE_ATTR,
STORE_ATTR_INSTANCE_VALUE, STORE_ATTR_WITH_HINT))
{
if (specialize_dict_access(owner, instr, type, kind, name, tp_version,
STORE_ATTR, STORE_ATTR_INSTANCE_VALUE,
STORE_ATTR_WITH_HINT)) {
goto success;
}
}
fail:
STAT_INC(STORE_ATTR, failure);
assert(!PyErr_Occurred());
instr->op.code = STORE_ATTR;
cache->counter = adaptive_counter_backoff(cache->counter);
Py_XDECREF(descr);
unspecialize(instr);
return;
success:
STAT_INC(STORE_ATTR, success);
assert(!PyErr_Occurred());
cache->counter = adaptive_counter_cooldown();
Py_XDECREF(descr);
return;
}
#ifndef Py_GIL_DISABLED
@ -1421,7 +1477,8 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr,
}
PyObject *descr = NULL;
DescriptorClassification kind = 0;
kind = analyze_descriptor(cls, name, &descr, 0);
kind = analyze_descriptor_load(cls, name, &descr);
Py_XDECREF(descr); // turn strong ref into a borrowed ref
if (type_get_version(cls, LOAD_ATTR) == 0) {
return -1;
}
@ -1714,7 +1771,6 @@ function_get_version(PyObject *o, int opcode)
}
return version;
}
#endif // Py_GIL_DISABLED
/* Returning 0 indicates a failure. */
static uint32_t
@ -1727,6 +1783,7 @@ type_get_version(PyTypeObject *t, int opcode)
}
return version;
}
#endif // Py_GIL_DISABLED
void
_Py_Specialize_BinarySubscr(