diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 7a783058c17..5314a70436b 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -463,6 +463,12 @@ _Py_atomic_load_ptr_acquire(const void *obj); static inline void _Py_atomic_store_ptr_release(void *obj, void *value); +static inline void +_Py_atomic_store_int_release(int *obj, int value); + +static inline int +_Py_atomic_load_int_acquire(const int *obj); + // --- _Py_atomic_fence ------------------------------------------------------ diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index f1a38c7b528..70f2b7e1b57 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -487,6 +487,14 @@ static inline void _Py_atomic_store_ptr_release(void *obj, void *value) { __atomic_store_n((void **)obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_int_release(int *obj, int value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + +static inline int +_Py_atomic_load_int_acquire(const int *obj) +{ return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } + // --- _Py_atomic_fence ------------------------------------------------------ diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 287ed43b571..601a0cf65af 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -912,6 +912,32 @@ _Py_atomic_store_ptr_release(void *obj, void *value) #endif } +static inline void +_Py_atomic_store_int_release(int *obj, int value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(int volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); + __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); +#else +# error "no implementation of _Py_atomic_store_int_release" +#endif +} + +static inline int +_Py_atomic_load_int_acquire(const int *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(int volatile *)obj; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); + return (int)__ldar32((unsigned __int32 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_int_acquire" +#endif +} + // --- _Py_atomic_fence ------------------------------------------------------ diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index bf74a90887c..a05bfaec47e 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -854,6 +854,23 @@ _Py_atomic_store_ptr_release(void *obj, void *value) memory_order_release); } +static inline void +_Py_atomic_store_int_release(int *obj, int value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(int)*)obj, value, + memory_order_release); +} + +static inline int +_Py_atomic_load_int_acquire(const int *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(int)*)obj, + memory_order_acquire); +} + + // --- _Py_atomic_fence ------------------------------------------------------ diff --git a/Include/internal/pycore_gil.h b/Include/internal/pycore_gil.h index daf1e73e782..19b0d23a685 100644 --- a/Include/internal/pycore_gil.h +++ b/Include/internal/pycore_gil.h @@ -8,7 +8,6 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_atomic.h" // _Py_atomic_int #include "pycore_condvar.h" // PyCOND_T #ifndef Py_HAVE_CONDVAR @@ -28,7 +27,7 @@ struct _gil_runtime_state { PyThreadState* last_holder; /* Whether the GIL is already taken (-1 if uninitialized). This is atomic because it can be read without any lock taken in ceval.c. */ - _Py_atomic_int locked; + int locked; /* Number of GIL switches since the beginning. */ unsigned long switch_number; /* This condition variable allows one or several threads to wait diff --git a/Modules/_testcapi/pyatomic.c b/Modules/_testcapi/pyatomic.c index 5aedf687705..4f72844535e 100644 --- a/Modules/_testcapi/pyatomic.c +++ b/Modules/_testcapi/pyatomic.c @@ -140,6 +140,21 @@ test_atomic_release_acquire(PyObject *self, PyObject *obj) { Py_RETURN_NONE; } +static PyObject * +test_atomic_load_store_int_release_acquire(PyObject *self, PyObject *obj) { \ + int x = 0; + int y = 1; + int z = 2; + assert(_Py_atomic_load_int_acquire(&x) == 0); + _Py_atomic_store_int_release(&x, y); + assert(x == y); + assert(_Py_atomic_load_int_acquire(&x) == y); + _Py_atomic_store_int_release(&x, z); + assert(x == z); + assert(_Py_atomic_load_int_acquire(&x) == z); + Py_RETURN_NONE; +} + // NOTE: all tests should start with "test_atomic_" to be included // in test_pyatomic.py @@ -162,6 +177,7 @@ static PyMethodDef test_methods[] = { FOR_BITWISE_TYPES(BIND_TEST_AND_OR) {"test_atomic_fences", test_atomic_fences, METH_NOARGS}, {"test_atomic_release_acquire", test_atomic_release_acquire, METH_NOARGS}, + {"test_atomic_load_store_int_release_acquire", test_atomic_load_store_int_release_acquire, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index bbb1e784dfa..97ef39e80aa 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -1,6 +1,6 @@ #include "Python.h" -#include "pycore_atomic.h" // _Py_atomic_int +#include "pycore_atomic.h" // _Py_ANNOTATE_RWLOCK_CREATE #include "pycore_ceval.h" // _PyEval_SignalReceived() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interp.h" // _Py_RunGC() @@ -120,9 +120,6 @@ UNSIGNAL_PENDING_CALLS(PyInterpreterState *interp) #include #include -#include "pycore_atomic.h" - - #include "condvar.h" #define MUTEX_INIT(mut) \ @@ -166,8 +163,7 @@ UNSIGNAL_PENDING_CALLS(PyInterpreterState *interp) static void _gil_initialize(struct _gil_runtime_state *gil) { - _Py_atomic_int uninitialized = {-1}; - gil->locked = uninitialized; + gil->locked = -1; gil->interval = DEFAULT_INTERVAL; } @@ -176,7 +172,7 @@ static int gil_created(struct _gil_runtime_state *gil) if (gil == NULL) { return 0; } - return (_Py_atomic_load_explicit(&gil->locked, _Py_memory_order_acquire) >= 0); + return (_Py_atomic_load_int_acquire(&gil->locked) >= 0); } static void create_gil(struct _gil_runtime_state *gil) @@ -191,7 +187,7 @@ static void create_gil(struct _gil_runtime_state *gil) #endif _Py_atomic_store_ptr_relaxed(&gil->last_holder, 0); _Py_ANNOTATE_RWLOCK_CREATE(&gil->locked); - _Py_atomic_store_explicit(&gil->locked, 0, _Py_memory_order_release); + _Py_atomic_store_int_release(&gil->locked, 0); } static void destroy_gil(struct _gil_runtime_state *gil) @@ -205,8 +201,7 @@ static void destroy_gil(struct _gil_runtime_state *gil) COND_FINI(gil->switch_cond); MUTEX_FINI(gil->switch_mutex); #endif - _Py_atomic_store_explicit(&gil->locked, -1, - _Py_memory_order_release); + _Py_atomic_store_int_release(&gil->locked, -1); _Py_ANNOTATE_RWLOCK_DESTROY(&gil->locked); } @@ -247,7 +242,7 @@ drop_gil(PyInterpreterState *interp, PyThreadState *tstate) MUTEX_LOCK(gil->mutex); _Py_ANNOTATE_RWLOCK_RELEASED(&gil->locked, /*is_write=*/1); - _Py_atomic_store_relaxed(&gil->locked, 0); + _Py_atomic_store_int_relaxed(&gil->locked, 0); COND_SIGNAL(gil->cond); MUTEX_UNLOCK(gil->mutex); @@ -313,12 +308,12 @@ take_gil(PyThreadState *tstate) MUTEX_LOCK(gil->mutex); - if (!_Py_atomic_load_relaxed(&gil->locked)) { + if (!_Py_atomic_load_int_relaxed(&gil->locked)) { goto _ready; } int drop_requested = 0; - while (_Py_atomic_load_relaxed(&gil->locked)) { + while (_Py_atomic_load_int_relaxed(&gil->locked)) { unsigned long saved_switchnum = gil->switch_number; unsigned long interval = (gil->interval >= 1 ? gil->interval : 1); @@ -328,7 +323,7 @@ take_gil(PyThreadState *tstate) /* If we timed out and no switch occurred in the meantime, it is time to ask the GIL-holding thread to drop it. */ if (timed_out && - _Py_atomic_load_relaxed(&gil->locked) && + _Py_atomic_load_int_relaxed(&gil->locked) && gil->switch_number == saved_switchnum) { if (_PyThreadState_MustExit(tstate)) { @@ -358,7 +353,7 @@ take_gil(PyThreadState *tstate) MUTEX_LOCK(gil->switch_mutex); #endif /* We now hold the GIL */ - _Py_atomic_store_relaxed(&gil->locked, 1); + _Py_atomic_store_int_relaxed(&gil->locked, 1); _Py_ANNOTATE_RWLOCK_ACQUIRED(&gil->locked, /*is_write=*/1); if (tstate != (PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder)) { @@ -437,7 +432,7 @@ current_thread_holds_gil(struct _gil_runtime_state *gil, PyThreadState *tstate) if (((PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder)) != tstate) { return 0; } - return _Py_atomic_load_relaxed(&gil->locked); + return _Py_atomic_load_int_relaxed(&gil->locked); } static void diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index 76a1f7763f2..7a6aef7ad18 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -1,5 +1,6 @@ #include "pycore_interp.h" // _PyInterpreterState.threads.stacksize #include "pycore_pythread.h" // _POSIX_SEMAPHORES +#include "pycore_atomic.h" // _Py_ANNOTATE_PURE_HAPPENS_BEFORE_MUTEX /* Posix threads interface */