diff --git a/Include/internal/pycore_cell.h b/Include/internal/pycore_cell.h index 27f67d57b2fb79..5a12a2f14571af 100644 --- a/Include/internal/pycore_cell.h +++ b/Include/internal/pycore_cell.h @@ -2,6 +2,7 @@ #define Py_INTERNAL_CELL_H #include "pycore_critical_section.h" +#include "pycore_object.h" #ifdef __cplusplus extern "C" { @@ -19,7 +20,7 @@ PyCell_SwapTakeRef(PyCellObject *cell, PyObject *value) PyObject *old_value; Py_BEGIN_CRITICAL_SECTION(cell); old_value = cell->ob_ref; - cell->ob_ref = value; + FT_ATOMIC_STORE_PTR_RELEASE(cell->ob_ref, value); Py_END_CRITICAL_SECTION(); return old_value; } @@ -42,6 +43,28 @@ PyCell_GetRef(PyCellObject *cell) return res; } +static inline +_PyStackRef _PyCell_GetStackRef(PyCellObject *cell) +{ + PyObject *value; +#ifdef Py_GIL_DISABLED + value = _Py_atomic_load_ptr(&cell->ob_ref); + if (value != NULL) { + if (_Py_IsImmortal(value) || _PyObject_HasDeferredRefcount(value)) { + return (_PyStackRef){ .bits = (uintptr_t)value | Py_TAG_DEFERRED }; + } + if (_Py_TryIncrefFast(value)) { + return _PyStackRef_FromPyObjectSteal(value); + } + } +#endif + value = PyCell_GetRef(cell); + if (value == NULL) { + return PyStackRef_NULL; + } + return PyStackRef_FromPyObjectSteal(value); +} + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 0af13b1bcda20b..cc6389dc6a4f2a 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -16,7 +16,6 @@ extern "C" { #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_uniqueid.h" // _PyType_IncrefSlow - #define _Py_IMMORTAL_REFCNT_LOOSE ((_Py_IMMORTAL_REFCNT >> 1) + 1) // This value is added to `ob_ref_shared` for objects that use deferred diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 3344ede5e92c07..b5fede10758f2e 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1154,7 +1154,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_COMMON_CONSTANT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG }, - [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 07606135d7a356..6171af241e845d 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -123,7 +123,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_MAKE_CELL] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, [_DELETE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_LOAD_FROM_DICT_OR_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_DEREF] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG, [_COPY_FREE_VARS] = HAS_ARG_FLAG, [_BUILD_STRING] = HAS_ARG_FLAG | HAS_ERROR_FLAG, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c712c772201e10..82ed7b2c099347 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1649,12 +1649,11 @@ dummy_func( inst(LOAD_DEREF, ( -- value)) { PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - PyObject *value_o = PyCell_GetRef(cell); - if (value_o == NULL) { + value = _PyCell_GetStackRef(cell); + if (PyStackRef_IsNull(value)) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); ERROR_IF(true, error); } - value = PyStackRef_FromPyObjectSteal(value_o); } inst(STORE_DEREF, (v --)) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index fdfec66b73c730..9f36420133b648 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1824,12 +1824,11 @@ _PyStackRef value; oparg = CURRENT_OPARG(); PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - PyObject *value_o = PyCell_GetRef(cell); - if (value_o == NULL) { + value = _PyCell_GetStackRef(cell); + if (PyStackRef_IsNull(value)) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); if (true) JUMP_TO_ERROR(); } - value = PyStackRef_FromPyObjectSteal(value_o); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 9de7554d4dfd55..753c3c94c88fad 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5502,12 +5502,11 @@ INSTRUCTION_STATS(LOAD_DEREF); _PyStackRef value; PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - PyObject *value_o = PyCell_GetRef(cell); - if (value_o == NULL) { + value = _PyCell_GetStackRef(cell); + if (PyStackRef_IsNull(value)) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); if (true) goto error; } - value = PyStackRef_FromPyObjectSteal(value_o); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index e5eb665ae212de..127d710bc58bdc 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -41,6 +41,8 @@ race_top:tstate_is_freed race_top:type_modified_unlocked race_top:write_thread_id race_top:PyThreadState_Clear +# see: https://github.com/python/cpython/issues/117721 +race_top:lock_PyThread_release_lock # Only seen on macOS, sample: https://gist.github.com/aisk/dda53f5d494a4556c35dde1fce03259c race_top:set_default_allocator_unlocked