Skip to content

gh-112075: Add try-incref functions from nogil branch for use in dict thread safety #114512

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
DinoV merged 3 commits into from
Jan 25, 2024
Merged

gh-112075: Add try-incref functions from nogil branch for use in dict thread safety #114512

Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
39c251e
Bring in a subset of biased reference counting:
DinoV Jan 24, 2024
a923bc9
Review feedback
DinoV Jan 24, 2024
87e9465
More renames
DinoV Jan 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Review feedback
  • Loading branch information
@DinoV
DinoV committed Jan 25, 2024
commit a923bc912ac75348267a3493a5eef63bf2ba1083
56 changes: 21 additions & 35 deletions Include/internal/pycore_object.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -385,7 +385,7 @@ static inline void _PyObject_GC_UNTRACK(
* where the reference count modification requires an atomic operation. This
* allows call sites to specialize for the immortal/local case.
*/
static inline Py_ALWAYS_INLINE int
static inline int
_Py_TryIncrefFast(PyObject *op) {
uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local);
local += 1;
Expand All @@ -394,6 +394,7 @@ _Py_TryIncrefFast(PyObject *op) {
return 1;
}
if (_Py_IsOwnedByCurrentThread(op)) {
_Py_INCREF_STAT_INC();
_Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local);
Comment thread
DinoV marked this conversation as resolved.
#ifdef Py_REF_DEBUG
_Py_IncRefTotal(_PyInterpreterState_GET());
Expand All @@ -403,12 +404,11 @@ _Py_TryIncrefFast(PyObject *op) {
return 0;
}

static inline Py_ALWAYS_INLINE int
static inline int
_Py_TryIncRefShared(PyObject *op)
{
Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared);
for (;;) {
Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared);

// If the shared refcount is zero and the object is either merged
// or may not have weak references, then we cannot incref it.
if (shared == 0 || shared == _Py_REF_MERGED) {
Expand All @@ -422,6 +422,7 @@ _Py_TryIncRefShared(PyObject *op)
#ifdef Py_REF_DEBUG
Comment thread
DinoV marked this conversation as resolved.
_Py_IncRefTotal(_PyInterpreterState_GET());
#endif
_Py_INCREF_STAT_INC();
return 1;
}
}
Expand All @@ -447,10 +448,9 @@ _Py_TryAcquireObject(PyObject **src, PyObject *op)
/* Loads and increfs an object from ptr, which may contain a NULL value.
Safe with concurrent (atomic) updates to ptr.
NOTE: The writer must set maybe-weakref on the stored object! */
static inline Py_ALWAYS_INLINE PyObject *
_Py_XFetchRef(PyObject **ptr)
static inline PyObject *
_Py_XGetRef(PyObject **ptr)
{
#ifdef Py_NOGIL
for (;;) {
PyObject *value = _Py_atomic_load_ptr(ptr);
if (value == NULL) {
Expand All @@ -460,14 +460,11 @@ _Py_XFetchRef(PyObject **ptr)
return value;
}
}
#else
return Py_XNewRef(*ptr);
#endif
}

/* Attempts to loads and increfs an object from ptr. Returns NULL
on failure, which may be due to a NULL value or a concurrent update. */
static inline Py_ALWAYS_INLINE PyObject *
static inline PyObject *
_Py_TryXFetchRef(PyObject **ptr)
Comment thread
DinoV marked this conversation as resolved.
Outdated
{
PyObject *value = _Py_atomic_load_ptr(ptr);
Expand All @@ -485,32 +482,21 @@ _Py_TryXFetchRef(PyObject **ptr)
static inline PyObject *
_Py_NewRefWithLock(PyObject *op)
{
_Py_INCREF_STAT_INC();
uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local);
local += 1;
if (local == 0) {
if (_Py_TryIncrefFast(op)) {
return op;
}

#ifdef Py_REF_DEBUG
_Py_IncRefTotal(_PyInterpreterState_GET());
#endif
if (_Py_IsOwnedByCurrentThread(op)) {
_Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local);
}
else {
for (;;) {
Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared);
Py_ssize_t new_shared = shared + (1 << _Py_REF_SHARED_SHIFT);
if ((shared & _Py_REF_SHARED_FLAG_MASK) == 0) {
new_shared |= _Py_REF_MAYBE_WEAKREF;
}
if (_Py_atomic_compare_exchange_ssize(
&op->ob_ref_shared,
&shared,
new_shared)) {
return op;
}
_Py_INCREF_STAT_INC();
for (;;) {
Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared);
Py_ssize_t new_shared = shared + (1 << _Py_REF_SHARED_SHIFT);
if ((shared & _Py_REF_SHARED_FLAG_MASK) == 0) {
new_shared |= _Py_REF_MAYBE_WEAKREF;
}
if (_Py_atomic_compare_exchange_ssize(
&op->ob_ref_shared,
&shared,
new_shared)) {
return op;
}
}
return op;
Comment thread
DinoV marked this conversation as resolved.
Outdated
Expand Down