Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 14 additions & 2 deletions Include/internal/pycore_qsbr.h
Original file line number Diff line number Diff line change
Expand Up @@ -69,10 +69,22 @@ struct _qsbr_thread_state {
struct _qsbr_thread_state *freelist_next;
};

// Size used to pad and align the per-thread QSBR states (and the end of
// _PyThreadStateImpl) to avoid false sharing. Apple Silicon has 128-byte cache
// lines; with 64-byte padding the `seq` fields of two threads share a line and
// the stores in _Py_qsbr_quiescent_state() contend on every eval loop
// back-edge. Use 128 bytes for all AArch64 targets: on cores with 64-byte
// lines this only costs 64 bytes per thread.
#if defined(__aarch64__) || defined(_M_ARM64)
# define _Py_QSBR_CACHE_LINE_SIZE 128
#else
# define _Py_QSBR_CACHE_LINE_SIZE 64
#endif

// Padding to avoid false sharing
struct _qsbr_pad {
struct _qsbr_thread_state qsbr;
char __padding[64 - sizeof(struct _qsbr_thread_state)];
char __padding[_Py_QSBR_CACHE_LINE_SIZE - sizeof(struct _qsbr_thread_state)];
};

// Per-interpreter state
Expand All @@ -83,7 +95,7 @@ struct _qsbr_shared {
// Minimum observed read sequence of all QSBR thread states
uint64_t rd_seq;

// Array of QSBR thread states (aligned to 64 bytes).
// Array of QSBR thread states (aligned to _Py_QSBR_CACHE_LINE_SIZE).
struct _qsbr_pad *array;
void *array_raw; // raw allocation pointer (for free)
Py_ssize_t size;
Expand Down
2 changes: 1 addition & 1 deletion Include/internal/pycore_tstate.h
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ typedef struct _PyThreadStateImpl {
#ifdef Py_GIL_DISABLED
// gh-144438: Add padding to ensure that the fields above don't share a
// cache line with other allocations.
char __padding[64];
char __padding[_Py_QSBR_CACHE_LINE_SIZE];
#endif
} _PyThreadStateImpl;

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Fix false sharing between the per-thread QSBR states in the
:term:`free-threaded build` on AArch64 CPUs with 128-byte cache lines, such
as Apple Silicon. CPU-bound threads that share no objects now scale like
processes there.
4 changes: 2 additions & 2 deletions Python/qsbr.c
Original file line number Diff line number Diff line change
Expand Up @@ -85,10 +85,10 @@ grow_thread_array(struct _qsbr_shared *shared)
new_size = MIN_ARRAY_SIZE;
}

// Overallocate by 63 bytes so we can align to a 64-byte boundary.
// Overallocate so we can align to a cache line boundary.
// This avoids potential false sharing between the first entry and other
// allocations.
size_t alignment = 64;
size_t alignment = _Py_QSBR_CACHE_LINE_SIZE;
size_t alloc_size = (size_t)new_size * sizeof(struct _qsbr_pad) + alignment - 1;
void *raw = PyMem_RawCalloc(1, alloc_size);
if (raw == NULL) {
Expand Down
Loading