Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 57 additions & 5 deletions Include/cpython/pyatomic.h
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,14 @@
// Operations are sequentially consistent unless they have a suffix indicating
// otherwise. If in doubt, prefer the sequentially consistent operations.
//
// The "_relaxed" suffix for load and store operations indicates the "relaxed"
// memory order. They don't provide synchronization, but (roughly speaking)
// guarantee somewhat sane behavior for races instead of undefined behavior.
// In practice, they correspond to "normal" hardware load and store
// The "_relaxed" suffix indicates the "relaxed" memory order. Relaxed
// operations don't provide synchronization, but (roughly speaking) guarantee
// somewhat sane behavior for races instead of undefined behavior. In practice,
// relaxed loads and stores correspond to "normal" hardware load and store
// instructions, so they are almost as inexpensive as plain loads and stores
// in C.
// in C. Relaxed read-modify-write operations, such as
// _Py_atomic_add_*_relaxed, are still atomic, but do not order surrounding
// memory accesses.
//
// Note that atomic read-modify-write operations like _Py_atomic_add_* return
// the previous value of the atomic variable, not the new value.
Expand Down Expand Up @@ -55,6 +57,12 @@
// obj += value
// return old_obj
//
// def _Py_atomic_add_relaxed(obj, value):
// # relaxed consistency
// old_obj = obj
// obj += value
// return old_obj
//
// def _Py_atomic_and(obj, value):
// # sequential consistency
// old_obj = obj
Expand Down Expand Up @@ -130,6 +138,50 @@ static inline Py_ssize_t
_Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value);


// --- _Py_atomic_add_relaxed ------------------------------------------------
// Atomically adds `value` to `obj` and returns the previous value
// (relaxed consistency, i.e., no ordering)

static inline int
_Py_atomic_add_int_relaxed(int *obj, int value);

static inline int8_t
_Py_atomic_add_int8_relaxed(int8_t *obj, int8_t value);

static inline int16_t
_Py_atomic_add_int16_relaxed(int16_t *obj, int16_t value);

static inline int32_t
_Py_atomic_add_int32_relaxed(int32_t *obj, int32_t value);

static inline int64_t
_Py_atomic_add_int64_relaxed(int64_t *obj, int64_t value);

static inline intptr_t
_Py_atomic_add_intptr_relaxed(intptr_t *obj, intptr_t value);

static inline unsigned int
_Py_atomic_add_uint_relaxed(unsigned int *obj, unsigned int value);

static inline uint8_t
_Py_atomic_add_uint8_relaxed(uint8_t *obj, uint8_t value);

static inline uint16_t
_Py_atomic_add_uint16_relaxed(uint16_t *obj, uint16_t value);

static inline uint32_t
_Py_atomic_add_uint32_relaxed(uint32_t *obj, uint32_t value);

static inline uint64_t
_Py_atomic_add_uint64_relaxed(uint64_t *obj, uint64_t value);

static inline uintptr_t
_Py_atomic_add_uintptr_relaxed(uintptr_t *obj, uintptr_t value);

static inline Py_ssize_t
_Py_atomic_add_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value);


// --- _Py_atomic_compare_exchange -------------------------------------------
// Performs an atomic compare-and-exchange.
//
Expand Down
55 changes: 55 additions & 0 deletions Include/cpython/pyatomic_gcc.h
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,61 @@ _Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value)
{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); }


// --- _Py_atomic_add_relaxed ------------------------------------------------

static inline int
_Py_atomic_add_int_relaxed(int *obj, int value)
{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); }

static inline int8_t
_Py_atomic_add_int8_relaxed(int8_t *obj, int8_t value)
{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); }

static inline int16_t
_Py_atomic_add_int16_relaxed(int16_t *obj, int16_t value)
{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); }

static inline int32_t
_Py_atomic_add_int32_relaxed(int32_t *obj, int32_t value)
{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); }

static inline int64_t
_Py_atomic_add_int64_relaxed(int64_t *obj, int64_t value)
{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); }

static inline intptr_t
_Py_atomic_add_intptr_relaxed(intptr_t *obj, intptr_t value)
{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); }

static inline unsigned int
_Py_atomic_add_uint_relaxed(unsigned int *obj, unsigned int value)
{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); }

static inline uint8_t
_Py_atomic_add_uint8_relaxed(uint8_t *obj, uint8_t value)
{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); }

static inline uint16_t
_Py_atomic_add_uint16_relaxed(uint16_t *obj, uint16_t value)
{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); }

static inline uint32_t
_Py_atomic_add_uint32_relaxed(uint32_t *obj, uint32_t value)
{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); }

static inline uint64_t
_Py_atomic_add_uint64_relaxed(uint64_t *obj, uint64_t value)
{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); }

static inline uintptr_t
_Py_atomic_add_uintptr_relaxed(uintptr_t *obj, uintptr_t value)
{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); }

static inline Py_ssize_t
_Py_atomic_add_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value)
{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); }


// --- _Py_atomic_compare_exchange -------------------------------------------

static inline int
Expand Down
126 changes: 126 additions & 0 deletions Include/cpython/pyatomic_msc.h
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,132 @@ _Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value)
}


// --- _Py_atomic_add_relaxed ------------------------------------------------

// The "_nf" (no fence) intrinsic variants provide relaxed memory order on
// ARM64. On x86 and x86-64 they do not exist; the plain interlocked
// intrinsics are used instead, which have stronger (sequentially consistent)
// ordering. That is a conforming implementation of relaxed memory order;
// x86 simply has no cheaper atomic read-modify-write.

static inline int8_t
_Py_atomic_add_int8_relaxed(int8_t *obj, int8_t value)
{
_Py_atomic_ASSERT_ARG_TYPE(char);
#if defined(_M_ARM64)
return (int8_t)_InterlockedExchangeAdd8_nf((volatile char *)obj, (char)value);
#else
return (int8_t)_InterlockedExchangeAdd8((volatile char *)obj, (char)value);
#endif
}

static inline int16_t
_Py_atomic_add_int16_relaxed(int16_t *obj, int16_t value)
{
_Py_atomic_ASSERT_ARG_TYPE(short);
#if defined(_M_ARM64)
return (int16_t)_InterlockedExchangeAdd16_nf((volatile short *)obj, (short)value);
#else
return (int16_t)_InterlockedExchangeAdd16((volatile short *)obj, (short)value);
#endif
}

static inline int32_t
_Py_atomic_add_int32_relaxed(int32_t *obj, int32_t value)
{
_Py_atomic_ASSERT_ARG_TYPE(long);
#if defined(_M_ARM64)
return (int32_t)_InterlockedExchangeAdd_nf((volatile long *)obj, (long)value);
#else
return (int32_t)_InterlockedExchangeAdd((volatile long *)obj, (long)value);
#endif
}

static inline int64_t
_Py_atomic_add_int64_relaxed(int64_t *obj, int64_t value)
{
#if defined(_M_ARM64)
_Py_atomic_ASSERT_ARG_TYPE(__int64);
return (int64_t)_InterlockedExchangeAdd64_nf((volatile __int64 *)obj, (__int64)value);
#elif defined(_M_X64)
_Py_atomic_ASSERT_ARG_TYPE(__int64);
return (int64_t)_InterlockedExchangeAdd64((volatile __int64 *)obj, (__int64)value);
#else
int64_t old_value = _Py_atomic_load_int64_relaxed(obj);
for (;;) {
int64_t new_value = old_value + value;
if (_Py_atomic_compare_exchange_int64(obj, &old_value, new_value)) {
return old_value;
}
}
#endif
}

static inline uint8_t
_Py_atomic_add_uint8_relaxed(uint8_t *obj, uint8_t value)
{
return (uint8_t)_Py_atomic_add_int8_relaxed((int8_t *)obj, (int8_t)value);
}

static inline uint16_t
_Py_atomic_add_uint16_relaxed(uint16_t *obj, uint16_t value)
{
return (uint16_t)_Py_atomic_add_int16_relaxed((int16_t *)obj, (int16_t)value);
}

static inline uint32_t
_Py_atomic_add_uint32_relaxed(uint32_t *obj, uint32_t value)
{
return (uint32_t)_Py_atomic_add_int32_relaxed((int32_t *)obj, (int32_t)value);
}

static inline int
_Py_atomic_add_int_relaxed(int *obj, int value)
{
_Py_atomic_ASSERT_ARG_TYPE(int32_t);
return (int)_Py_atomic_add_int32_relaxed((int32_t *)obj, (int32_t)value);
}

static inline unsigned int
_Py_atomic_add_uint_relaxed(unsigned int *obj, unsigned int value)
{
_Py_atomic_ASSERT_ARG_TYPE(int32_t);
return (unsigned int)_Py_atomic_add_int32_relaxed((int32_t *)obj, (int32_t)value);
}

static inline uint64_t
_Py_atomic_add_uint64_relaxed(uint64_t *obj, uint64_t value)
{
return (uint64_t)_Py_atomic_add_int64_relaxed((int64_t *)obj, (int64_t)value);
}

static inline intptr_t
_Py_atomic_add_intptr_relaxed(intptr_t *obj, intptr_t value)
{
#if SIZEOF_VOID_P == 8
_Py_atomic_ASSERT_ARG_TYPE(int64_t);
return (intptr_t)_Py_atomic_add_int64_relaxed((int64_t *)obj, (int64_t)value);
#else
_Py_atomic_ASSERT_ARG_TYPE(int32_t);
return (intptr_t)_Py_atomic_add_int32_relaxed((int32_t *)obj, (int32_t)value);
#endif
}

static inline uintptr_t
_Py_atomic_add_uintptr_relaxed(uintptr_t *obj, uintptr_t value)
{
_Py_atomic_ASSERT_ARG_TYPE(intptr_t);
return (uintptr_t)_Py_atomic_add_intptr_relaxed((intptr_t *)obj, (intptr_t)value);
}

static inline Py_ssize_t
_Py_atomic_add_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value)
{
_Py_atomic_ASSERT_ARG_TYPE(intptr_t);
return (Py_ssize_t)_Py_atomic_add_intptr_relaxed((intptr_t *)obj, (intptr_t)value);
}


// --- _Py_atomic_compare_exchange -------------------------------------------

static inline int
Expand Down
107 changes: 107 additions & 0 deletions Include/cpython/pyatomic_std.h
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,113 @@ _Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value)
}


// --- _Py_atomic_add_relaxed ------------------------------------------------

static inline int
_Py_atomic_add_int_relaxed(int *obj, int value)
{
_Py_USING_STD;
return atomic_fetch_add_explicit((_Atomic(int)*)obj, value,
memory_order_relaxed);
}

static inline int8_t
_Py_atomic_add_int8_relaxed(int8_t *obj, int8_t value)
{
_Py_USING_STD;
return atomic_fetch_add_explicit((_Atomic(int8_t)*)obj, value,
memory_order_relaxed);
}

static inline int16_t
_Py_atomic_add_int16_relaxed(int16_t *obj, int16_t value)
{
_Py_USING_STD;
return atomic_fetch_add_explicit((_Atomic(int16_t)*)obj, value,
memory_order_relaxed);
}

static inline int32_t
_Py_atomic_add_int32_relaxed(int32_t *obj, int32_t value)
{
_Py_USING_STD;
return atomic_fetch_add_explicit((_Atomic(int32_t)*)obj, value,
memory_order_relaxed);
}

static inline int64_t
_Py_atomic_add_int64_relaxed(int64_t *obj, int64_t value)
{
_Py_USING_STD;
return atomic_fetch_add_explicit((_Atomic(int64_t)*)obj, value,
memory_order_relaxed);
}

static inline intptr_t
_Py_atomic_add_intptr_relaxed(intptr_t *obj, intptr_t value)
{
_Py_USING_STD;
return atomic_fetch_add_explicit((_Atomic(intptr_t)*)obj, value,
memory_order_relaxed);
}

static inline unsigned int
_Py_atomic_add_uint_relaxed(unsigned int *obj, unsigned int value)
{
_Py_USING_STD;
return atomic_fetch_add_explicit((_Atomic(unsigned int)*)obj, value,
memory_order_relaxed);
}

static inline uint8_t
_Py_atomic_add_uint8_relaxed(uint8_t *obj, uint8_t value)
{
_Py_USING_STD;
return atomic_fetch_add_explicit((_Atomic(uint8_t)*)obj, value,
memory_order_relaxed);
}

static inline uint16_t
_Py_atomic_add_uint16_relaxed(uint16_t *obj, uint16_t value)
{
_Py_USING_STD;
return atomic_fetch_add_explicit((_Atomic(uint16_t)*)obj, value,
memory_order_relaxed);
}

static inline uint32_t
_Py_atomic_add_uint32_relaxed(uint32_t *obj, uint32_t value)
{
_Py_USING_STD;
return atomic_fetch_add_explicit((_Atomic(uint32_t)*)obj, value,
memory_order_relaxed);
}

static inline uint64_t
_Py_atomic_add_uint64_relaxed(uint64_t *obj, uint64_t value)
{
_Py_USING_STD;
return atomic_fetch_add_explicit((_Atomic(uint64_t)*)obj, value,
memory_order_relaxed);
}

static inline uintptr_t
_Py_atomic_add_uintptr_relaxed(uintptr_t *obj, uintptr_t value)
{
_Py_USING_STD;
return atomic_fetch_add_explicit((_Atomic(uintptr_t)*)obj, value,
memory_order_relaxed);
}

static inline Py_ssize_t
_Py_atomic_add_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value)
{
_Py_USING_STD;
return atomic_fetch_add_explicit((_Atomic(Py_ssize_t)*)obj, value,
memory_order_relaxed);
}


// --- _Py_atomic_compare_exchange -------------------------------------------

static inline int
Expand Down
Loading
Loading