Skip to content

Commit 4d3199e

Browse files
Davidlohr Buesoingomolnar
Davidlohr Bueso
authored and
ingomolnar
committed
locking: Remove ACCESS_ONCE() usage
With the new standardized functions, we can replace all ACCESS_ONCE() calls across relevant locking - this includes lockref and seqlock while at it. ACCESS_ONCE() does not work reliably on non-scalar types. For example gcc 4.6 and 4.7 might remove the volatile tag for such accesses during the SRA (scalar replacement of aggregates) step: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58145 Update the new calls regardless of if it is a scalar type, this is cleaner than having three alternatives. Signed-off-by: Davidlohr Bueso <dbueso@suse.de> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Link: http://lkml.kernel.org/r/1424662301.6539.18.camel@stgolabs.net Signed-off-by: Ingo Molnar <mingo@kernel.org>
1 parent 2ae7902 commit 4d3199e

6 files changed

Lines changed: 23 additions & 23 deletions

File tree

‎include/linux/seqlock.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ static inline unsigned __read_seqcount_begin(const seqcount_t *s)
108108
unsigned ret;
109109

110110
repeat:
111-
ret = ACCESS_ONCE(s->sequence);
111+
ret = READ_ONCE(s->sequence);
112112
if (unlikely(ret & 1)) {
113113
cpu_relax();
114114
goto repeat;
@@ -127,7 +127,7 @@ static inline unsigned __read_seqcount_begin(const seqcount_t *s)
127127
*/
128128
static inline unsigned raw_read_seqcount(const seqcount_t *s)
129129
{
130-
unsigned ret = ACCESS_ONCE(s->sequence);
130+
unsigned ret = READ_ONCE(s->sequence);
131131
smp_rmb();
132132
return ret;
133133
}
@@ -179,7 +179,7 @@ static inline unsigned read_seqcount_begin(const seqcount_t *s)
179179
*/
180180
static inline unsigned raw_seqcount_begin(const seqcount_t *s)
181181
{
182-
unsigned ret = ACCESS_ONCE(s->sequence);
182+
unsigned ret = READ_ONCE(s->sequence);
183183
smp_rmb();
184184
return ret & ~1;
185185
}

‎kernel/locking/mcs_spinlock.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ void mcs_spin_lock(struct mcs_spinlock **lock, struct mcs_spinlock *node)
7878
*/
7979
return;
8080
}
81-
ACCESS_ONCE(prev->next) = node;
81+
WRITE_ONCE(prev->next, node);
8282

8383
/* Wait until the lock holder passes the lock down. */
8484
arch_mcs_spin_lock_contended(&node->locked);
@@ -91,7 +91,7 @@ void mcs_spin_lock(struct mcs_spinlock **lock, struct mcs_spinlock *node)
9191
static inline
9292
void mcs_spin_unlock(struct mcs_spinlock **lock, struct mcs_spinlock *node)
9393
{
94-
struct mcs_spinlock *next = ACCESS_ONCE(node->next);
94+
struct mcs_spinlock *next = READ_ONCE(node->next);
9595

9696
if (likely(!next)) {
9797
/*
@@ -100,7 +100,7 @@ void mcs_spin_unlock(struct mcs_spinlock **lock, struct mcs_spinlock *node)
100100
if (likely(cmpxchg(lock, node, NULL) == node))
101101
return;
102102
/* Wait until the next pointer is set */
103-
while (!(next = ACCESS_ONCE(node->next)))
103+
while (!(next = READ_ONCE(node->next)))
104104
cpu_relax_lowlatency();
105105
}
106106

‎kernel/locking/mutex.c‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -266,7 +266,7 @@ static inline int mutex_can_spin_on_owner(struct mutex *lock)
266266
return 0;
267267

268268
rcu_read_lock();
269-
owner = ACCESS_ONCE(lock->owner);
269+
owner = READ_ONCE(lock->owner);
270270
if (owner)
271271
retval = owner->on_cpu;
272272
rcu_read_unlock();
@@ -340,15 +340,15 @@ static bool mutex_optimistic_spin(struct mutex *lock,
340340
* As such, when deadlock detection needs to be
341341
* performed the optimistic spinning cannot be done.
342342
*/
343-
if (ACCESS_ONCE(ww->ctx))
343+
if (READ_ONCE(ww->ctx))
344344
break;
345345
}
346346

347347
/*
348348
* If there's an owner, wait for it to either
349349
* release the lock or go to sleep.
350350
*/
351-
owner = ACCESS_ONCE(lock->owner);
351+
owner = READ_ONCE(lock->owner);
352352
if (owner && !mutex_spin_on_owner(lock, owner))
353353
break;
354354

@@ -487,7 +487,7 @@ static inline int __sched
487487
__ww_mutex_lock_check_stamp(struct mutex *lock, struct ww_acquire_ctx *ctx)
488488
{
489489
struct ww_mutex *ww = container_of(lock, struct ww_mutex, base);
490-
struct ww_acquire_ctx *hold_ctx = ACCESS_ONCE(ww->ctx);
490+
struct ww_acquire_ctx *hold_ctx = READ_ONCE(ww->ctx);
491491

492492
if (!hold_ctx)
493493
return 0;

‎kernel/locking/osq_lock.c‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ bool osq_lock(struct optimistic_spin_queue *lock)
9898

9999
prev = decode_cpu(old);
100100
node->prev = prev;
101-
ACCESS_ONCE(prev->next) = node;
101+
WRITE_ONCE(prev->next, node);
102102

103103
/*
104104
* Normally @prev is untouchable after the above store; because at that
@@ -109,7 +109,7 @@ bool osq_lock(struct optimistic_spin_queue *lock)
109109
* cmpxchg in an attempt to undo our queueing.
110110
*/
111111

112-
while (!ACCESS_ONCE(node->locked)) {
112+
while (!READ_ONCE(node->locked)) {
113113
/*
114114
* If we need to reschedule bail... so we can block.
115115
*/
@@ -148,7 +148,7 @@ bool osq_lock(struct optimistic_spin_queue *lock)
148148
* Or we race against a concurrent unqueue()'s step-B, in which
149149
* case its step-C will write us a new @node->prev pointer.
150150
*/
151-
prev = ACCESS_ONCE(node->prev);
151+
prev = READ_ONCE(node->prev);
152152
}
153153

154154
/*
@@ -170,8 +170,8 @@ bool osq_lock(struct optimistic_spin_queue *lock)
170170
* it will wait in Step-A.
171171
*/
172172

173-
ACCESS_ONCE(next->prev) = prev;
174-
ACCESS_ONCE(prev->next) = next;
173+
WRITE_ONCE(next->prev, prev);
174+
WRITE_ONCE(prev->next, next);
175175

176176
return false;
177177
}
@@ -193,11 +193,11 @@ void osq_unlock(struct optimistic_spin_queue *lock)
193193
node = this_cpu_ptr(&osq_node);
194194
next = xchg(&node->next, NULL);
195195
if (next) {
196-
ACCESS_ONCE(next->locked) = 1;
196+
WRITE_ONCE(next->locked, 1);
197197
return;
198198
}
199199

200200
next = osq_wait_next(lock, node, NULL);
201201
if (next)
202-
ACCESS_ONCE(next->locked) = 1;
202+
WRITE_ONCE(next->locked, 1);
203203
}

‎kernel/locking/rwsem-xadd.c‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -279,7 +279,7 @@ static inline bool rwsem_try_write_lock(long count, struct rw_semaphore *sem)
279279
*/
280280
static inline bool rwsem_try_write_lock_unqueued(struct rw_semaphore *sem)
281281
{
282-
long old, count = ACCESS_ONCE(sem->count);
282+
long old, count = READ_ONCE(sem->count);
283283

284284
while (true) {
285285
if (!(count == 0 || count == RWSEM_WAITING_BIAS))
@@ -304,9 +304,9 @@ static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem)
304304
return false;
305305

306306
rcu_read_lock();
307-
owner = ACCESS_ONCE(sem->owner);
307+
owner = READ_ONCE(sem->owner);
308308
if (!owner) {
309-
long count = ACCESS_ONCE(sem->count);
309+
long count = READ_ONCE(sem->count);
310310
/*
311311
* If sem->owner is not set, yet we have just recently entered the
312312
* slowpath with the lock being active, then there is a possibility
@@ -385,7 +385,7 @@ static bool rwsem_optimistic_spin(struct rw_semaphore *sem)
385385
goto done;
386386

387387
while (true) {
388-
owner = ACCESS_ONCE(sem->owner);
388+
owner = READ_ONCE(sem->owner);
389389
if (owner && !rwsem_spin_on_owner(sem, owner))
390390
break;
391391

@@ -459,7 +459,7 @@ struct rw_semaphore __sched *rwsem_down_write_failed(struct rw_semaphore *sem)
459459

460460
/* we're now waiting on the lock, but no longer actively locking */
461461
if (waiting) {
462-
count = ACCESS_ONCE(sem->count);
462+
count = READ_ONCE(sem->count);
463463

464464
/*
465465
* If there were already threads queued before us and there are

‎lib/lockref.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
#define CMPXCHG_LOOP(CODE, SUCCESS) do { \
1919
struct lockref old; \
2020
BUILD_BUG_ON(sizeof(old) != 8); \
21-
old.lock_count = ACCESS_ONCE(lockref->lock_count); \
21+
old.lock_count = READ_ONCE(lockref->lock_count); \
2222
while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) { \
2323
struct lockref new = old, prev = old; \
2424
CODE \

0 commit comments

Comments
 (0)