Skip to content

Commit c353f4f

Browse files
franz1981franz1981
authored
Faster Recycler's claim/release (Fixes #13153) (#13220)
Motivation: Recycler's claim/release can be made faster by saving expensive volatile ops, when not needed. For claim, always, while for release, if the owner thread is performing release itself. Modification: Replacing expensive volatile ops with ordered ones. Result: Faster Recycler's claim/release
1 parent 84cf7d6 commit c353f4f

8 files changed

Lines changed: 149 additions & 53 deletions

File tree

‎buffer/src/main/java/io/netty/buffer/AbstractPooledDerivedByteBuf.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
package io.netty.buffer;
1818

19+
import io.netty.util.Recycler.EnhancedHandle;
1920
import io.netty.util.internal.ObjectPool.Handle;
2021

2122
import java.nio.ByteBuffer;
@@ -26,7 +27,7 @@
2627
*/
2728
abstract class AbstractPooledDerivedByteBuf extends AbstractReferenceCountedByteBuf {
2829

29-
private final Handle<AbstractPooledDerivedByteBuf> recyclerHandle;
30+
private final EnhancedHandle<AbstractPooledDerivedByteBuf> recyclerHandle;
3031
private AbstractByteBuf rootParent;
3132
/**
3233
* Deallocations of a pooled derived buffer should always propagate through the entire chain of derived buffers.
@@ -39,7 +40,7 @@ abstract class AbstractPooledDerivedByteBuf extends AbstractReferenceCountedByte
3940
@SuppressWarnings("unchecked")
4041
AbstractPooledDerivedByteBuf(Handle<? extends AbstractPooledDerivedByteBuf> recyclerHandle) {
4142
super(0);
42-
this.recyclerHandle = (Handle<AbstractPooledDerivedByteBuf>) recyclerHandle;
43+
this.recyclerHandle = (EnhancedHandle<AbstractPooledDerivedByteBuf>) recyclerHandle;
4344
}
4445

4546
// Called from within SimpleLeakAwareByteBuf and AdvancedLeakAwareByteBuf.
@@ -82,7 +83,7 @@ protected final void deallocate() {
8283
// otherwise it is possible that the same AbstractPooledDerivedByteBuf is again obtained and init(...) is
8384
// called before we actually have a chance to call release(). This leads to call release() on the wrong parent.
8485
ByteBuf parent = this.parent;
85-
recyclerHandle.recycle(this);
86+
recyclerHandle.unguardedRecycle(this);
8687
parent.release();
8788
}
8889

‎buffer/src/main/java/io/netty/buffer/ByteBufUtil.java‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import io.netty.util.ByteProcessor;
2020
import io.netty.util.CharsetUtil;
2121
import io.netty.util.IllegalReferenceCountException;
22+
import io.netty.util.Recycler.EnhancedHandle;
2223
import io.netty.util.concurrent.FastThreadLocal;
2324
import io.netty.util.internal.MathUtil;
2425
import io.netty.util.internal.ObjectPool;
@@ -1632,11 +1633,11 @@ static ThreadLocalUnsafeDirectByteBuf newInstance() {
16321633
return buf;
16331634
}
16341635

1635-
private final Handle<ThreadLocalUnsafeDirectByteBuf> handle;
1636+
private final EnhancedHandle<ThreadLocalUnsafeDirectByteBuf> handle;
16361637

16371638
private ThreadLocalUnsafeDirectByteBuf(Handle<ThreadLocalUnsafeDirectByteBuf> handle) {
16381639
super(UnpooledByteBufAllocator.DEFAULT, 256, Integer.MAX_VALUE);
1639-
this.handle = handle;
1640+
this.handle = (EnhancedHandle<ThreadLocalUnsafeDirectByteBuf>) handle;
16401641
}
16411642

16421643
@Override
@@ -1645,7 +1646,7 @@ protected void deallocate() {
16451646
super.deallocate();
16461647
} else {
16471648
clear();
1648-
handle.recycle(this);
1649+
handle.unguardedRecycle(this);
16491650
}
16501651
}
16511652
}
@@ -1666,11 +1667,11 @@ static ThreadLocalDirectByteBuf newInstance() {
16661667
return buf;
16671668
}
16681669

1669-
private final Handle<ThreadLocalDirectByteBuf> handle;
1670+
private final EnhancedHandle<ThreadLocalDirectByteBuf> handle;
16701671

16711672
private ThreadLocalDirectByteBuf(Handle<ThreadLocalDirectByteBuf> handle) {
16721673
super(UnpooledByteBufAllocator.DEFAULT, 256, Integer.MAX_VALUE);
1673-
this.handle = handle;
1674+
this.handle = (EnhancedHandle<ThreadLocalDirectByteBuf>) handle;
16741675
}
16751676

16761677
@Override
@@ -1679,7 +1680,7 @@ protected void deallocate() {
16791680
super.deallocate();
16801681
} else {
16811682
clear();
1682-
handle.recycle(this);
1683+
handle.unguardedRecycle(this);
16831684
}
16841685
}
16851686
}

‎buffer/src/main/java/io/netty/buffer/PoolThreadCache.java‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import static io.netty.util.internal.ObjectUtil.checkPositiveOrZero;
2121

2222
import io.netty.buffer.PoolArena.SizeClass;
23+
import io.netty.util.Recycler.EnhancedHandle;
2324
import io.netty.util.internal.MathUtil;
2425
import io.netty.util.internal.ObjectPool;
2526
import io.netty.util.internal.ObjectPool.Handle;
@@ -377,7 +378,7 @@ public final boolean add(PoolChunk<T> chunk, ByteBuffer nioBuffer, long handle,
377378
boolean queued = queue.offer(entry);
378379
if (!queued) {
379380
// If it was not possible to cache the chunk, immediately recycle the entry
380-
entry.recycle();
381+
entry.unguardedRecycle();
381382
}
382383

383384
return queued;
@@ -392,7 +393,7 @@ public final boolean allocate(PooledByteBuf<T> buf, int reqCapacity, PoolThreadC
392393
return false;
393394
}
394395
initBuf(entry.chunk, entry.nioBuffer, entry.handle, buf, reqCapacity, threadCache);
395-
entry.recycle();
396+
entry.unguardedRecycle();
396397

397398
// allocations is not thread-safe which is fine as this is only called from the same thread all time.
398399
++ allocations;
@@ -451,14 +452,14 @@ private void freeEntry(Entry entry, boolean finalizer) {
451452
}
452453

453454
static final class Entry<T> {
454-
final Handle<Entry<?>> recyclerHandle;
455+
final EnhancedHandle<Entry<?>> recyclerHandle;
455456
PoolChunk<T> chunk;
456457
ByteBuffer nioBuffer;
457458
long handle = -1;
458459
int normCapacity;
459460

460461
Entry(Handle<Entry<?>> recyclerHandle) {
461-
this.recyclerHandle = recyclerHandle;
462+
this.recyclerHandle = (EnhancedHandle<Entry<?>>) recyclerHandle;
462463
}
463464

464465
void recycle() {
@@ -467,6 +468,13 @@ void recycle() {
467468
handle = -1;
468469
recyclerHandle.recycle(this);
469470
}
471+
472+
void unguardedRecycle() {
473+
chunk = null;
474+
nioBuffer = null;
475+
handle = -1;
476+
recyclerHandle.unguardedRecycle(this);
477+
}
470478
}
471479

472480
@SuppressWarnings("rawtypes")

‎buffer/src/main/java/io/netty/buffer/PooledByteBuf.java‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
package io.netty.buffer;
1818

19+
import io.netty.util.Recycler.EnhancedHandle;
1920
import io.netty.util.internal.ObjectPool.Handle;
2021

2122
import java.io.IOException;
@@ -28,7 +29,7 @@
2829

2930
abstract class PooledByteBuf<T> extends AbstractReferenceCountedByteBuf {
3031

31-
private final Handle<PooledByteBuf<T>> recyclerHandle;
32+
private final EnhancedHandle<PooledByteBuf<T>> recyclerHandle;
3233

3334
protected PoolChunk<T> chunk;
3435
protected long handle;
@@ -43,7 +44,7 @@ abstract class PooledByteBuf<T> extends AbstractReferenceCountedByteBuf {
4344
@SuppressWarnings("unchecked")
4445
protected PooledByteBuf(Handle<? extends PooledByteBuf<T>> recyclerHandle, int maxCapacity) {
4546
super(maxCapacity);
46-
this.recyclerHandle = (Handle<PooledByteBuf<T>>) recyclerHandle;
47+
this.recyclerHandle = (EnhancedHandle<PooledByteBuf<T>>) recyclerHandle;
4748
}
4849

4950
void init(PoolChunk<T> chunk, ByteBuffer nioBuffer,
@@ -177,14 +178,10 @@ protected final void deallocate() {
177178
tmpNioBuf = null;
178179
chunk = null;
179180
cache = null;
180-
recycle();
181+
this.recyclerHandle.unguardedRecycle(this);
181182
}
182183
}
183184

184-
private void recycle() {
185-
recyclerHandle.recycle(this);
186-
}
187-
188185
protected final int idx(int index) {
189186
return offset + index;
190187
}

‎common/src/main/java/io/netty/util/Recycler.java‎

Lines changed: 41 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import io.netty.util.internal.ObjectPool;
2121
import io.netty.util.internal.PlatformDependent;
2222
import io.netty.util.internal.SystemPropertyUtil;
23+
import io.netty.util.internal.UnstableApi;
2324
import io.netty.util.internal.logging.InternalLogger;
2425
import io.netty.util.internal.logging.InternalLoggerFactory;
2526
import org.jctools.queues.MessagePassingQueue;
@@ -40,12 +41,17 @@
4041
*/
4142
public abstract class Recycler<T> {
4243
private static final InternalLogger logger = InternalLoggerFactory.getInstance(Recycler.class);
43-
private static final Handle<?> NOOP_HANDLE = new Handle<Object>() {
44+
private static final EnhancedHandle<?> NOOP_HANDLE = new EnhancedHandle<Object>() {
4445
@Override
4546
public void recycle(Object object) {
4647
// NOOP
4748
}
4849

50+
@Override
51+
public void unguardedRecycle(final Object object) {
52+
// NOOP
53+
}
54+
4955
@Override
5056
public String toString() {
5157
return "NOOP_HANDLE";
@@ -216,7 +222,16 @@ final int threadLocalSize() {
216222
@SuppressWarnings("ClassNameSameAsAncestorName") // Can't change this due to compatibility.
217223
public interface Handle<T> extends ObjectPool.Handle<T> { }
218224

219-
private static final class DefaultHandle<T> implements Handle<T> {
225+
@UnstableApi
226+
public abstract static class EnhancedHandle<T> implements Handle<T> {
227+
228+
public abstract void unguardedRecycle(Object object);
229+
230+
private EnhancedHandle() {
231+
}
232+
}
233+
234+
private static final class DefaultHandle<T> extends EnhancedHandle<T> {
220235
private static final int STATE_CLAIMED = 0;
221236
private static final int STATE_AVAILABLE = 1;
222237
private static final AtomicIntegerFieldUpdater<DefaultHandle<?>> STATE_UPDATER;
@@ -239,7 +254,15 @@ public void recycle(Object object) {
239254
if (object != value) {
240255
throw new IllegalArgumentException("object does not belong to handle");
241256
}
242-
localPool.release(this);
257+
localPool.release(this, true);
258+
}
259+
260+
@Override
261+
public void unguardedRecycle(Object object) {
262+
if (object != value) {
263+
throw new IllegalArgumentException("object does not belong to handle");
264+
}
265+
localPool.release(this, false);
243266
}
244267

245268
T get() {
@@ -252,7 +275,7 @@ void set(T value) {
252275

253276
void toClaimed() {
254277
assert state == STATE_AVAILABLE;
255-
state = STATE_CLAIMED;
278+
STATE_UPDATER.lazySet(this, STATE_CLAIMED);
256279
}
257280

258281
void toAvailable() {
@@ -261,6 +284,14 @@ void toAvailable() {
261284
throw new IllegalStateException("Object has been recycled already.");
262285
}
263286
}
287+
288+
void unguardedToAvailable() {
289+
int prev = state;
290+
if (prev == STATE_AVAILABLE) {
291+
throw new IllegalStateException("Object has been recycled already.");
292+
}
293+
STATE_UPDATER.lazySet(this, STATE_AVAILABLE);
294+
}
264295
}
265296

266297
private static final class LocalPool<T> implements MessagePassingQueue.Consumer<DefaultHandle<T>> {
@@ -301,8 +332,12 @@ DefaultHandle<T> claim() {
301332
return handle;
302333
}
303334

304-
void release(DefaultHandle<T> handle) {
305-
handle.toAvailable();
335+
void release(DefaultHandle<T> handle, boolean guarded) {
336+
if (guarded) {
337+
handle.toAvailable();
338+
} else {
339+
handle.unguardedToAvailable();
340+
}
306341
Thread owner = this.owner;
307342
if (owner != null && Thread.currentThread() == owner && batch.size() < chunkSize) {
308343
accept(handle);

‎common/src/main/java/io/netty/util/internal/ReferenceCountUpdater.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,8 @@ public final void setRefCnt(T instance, int refCnt) {
112112
* Resets the reference count to 1
113113
*/
114114
public final void resetRefCnt(T instance) {
115-
updater().set(instance, initialValue());
115+
// no need of a volatile set, it should happen in a quiescent state
116+
updater().lazySet(instance, initialValue());
116117
}
117118

118119
public final T retain(T instance) {

0 commit comments

Comments
 (0)