Skip to content

[WIP] feat: SocketTracer — PLT-patch Netty native transport for socket I/O events #457

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jbachorik wants to merge 4 commits into from
Closed

[WIP] feat: SocketTracer — PLT-patch Netty native transport for socket I/O events #457

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f8d2363
Add native Netty socket event tracking via PLT patching
jbachorik Mar 6, 2026
9b2d714
Fix review findings: add accept4 op name and aarch64 native dep
jbachorik Mar 6, 2026
9528c13
Add stacktraces to SocketIO JFR events with native frame skipping
jbachorik Mar 6, 2026
865f888
Potential fix for pull request finding 'Inefficient empty string test'
jbachorik Apr 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Add stacktraces to SocketIO JFR events with native frame skipping 
Integrate socket I/O events into the standard recordSample flow to
capture full stacktraces including native frames. Add skip_frames
parameter to walkFP, walkDwarf, and walkVM to skip internal profiler
frames (emitSocketIOEvent and hook functions) from the captured trace.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
  • Loading branch information
@jbachorik @claude
jbachorik and claude committed Mar 6, 2026
commit 9528c139a988de3fa6d45a7aceb7ff2939d20b59
1 change: 1 addition & 0 deletions ddprof-lib/src/main/cpp/counters.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@
X(SKIPPED_WALLCLOCK_UNWINDS, "skipped_wallclock_unwinds") \
X(UNWINDING_TIME_ASYNC, "unwinding_ticks_async") \
X(UNWINDING_TIME_JVMTI, "unwinding_ticks_jvmti") \
X(UNWINDING_TIME_SOCKETIO, "unwinding_ticks_socketio") \
X(CALLTRACE_STORAGE_DROPPED, "calltrace_storage_dropped_traces") \
X(LINE_NUMBER_TABLES, "line_number_tables") \
X(REMOTE_SYMBOLICATION_FRAMES, "remote_symbolication_frames") \
Expand Down
17 changes: 5 additions & 12 deletions ddprof-lib/src/main/cpp/flightRecorder.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1548,12 +1548,13 @@ void Recording::recordQueueTime(Buffer *buf, int tid, QueueTimeEvent *event) {
flushIfNeeded(buf);
}

void Recording::recordSocketIO(Buffer *buf, int tid, SocketIOEvent *event) {
void Recording::recordSocketIO(Buffer *buf, int tid, u64 call_trace_id, SocketIOEvent *event) {
int start = buf->skip(1);
buf->putVar64(T_SOCKET_IO);
buf->putVar64(event->_start);
buf->putVar64(event->_end - event->_start);
buf->putVar64(tid);
buf->putVar64(call_trace_id);
buf->putUtf8(event->_operation);
buf->putVar64(event->_bytes);
writeContext(buf, Contexts::get());
Expand Down Expand Up @@ -1764,17 +1765,6 @@ void FlightRecorder::recordQueueTime(int lock_index, int tid,
}
}

void FlightRecorder::recordSocketIO(int lock_index, int tid,
SocketIOEvent *event) {
OptionalSharedLockGuard locker(&_rec_lock);
if (locker.ownsLock()) {
Recording *rec = _rec;
if (rec != nullptr) {
Buffer *buf = rec->buffer(lock_index);
rec->recordSocketIO(buf, tid, event);
}
}
}

void FlightRecorder::recordDatadogSetting(int lock_index, int length,
const char *name, const char *value,
Expand Down Expand Up @@ -1829,6 +1819,9 @@ void FlightRecorder::recordEvent(int lock_index, int tid, u64 call_trace_id,
case BCI_PARK:
rec->recordThreadPark(buf, tid, call_trace_id, (LockEvent *)event);
break;
case BCI_SOCKET_IO:
rec->recordSocketIO(buf, tid, call_trace_id, (SocketIOEvent *)event);
break;
}
rec->flushIfNeeded(buf);
rec->addThread(lock_index, tid);
Expand Down
3 changes: 1 addition & 2 deletions ddprof-lib/src/main/cpp/flightRecorder.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -278,7 +278,7 @@ class Recording {
void recordWallClockEpoch(Buffer *buf, WallClockEpochEvent *event);
void recordTraceRoot(Buffer *buf, int tid, TraceRootEvent *event);
void recordQueueTime(Buffer *buf, int tid, QueueTimeEvent *event);
void recordSocketIO(Buffer *buf, int tid, SocketIOEvent *event);
void recordSocketIO(Buffer *buf, int tid, u64 call_trace_id, SocketIOEvent *event);
void recordAllocation(RecordingBuffer *buf, int tid, u64 call_trace_id,
AllocEvent *event);
void recordHeapLiveObject(Buffer *buf, int tid, u64 call_trace_id,
Expand Down Expand Up @@ -345,7 +345,6 @@ class FlightRecorder {
void wallClockEpoch(int lock_index, WallClockEpochEvent *event);
void recordTraceRoot(int lock_index, int tid, TraceRootEvent *event);
void recordQueueTime(int lock_index, int tid, QueueTimeEvent *event);
void recordSocketIO(int lock_index, int tid, SocketIOEvent *event);

bool active() const { return _rec != NULL; }

Expand Down
1 change: 1 addition & 0 deletions ddprof-lib/src/main/cpp/jfrMetadata.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -291,6 +291,7 @@ void JfrMetadata::initialize(
<< field("startTime", T_LONG, "Start Time", F_TIME_TICKS)
<< field("duration", T_LONG, "Duration", F_DURATION_TICKS)
<< field("eventThread", T_THREAD, "Event Thread", F_CPOOL)
<< field("stackTrace", T_STACK_TRACE, "Stack Trace", F_CPOOL)
<< field("operation", T_STRING, "Operation")
<< field("bytesTransferred", T_LONG, "Bytes Transferred")
<< field("spanId", T_LONG, "Span ID")
Expand Down
25 changes: 10 additions & 15 deletions ddprof-lib/src/main/cpp/profiler.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
#include "thread.h"
#include "tsc.h"
#include "vmStructs.h"
#include <ucontext.h>
#include "wallClock.h"
#include <algorithm>
#include <dlfcn.h>
Expand Down Expand Up @@ -299,14 +300,17 @@ int Profiler::getNativeTrace(void *ucontext, ASGCT_CallFrame *frames,
bool *truncated, int lock_index) {
if (_cstack == CSTACK_NO ||
(event_type == BCI_ALLOC || event_type == BCI_ALLOC_OUTSIDE_TLAB) ||
(event_type != BCI_CPU && event_type != BCI_WALL &&
(event_type != BCI_CPU && event_type != BCI_WALL && event_type != BCI_SOCKET_IO &&
_cstack == CSTACK_DEFAULT)) {
return 0;
}
int max_depth = min(_max_stack_depth, MAX_NATIVE_FRAMES);
const void *callchain[max_depth + 1]; // we can read one frame past when trying to figure out whether the result is truncated
int native_frames = 0;

// For socket I/O events, skip emitSocketIOEvent and the hook frame (e.g., epoll_wait_hook)
int skip_frames = (event_type == BCI_SOCKET_IO) ? 2 : 0;

if (event_type == BCI_CPU && _cpu_engine == &perf_events) {
native_frames +=
PerfEvents::walkKernel(tid, callchain + native_frames,
Expand All @@ -317,11 +321,11 @@ int Profiler::getNativeTrace(void *ucontext, ASGCT_CallFrame *frames,
} else if (_cstack == CSTACK_DWARF) {
native_frames += StackWalker::walkDwarf(ucontext, callchain + native_frames,
max_depth - native_frames,
java_ctx, truncated);
java_ctx, truncated, skip_frames);
} else {
native_frames += StackWalker::walkFP(ucontext, callchain + native_frames,
max_depth - native_frames,
java_ctx, truncated);
java_ctx, truncated, skip_frames);
}

return convertNativeTrace(native_frames, callchain, frames, lock_index);
Expand Down Expand Up @@ -839,10 +843,11 @@ void Profiler::recordSample(void *ucontext, u64 counter, int tid,
int vm_start = num_frames;
int vm_frames = StackWalker::walkVM(ucontext, frames + vm_start, max_remaining, _features, eventTypeFromBCI(event_type), lock_index, &truncated);
Copy link
Copy Markdown
Collaborator Author

@jbachorik jbachorik Apr 9, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[terminus] [Confidence: 83%] [Severity: MEDIUM→HIGH] — incorrect-stack-trace

When _features.mixed is true and event_type == BCI_SOCKET_IO, skip_frames is not passed. The non-mixed path at line 850 correctly passes skip_frames=2. In mixed mode, emitSocketIOEvent and the hook frame will appear as visible frames in socket-IO stack traces.

Validated by: janus, occam (3 validates, 0 challenges).

Suggested change
int vm_frames = StackWalker::walkVM(ucontext, frames + vm_start, max_remaining, _features, eventTypeFromBCI(event_type), lock_index, &truncated);
int skip = (event_type == BCI_SOCKET_IO) ? 2 : 0;
int vm_frames = StackWalker::walkVM(ucontext, frames + vm_start, max_remaining, _features, eventTypeFromBCI(event_type), lock_index, &truncated, skip);

num_frames += vm_frames;
} else if (event_type == BCI_CPU || event_type == BCI_WALL) {
} else if (event_type == BCI_CPU || event_type == BCI_WALL || event_type == BCI_SOCKET_IO) {
if (_cstack >= CSTACK_VM) {
int vm_start = num_frames;
int vm_frames = StackWalker::walkVM(ucontext, frames + vm_start, max_remaining, _features, eventTypeFromBCI(event_type), lock_index, &truncated);
int skip_frames = (event_type == BCI_SOCKET_IO) ? 2 : 0;
int vm_frames = StackWalker::walkVM(ucontext, frames + vm_start, max_remaining, _features, eventTypeFromBCI(event_type), lock_index, &truncated, skip_frames);
num_frames += vm_frames;
} else {
// Async events
Expand Down Expand Up @@ -914,16 +919,6 @@ void Profiler::recordQueueTime(int tid, QueueTimeEvent *event) {
_locks[lock_index].unlock();
}

void Profiler::recordSocketIO(int tid, SocketIOEvent *event) {
u32 lock_index = getLockIndex(tid);
if (!_locks[lock_index].tryLock() &&
!_locks[lock_index = (lock_index + 1) % CONCURRENCY_LEVEL].tryLock() &&
!_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock()) {
return;
}
_jfr.recordSocketIO(lock_index, tid, event);
_locks[lock_index].unlock();
}

void Profiler::recordExternalSample(u64 weight, int tid, int num_frames,
ASGCT_CallFrame *frames, bool truncated,
Expand Down
1 change: 0 additions & 1 deletion ddprof-lib/src/main/cpp/profiler.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -375,7 +375,6 @@ class alignas(alignof(SpinLock)) Profiler {
void recordWallClockEpoch(int tid, WallClockEpochEvent *event);
void recordTraceRoot(int tid, TraceRootEvent *event);
void recordQueueTime(int tid, QueueTimeEvent *event);
void recordSocketIO(int tid, SocketIOEvent *event);
void writeLog(LogLevel level, const char *message);
void writeLog(LogLevel level, const char *message, size_t len);
void writeDatadogProfilerSetting(int tid, int length, const char *name,
Expand Down
13 changes: 12 additions & 1 deletion ddprof-lib/src/main/cpp/socketTracer_linux.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
#include <sys/epoll.h>
#include <sys/socket.h>
#include <sys/uio.h>
#include <ucontext.h>
#include <unistd.h>

// --------------------------------------------------------------------------
Expand Down Expand Up @@ -109,7 +110,17 @@ static inline void emitSocketIOEvent(const char* operation, u64 start, u64 end,
event._end = end;
event._operation = operation;
event._bytes = bytes;
Profiler::instance()->recordSocketIO(tid, &event);

// Capture current context for stack walking
ucontext_t uctx;
void* ucontext_ptr = nullptr;
if (getcontext(&uctx) == 0) {
ucontext_ptr = &uctx;
}

// Call recordSample with BCI_SOCKET_IO event type
// counter=0, call_trace_id=0 (will be captured by recordSample)
Profiler::instance()->recordSample(ucontext_ptr, 0, tid, BCI_SOCKET_IO, 0, (Event*)&event);
}

// --------------------------------------------------------------------------
Expand Down
Loading
Loading