DataDog · zhengyu123 · Jul 9, 2025 · Jun 30, 2025 · Jun 30, 2025 · Jun 30, 2025
@@ -607,6 +607,7 @@ u32 Profiler::recordJVMTISample(u64 counter, int tid, jthread thread, jint event
 
     int num_frames = 0;
 
+    // This is a racy call, because the thread may not be alive at this moment.
     if (VM::jvmti()->GetStackTrace(thread, 0, _max_stack_depth, jvmti_frames, &num_frames) == JVMTI_ERROR_NONE && num_frames > 0) {
       // Convert to AsyncGetCallTrace format.
       // Note: jvmti_frames and frames may overlap.
@@ -618,18 +619,17 @@ u32 Profiler::recordJVMTISample(u64 counter, int tid, jthread thread, jint event
         // see https://github.com/async-profiler/async-profiler/pull/1090
         LP64_ONLY(frames[i].padding = 0;)
       }
-    }
 
-    call_trace_id = _call_trace_storage.put(num_frames, frames, false, counter);
-    u64 duration = TSC::ticks() - startTime;
-    if (duration > 0) {
-      Counters::increment(UNWINDING_TIME_JVMTI, duration); // increment the JVMTI specific counter
-    }
-  }
-  if (!deferred) {
-    _jfr.recordEvent(lock_index, tid, call_trace_id, event_type, event);
+        call_trace_id = _call_trace_storage.put(num_frames, frames, false, counter);
+        u64 duration = TSC::ticks() - startTime;
+        if (duration > 0) {
+          Counters::increment(UNWINDING_TIME_JVMTI, duration); // increment the JVMTI specific counter
+        }
+      }
+      if (!deferred) {
+        _jfr.recordEvent(lock_index, tid, call_trace_id, event_type, event);
+      }
   }
-
   _locks[lock_index].unlock();
   return call_trace_id;
 }

diff --git a/ddprof-lib/src/main/cpp/threadState.h b/ddprof-lib/src/main/cpp/threadState.h
@@ -1,9 +1,10 @@
 #ifndef JAVA_PROFILER_LIBRARY_THREAD_STATE_H
 #define JAVA_PROFILER_LIBRARY_THREAD_STATE_H
 
+#include "arch.h"
 #include "jvmti.h"
 
-enum class OSThreadState : int {
+enum class OSThreadState : u32 {
   UNKNOWN = 0,
   NEW = 1,      // The thread has been initialized but yet started
   RUNNABLE = 2, // Has been started and is runnable, but not necessarily running

@@ -20,6 +20,7 @@
 #include "common.h"
 #include "jniHelper.h"
 #include "jvmHeap.h"
+#include "safeAccess.h"
 #include "threadState.h"
 #include "vmEntry.h"
 #include "vmStructs.h"
@@ -66,9 +67,19 @@ namespace ddprof {
     inline static int thread_osthread_offset() {
       return _thread_osthread_offset;
     }
+
     inline static int osthread_state_offset() {
       return _osthread_state_offset;
     }
+
+    inline static int osthread_id_offset() {
+      return _osthread_id_offset;
+    }
+
+    inline static int thread_state_offset() {
+      return _thread_state_offset;
+    }
+
     inline static int flag_type_offset() {
       return _flag_type_offset;
     }
@@ -139,6 +150,53 @@ namespace ddprof {
       }
       return OSThreadState::UNKNOWN;
     }
+
+    OSThreadState osThreadStateSafe() {
+      if (ddprof::VMStructs::thread_osthread_offset() >= 0 && ddprof::VMStructs::osthread_state_offset() >= 0) {
+        u32 *osthread = *(u32 **)at(ddprof::VMStructs::thread_osthread_offset());
+        if (osthread != nullptr) {
+          // If the location is accessible, the thread must have been terminated
+          u32 value = SafeAccess::load32(osthread + ddprof::VMStructs::osthread_state_offset(),
+                                         static_cast<u32>(OSThreadState::TERMINATED));
+          // Bad value, treat it as terminated
+          if (value > static_cast<u32>(OSThreadState::SYSCALL)) {
+            return OSThreadState::TERMINATED;
+          }
+          return static_cast<OSThreadState>(value);
+        }
+     }
+     return OSThreadState::UNKNOWN;
+    }
+
+    int osThreadIdSafe() {
+      if (ddprof::VMStructs::thread_osthread_offset() >= 0 && ddprof::VMStructs::osthread_id_offset() >=0) {
+        u32* osthread = *(u32**) at(ddprof::VMStructs::thread_osthread_offset());
+        if (osthread == nullptr) {
+          return -1;
+        } else {
+          u32 value = SafeAccess::load32(osthread + ddprof::VMStructs::osthread_id_offset(), -1);
+          return static_cast<int>(value);
+        }
+      }
+      return -1;
+    }
+
+    int stateSafe() {
+      int offset = ddprof::VMStructs::thread_state_offset();
+      if (offset >= 0) {
+          u32* state = *(u32**)at(offset);
+          if (state == nullptr) {
+            return 0;
+          } else {
+            u32 value = SafeAccess::load32(state, 0);
+            if (value > static_cast<u32>(_thread_max_state)) {
+              value = 0;
+            }
+            return static_cast<int>(value);
+          }
+      }
+      return 0;
+    }
   };
 
   class JVMFlag : public VMStructs {

@@ -153,6 +153,13 @@
   OS::installSignalHandler(SIGVTALRM, sharedSignalHandler);
 }
 
+static long eventCount = 0;
+
+/* This method is extremely racy!
+ * Thread references that are returned from JVMTI GetAllThreads(), only guarantee thread objects
+ * not to be collected by GC, they don't prevent threads from exiting.
+ * We have to be extremely cautious when accessing thread's data.
+ */
 void WallClockJVMTI::timerLoop() {
     // Check for enablement before attaching/dettaching the current thread
   if (!isEnabled()) {
@@ -181,9 +188,10 @@
             if (nThread == nullptr) {
               continue;
             }
-            int tid = nThread->osThreadId();
+            // Racy, use safe version
+            int tid = nThread->osThreadIdSafe();
             if (tid != self && (!do_filter || Profiler::instance()->threadFilter()->accept(tid))) {
-              threads.push_back({nThread, thread});
+              threads.push_back({nThread, thread, tid});
             }
           }
         }
@@ -192,35 +200,56 @@
 
   auto sampleThreads = [&](ThreadEntry& thread_entry, int& num_failures, int& threads_already_exited, int& permission_denied) {
     static jint max_stack_depth = (jint)Profiler::instance()->max_stack_depth();
+    jvmtiEnv* jvmti = VM::jvmti();
+    jint thread_state;
+    // Reliable test
+    if (jvmti->GetThreadState(thread_entry.java, &thread_state) != JVMTI_ERROR_NONE ||
+        (thread_state & JVMTI_THREAD_STATE_ALIVE) == 0) {
+        printf("Thread no longer alive\n");
+      return false;
+    }
 
+    // Following code is racy, use safe version to access native structure.
     ExecutionEvent event;
     ddprof::VMThread* vm_thread = thread_entry.native;
-    int raw_thread_state = vm_thread->state();
+    int raw_thread_state = vm_thread->stateSafe();
     bool is_initialized = raw_thread_state >= ddprof::JVMJavaThreadState::_thread_in_native &&
                           raw_thread_state < ddprof::JVMJavaThreadState::_thread_max_state;
     OSThreadState state = OSThreadState::UNKNOWN;
     ExecutionMode mode = ExecutionMode::UNKNOWN;
-    if (vm_thread && is_initialized) {
-      OSThreadState os_state = vm_thread->osThreadState();
-      if (os_state != OSThreadState::UNKNOWN) {
-        state = os_state;
-      }
-      mode = convertJvmExecutionState(raw_thread_state);
+    if (vm_thread == nullptr || !is_initialized) {
+    printf("Thread not initialized\n");
+        return false;
     }
-    if (state == OSThreadState::UNKNOWN) {
+    OSThreadState os_state = vm_thread->osThreadStateSafe();
+    if (state == OSThreadState::TERMINATED) {
+    printf("Thread terminated\n");
+      return false;
+    } else if (state == OSThreadState::UNKNOWN) {
       state = OSThreadState::RUNNABLE;
+    } else {
+      state = os_state;
     }
+    mode = convertJvmExecutionState(raw_thread_state);
+
     event._thread_state = state;
     event._execution_mode = mode;
     event._weight =  1;
 
-    Profiler::instance()->recordJVMTISample(1, thread_entry.native->osThreadId(), thread_entry.java, BCI_WALL, &event, false);
+    Profiler::instance()->recordJVMTISample(1, thread_entry.tid, thread_entry.java, BCI_WALL, &event, false);
+    eventCount ++;
     return true;
   };
 
-  timerLoopCommon<ThreadEntry>(collectThreads, sampleThreads, _reservoir_size, _interval);
+  auto clearThreadRefs = [](ThreadEntry& thread_entry) {
+    VM::jni()->DeleteLocalRef(thread_entry.java);
+  };
+
+  eventCount = 0;
+  timerLoopCommon<ThreadEntry>(collectThreads, sampleThreads, clearThreadRefs, _reservoir_size, _interval);
   // Don't forget to detach the thread
   VM::detachThread();
+  printf("Event generated: %d\n", eventCount);
 }
 
 void WallClockASGCT::timerLoop() {
@@ -257,5 +286,8 @@
       return true;
     };
 
-    timerLoopCommon<int>(collectThreads, sampleThreads, _reservoir_size, _interval);
+    auto doNothing = [](int tid) {
+    };
+
+    timerLoopCommon<int>(collectThreads, sampleThreads, doNothing, _reservoir_size, _interval);
 }
@@ -50,8 +50,8 @@ class BaseWallClock : public Engine {
 
     bool isEnabled() const;
 
-    template <typename ThreadType, typename CollectThreadsFunc, typename SampleThreadsFunc>
-    void timerLoopCommon(CollectThreadsFunc collectThreads, SampleThreadsFunc sampleThreads, int reservoirSize, u64 interval) {
+    template <typename ThreadType, typename CollectThreadsFunc, typename SampleThreadsFunc, typename CleanThreadFunc>
+    void timerLoopCommon(CollectThreadsFunc collectThreads, SampleThreadsFunc sampleThreads, CleanThreadFunc cleanThreads, int reservoirSize, u64 interval) {
       if (!_enabled.load(std::memory_order_acquire)) {
         return;
       }
@@ -103,6 +103,10 @@ class BaseWallClock : public Engine {
           epoch.clean();
         }
 
+        for (ThreadType thread : threads) {
+          cleanThreads(thread);
+        }
+
         threads.clear();
         // Get a random sleep duration
         // clamp the random interval to <1,2N-1>
@@ -161,6 +165,7 @@ class WallClockJVMTI : public BaseWallClock {
     struct ThreadEntry {
         ddprof::VMThread* native;
         jthread java;
+        int tid;
     };
     WallClockJVMTI() : BaseWallClock() {}
     const char* name() override {