Add likely/unlikely hints across hot-path subsystems

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>

Author: jbachorik

ddprof-lib/src/main/cpp/buffers.h

@@ -42,7 +42,7 @@ class Buffer {
   virtual int limit() const { return _limit; }
 
   bool flushIfNeeded(FlushCallback callback, int limit = BUFFER_LIMIT) {
-    if (_offset > limit) {
+    if (unlikely(_offset > limit)) {
       if (callback(_data, _offset) == _offset) {
         reset();
         return true;

ddprof-lib/src/main/cpp/codeCache.cpp

@@ -25,11 +25,11 @@ char *NativeFunc::create(const char *name, short lib_index) {
 void NativeFunc::destroy(char *name) { free(from(name)); }
 
 char NativeFunc::read_mark(const char* name) {
-  if (name == nullptr) {
+  if (unlikely(name == nullptr)) {
     return 0;
   }
   NativeFunc* func = from(name);
-  if (!is_aligned(func, sizeof(func))) {
+  if (unlikely(!is_aligned(func, sizeof(func)))) {
     return 0;
   }
   // Use SafeAccess to read the mark field in signal handler context
@@ -394,7 +394,7 @@ void CodeCache::setDwarfTable(FrameDesc *table, int length) {
 }
 
 FrameDesc CodeCache::findFrameDesc(const void *pc) {
-  if (_dwarf_table == NULL || _dwarf_table_length == 0) {
+  if (unlikely(_dwarf_table == NULL || _dwarf_table_length == 0)) {
     // No DWARF data available - use default frame pointer unwinding
     // This handles OpenJ9 and other VMs that don't provide DWARF info
     return FrameDesc::default_frame;
@@ -415,7 +415,7 @@ FrameDesc CodeCache::findFrameDesc(const void *pc) {
     }
   }
 
-  if (low > 0) {
+  if (likely(low > 0)) {
     return _dwarf_table[low - 1];
   } else if (target_loc - _plt_offset < _plt_size) {
     return FrameDesc::empty_frame;

ddprof-lib/src/main/cpp/context.cpp

@@ -40,7 +40,7 @@ Context& Contexts::initializeContextTls() {
 
 Context& Contexts::get() {
   ProfiledThread* thrd = ProfiledThread::currentSignalSafe();
-  if (thrd == nullptr || !thrd->isContextTlsInitialized()) {
+  if (unlikely(thrd == nullptr || !thrd->isContextTlsInitialized())) {
     return DD_EMPTY_CONTEXT;
   }
   // Return via stored pointer - never access context_tls_v1 from signal handler

ddprof-lib/src/main/cpp/flightRecorder.cpp

@@ -794,7 +794,7 @@ void Recording::flush(Buffer *buf) {
 }
 
 void Recording::flushIfNeeded(Buffer *buf, int limit) {
-  if (buf->offset() >= limit) {
+  if (unlikely(buf->offset() >= limit)) {
     flush(buf);
   }
 }

ddprof-lib/src/main/cpp/linearAllocator.cpp

@@ -138,7 +138,7 @@ void *LinearAllocator::alloc(size_t size) {
 
   // CRITICAL FIX: After detachChunks() fails, _tail may be nullptr.
   // We must handle this gracefully to prevent crash.
-  if (chunk == nullptr) {
+  if (unlikely(chunk == nullptr)) {
     return nullptr;
   }
 
@@ -157,7 +157,7 @@ void *LinearAllocator::alloc(size_t size) {
         ASAN_UNPOISON_MEMORY_REGION(allocated_ptr, size);
         #endif
 
-        if (_chunk_size / 2 - offs < size) {
+        if (unlikely(_chunk_size / 2 - offs < size)) {
           // Stepped over a middle of the chunk - it's time to prepare a new one
           reserveChunk(chunk);
         }
@@ -171,7 +171,7 @@ void *LinearAllocator::alloc(size_t size) {
 
 Chunk *LinearAllocator::allocateChunk(Chunk *current) {
   Chunk *chunk = (Chunk *)OS::safeAlloc(_chunk_size);
-  if (chunk != NULL) {
+  if (likely(chunk != NULL)) {
     chunk->prev = current;
     chunk->offs = sizeof(Chunk);
 
@@ -208,18 +208,18 @@ void LinearAllocator::reserveChunk(Chunk *current) {
 Chunk *LinearAllocator::getNextChunk(Chunk *current) {
   Chunk *reserve = _reserve;
 
-  if (reserve == current) {
+  if (unlikely(reserve == current)) {
     // Unlikely case: no reserve yet.
     // It's probably being allocated right now, so let's compete
     reserve = allocateChunk(current);
-    if (reserve == NULL) {
+    if (unlikely(reserve == NULL)) {
       // Not enough memory
       return NULL;
     }
 
     Chunk *prev_reserve =
         __sync_val_compare_and_swap(&_reserve, current, reserve);
-    if (prev_reserve != current) {
+    if (unlikely(prev_reserve != current)) {
       freeChunk(reserve);
       reserve = prev_reserve;
     }

ddprof-lib/src/main/cpp/profiler.cpp

@@ -762,9 +762,9 @@ u64 Profiler::recordJVMTISample(u64 counter, int tid, jthread thread, jint event
   atomicIncRelaxed(_total_samples);
 
   u32 lock_index = getLockIndex(tid);
-  if (!_locks[lock_index].tryLock() &&
+  if (unlikely(!_locks[lock_index].tryLock() &&
       !_locks[lock_index = (lock_index + 1) % CONCURRENCY_LEVEL].tryLock() &&
-      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock()) {
+      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock())) {
     // Too many concurrent signals already
     atomicIncRelaxed(_failures[-ticks_skipped]);
 
@@ -809,9 +809,9 @@ void Profiler::recordDeferredSample(int tid, u64 call_trace_id, jint event_type,
   atomicIncRelaxed(_total_samples);
 
   u32 lock_index = getLockIndex(tid);
-  if (!_locks[lock_index].tryLock() &&
+  if (unlikely(!_locks[lock_index].tryLock() &&
       !_locks[lock_index = (lock_index + 1) % CONCURRENCY_LEVEL].tryLock() &&
-      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock()) {
+      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock())) {
     // Too many concurrent signals already
     atomicIncRelaxed(_failures[-ticks_skipped]);
     return;
@@ -827,9 +827,9 @@ void Profiler::recordSample(void *ucontext, u64 counter, int tid,
   atomicIncRelaxed(_total_samples);
 
   u32 lock_index = getLockIndex(tid);
-  if (!_locks[lock_index].tryLock() &&
+  if (unlikely(!_locks[lock_index].tryLock() &&
       !_locks[lock_index = (lock_index + 1) % CONCURRENCY_LEVEL].tryLock() &&
-      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock()) {
+      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock())) {
     // Too many concurrent signals already
     atomicIncRelaxed(_failures[-ticks_skipped]);
 
@@ -847,7 +847,7 @@ void Profiler::recordSample(void *ucontext, u64 counter, int tid,
   // record a null stacktrace we can skip the unwind if we've got a
   // call_trace_id determined to be reusable at a higher level
 
-  if (!_omit_stacktraces && call_trace_id == 0) {
+  if (likely(!_omit_stacktraces && call_trace_id == 0)) {
     u64 startTime = TSC::ticks();
     ASGCT_CallFrame *frames = _calltrace_buffer[lock_index]->_asgct_frames;
 
@@ -857,7 +857,7 @@ void Profiler::recordSample(void *ucontext, u64 counter, int tid,
     ASGCT_CallFrame *native_stop = frames + num_frames;
     num_frames += getNativeTrace(ucontext, native_stop, event_type, tid,
                                  &java_ctx, &truncated, lock_index);
-    if (num_frames < _max_stack_depth) {
+    if (likely(num_frames < _max_stack_depth)) {
       int max_remaining = _max_stack_depth - num_frames;
       if (_features.mixed) {
         int vm_start = num_frames;
@@ -885,14 +885,14 @@ void Profiler::recordSample(void *ucontext, u64 counter, int tid,
         }
       }
     }
-    if (num_frames == 0) {
+    if (unlikely(num_frames == 0)) {
       num_frames += makeFrame(frames + num_frames, BCI_ERROR, "no_Java_frame");
     }
 
     call_trace_id =
         _call_trace_storage.put(num_frames, frames, truncated, counter);
     ProfiledThread *thread = ProfiledThread::currentSignalSafe();
-    if (thread != nullptr) {
+    if (likely(thread != nullptr)) {
       thread->recordCallTraceId(call_trace_id);
     }
     u64 duration = TSC::ticks() - startTime;
@@ -907,9 +907,9 @@ void Profiler::recordSample(void *ucontext, u64 counter, int tid,
 
 void Profiler::recordWallClockEpoch(int tid, WallClockEpochEvent *event) {
   u32 lock_index = getLockIndex(tid);
-  if (!_locks[lock_index].tryLock() &&
+  if (unlikely(!_locks[lock_index].tryLock() &&
       !_locks[lock_index = (lock_index + 1) % CONCURRENCY_LEVEL].tryLock() &&
-      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock()) {
+      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock())) {
     return;
   }
   _jfr.wallClockEpoch(lock_index, event);
@@ -918,9 +918,9 @@ void Profiler::recordWallClockEpoch(int tid, WallClockEpochEvent *event) {
 
 void Profiler::recordTraceRoot(int tid, TraceRootEvent *event) {
   u32 lock_index = getLockIndex(tid);
-  if (!_locks[lock_index].tryLock() &&
+  if (unlikely(!_locks[lock_index].tryLock() &&
       !_locks[lock_index = (lock_index + 1) % CONCURRENCY_LEVEL].tryLock() &&
-      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock()) {
+      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock())) {
     return;
   }
   _jfr.recordTraceRoot(lock_index, tid, event);
@@ -929,9 +929,9 @@ void Profiler::recordTraceRoot(int tid, TraceRootEvent *event) {
 
 void Profiler::recordQueueTime(int tid, QueueTimeEvent *event) {
   u32 lock_index = getLockIndex(tid);
-  if (!_locks[lock_index].tryLock() &&
+  if (unlikely(!_locks[lock_index].tryLock() &&
       !_locks[lock_index = (lock_index + 1) % CONCURRENCY_LEVEL].tryLock() &&
-      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock()) {
+      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock())) {
     return;
   }
   _jfr.recordQueueTime(lock_index, tid, event);
@@ -955,9 +955,9 @@ void Profiler::recordExternalSample(u64 weight, int tid, int num_frames,
 
   u64 call_trace_id =
       _call_trace_storage.put(num_frames, extended_frames, truncated, weight);
-  if (!_locks[lock_index].tryLock() &&
+  if (unlikely(!_locks[lock_index].tryLock() &&
       !_locks[lock_index = (lock_index + 1) % CONCURRENCY_LEVEL].tryLock() &&
-      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock()) {
+      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock())) {
     // Too many concurrent signals already
     atomicIncRelaxed(_failures[-ticks_skipped]);
     return;
@@ -980,9 +980,9 @@ void Profiler::writeDatadogProfilerSetting(int tid, int length,
                                            const char *name, const char *value,
                                            const char *unit) {
   u32 lock_index = getLockIndex(tid);
-  if (!_locks[lock_index].tryLock() &&
+  if (unlikely(!_locks[lock_index].tryLock() &&
       !_locks[lock_index = (lock_index + 1) % CONCURRENCY_LEVEL].tryLock() &&
-      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock()) {
+      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock())) {
     return;
   }
   _jfr.recordDatadogSetting(lock_index, length, name, value, unit);
@@ -995,9 +995,9 @@ void Profiler::writeHeapUsage(long value, bool live) {
     return;
   }
   u32 lock_index = getLockIndex(tid);
-  if (!_locks[lock_index].tryLock() &&
+  if (unlikely(!_locks[lock_index].tryLock() &&
       !_locks[lock_index = (lock_index + 1) % CONCURRENCY_LEVEL].tryLock() &&
-      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock()) {
+      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock())) {
     return;
   }
   _jfr.recordHeapUsage(lock_index, value, live);

ddprof-lib/src/main/cpp/threadFilter.cpp

@@ -104,7 +104,7 @@ ThreadFilter::SlotID ThreadFilter::registerThread() {
 
     // Allocate a new slot
     SlotID index = _next_index.fetch_add(1, std::memory_order_relaxed);
-    if (index >= kMaxThreads) {
+    if (unlikely(index >= kMaxThreads)) {
         // Revert the increment and return failure
         _next_index.fetch_sub(1, std::memory_order_relaxed);
         return -1;
@@ -113,7 +113,7 @@ ThreadFilter::SlotID ThreadFilter::registerThread() {
     const int chunk_idx = index >> kChunkShift;
 
     // Ensure the chunk is initialized (lock-free)
-    if (chunk_idx >= _num_chunks.load(std::memory_order_acquire)) {
+    if (unlikely(chunk_idx >= _num_chunks.load(std::memory_order_acquire))) {
         // Update the chunk count atomically
         int expected_chunks = chunk_idx;
         int desired_chunks = chunk_idx + 1;