Skip to content

feat(filter): Introduce kernel thread callstack summary field #668

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rabbitstack merged 1 commit into from
May 12, 2026
Merged

feat(filter): Introduce kernel thread callstack summary field #668

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 23 additions & 15 deletions pkg/callstack/callstack.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,14 @@ const (
User
)

// SummaryMode defines the callstack summary mode
type SummaryMode uint8

const (
UserSummary SummaryMode = iota
KernelSummary
)

// unbacked represents the identifier for unbacked regions in stack frames
const unbacked = "unbacked"

Expand Down Expand Up @@ -260,15 +268,23 @@ func (s *Callstack) FinalKernelFrame() *Frame {
}

// Summary returns a sequence of non-repeated module names.
func (s Callstack) Summary() string {
func (s Callstack) Summary(mode SummaryMode) string {
var b strings.Builder
b.Grow(len(s) * 16) // preallocate the buffer

var prev string
var removeSep bool

for i := range s {
frame := s[len(s)-i-1]
if frame.Addr.InSystemRange() {
continue
switch mode {
case UserSummary:
if frame.Addr.InSystemRange() {
continue
}
case KernelSummary:
if !frame.Addr.InSystemRange() {
return b.String()
}
}

var n string
Expand All @@ -279,23 +295,15 @@ func (s Callstack) Summary() string {
}

if n == prev {
if i == len(s)-1 {
// last module equals to the previous
// which renders redundant separator
removeSep = true
}
continue
}

b.WriteString(n)
if i != len(s)-1 {
if b.Len() > 0 {
b.WriteRune('|')
}
prev = n
}
b.WriteString(n)

if removeSep {
return strings.TrimSuffix(b.String(), "|")
prev = n
}

return b.String()
Expand Down
55 changes: 53 additions & 2 deletions pkg/callstack/callstack_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,10 @@
package callstack

import (
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"testing"
)

func TestCallstack(t *testing.T) {
Expand All @@ -43,7 +44,8 @@ func TestCallstack(t *testing.T) {
assert.True(t, callstack.ContainsUnbacked())
assert.Equal(t, 9, callstack.Depth())
assert.Equal(t, "0xfffff8015690b644 C:\\WINDOWS\\system32\\ntoskrnl.exe!ObDeleteCapturedInsertInfo+0x45b4|0xfffff801568e9c33 C:\\WINDOWS\\system32\\ntoskrnl.exe!LpcRequestPort+0x2ef3|0xfffff8015662a605 C:\\WINDOWS\\system32\\ntoskrnl.exe!setjmpex+0x9125|0x7ffb5c1d0396 C:\\WINDOWS\\System32\\KERNELBASE.dll!CreateProcessW+0x66|0x7ffb5d8e61f4 C:\\WINDOWS\\System32\\KERNEL32.DLL!CreateProcessW+0x54|0x7ffb5c1d0396 C:\\WINDOWS\\System32\\KERNELBASE.dll!CreateProcessW+0x61|0x7ffb3138592e C:\\Program Files\\JetBrains\\GoLand 2021.2.3\\jbr\\bin\\java.dll!Java_java_lang_ProcessImpl_waitForTimeoutInterruptibly+0x3a2|0x7ffb313853b2 C:\\Program Files\\JetBrains\\GoLand 2021.2.3\\jbr\\bin\\java.dll!Java_java_lang_ProcessImpl_create+0x10a|0x2638e59e0a5 unbacked!?", callstack.String())
assert.Equal(t, "KERNELBASE.dll|KERNEL32.DLL|KERNELBASE.dll|java.dll|unbacked", callstack.Summary())
assert.Equal(t, "KERNELBASE.dll|KERNEL32.DLL|KERNELBASE.dll|java.dll|unbacked", callstack.Summary(UserSummary))
assert.Equal(t, "ntoskrnl.exe", callstack.Summary(KernelSummary))

uframe := callstack.FinalUserFrame()
require.NotNil(t, uframe)
Expand Down Expand Up @@ -122,6 +124,55 @@ func TestCallstackFinalUserFrame(t *testing.T) {
}
}

func TestCallstackSummary(t *testing.T) {
var tests = []struct {
callstack Callstack
expectedUserSummary string
expectedKernelSummary string
}{
{callstack: callstackFromFrames(
Frame{Addr: 0xf259de, Module: unbacked, Symbol: "?"},
Frame{Addr: 0x7ffe4fda6e3b, Module: "C:\\Windows\\System32\\KernelBase.dll", Symbol: "SetThreadContext"},
Frame{Addr: 0x7ffe52942b24, Module: "C:\\Windows\\System32\\ntdll.dll", Symbol: "ZwSetContextThread"},
Frame{Addr: 0xfffff807e228c555, Module: "C:\\WINDOWS\\system32\\ntoskrnl.exe", Symbol: "setjmpex"},
Frame{Addr: 0xfffff807e264805c, Module: "C:\\WINDOWS\\system32\\ntoskrnl.exe", Symbol: "ObOpenObjectByPointerWithTag"}),
expectedUserSummary: "ntdll.dll|KernelBase.dll|unbacked",
expectedKernelSummary: "ntoskrnl.exe",
},
{callstack: callstackFromFrames(
Frame{Addr: 0x7ffff0f3bf6c, Module: "C:\\Windows\\System32\\ntdll.dll", Symbol: "RtlUserThreadStart"},
Frame{Addr: 0x7ffff03ee8d7, Module: "C:\\Windows\\System32\\ntdll.dll", Symbol: "BaseThreadInitThunk"},
Frame{Addr: 0x7ffff0ee5f13, Module: "C:\\Windows\\System32\\ntdll.dll", Symbol: "TpCallbackMayRunLong"},
Frame{Addr: 0x7ffff0c78788, Module: "C:\\Windows\\System32\\rpcrt4.dll", Symbol: "RpcGetBufferWithObject"},
Frame{Addr: 0x7ffff0c797e3, Module: "C:\\Windows\\System32\\rpcrt4.dll", Symbol: "RpcImpersonateClient"},
Frame{Addr: 0x7fffee58d16a, Module: "C:\\Windows\\System32\\KernelBase.dll", Symbol: "CreateProcessInternalW"},
Frame{Addr: 0x7ffff0fe1204, Module: "C:\\Windows\\System32\\ntdll.dll", Symbol: "ZwCreateUserProcess"}),
expectedUserSummary: "ntdll.dll|KernelBase.dll|rpcrt4.dll|ntdll.dll",
expectedKernelSummary: "",
},
{callstack: callstackFromFrames(
Frame{Addr: 0x7fffa7e3bf6c, Module: "C:\\Windows\\System32\\ntdll.dll", Symbol: "RtlUserThreadStart"},
Frame{Addr: 0x7fffa60de8d7, Module: "C:\\Windows\\System32\\ntdll.dll", Symbol: "BaseThreadInitThunk"},
Frame{Addr: 0x7ff6163cfc68, Module: "C:\\Program Files\\Mozilla Firefox\\firefox.exe", Symbol: "TargetCreateThread"},
Frame{Addr: 0x7fffee58d16a, Module: "C:\\Windows\\System32\\ntdll.dll", Symbol: "ZwMapViewOfSection"},
Frame{Addr: 0xfffff8028deeed1d, Module: "C:\\WINDOWS\\system32\\ntoskrnl.exe", Symbol: "NtMapViewOfSection"},
Frame{Addr: 0xfffff8328deeed1d, Module: "C:\\WINDOWS\\system32\\drivers\\FLTMGR.SYS", Symbol: "FltCreateFileEx2"},
Frame{Addr: 0xfffff8528deeed1d, Module: "C:\\WINDOWS\\system32\\drirvers\\FLTMGR.SYS", Symbol: "FltCreateFileEx"},
Frame{Addr: 0xfffff8628deeed1d, Module: "C:\\WINDOWS\\system32\\ntoskrnl.exe", Symbol: "IofCallDriver"},
Frame{Addr: 0xfffff8728deeed1d, Module: "C:\\WINDOWS\\system32\\drivers\\fileinfo.sys", Symbol: "?"}),
expectedUserSummary: "ntdll.dll|firefox.exe|ntdll.dll",
expectedKernelSummary: "fileinfo.sys|ntoskrnl.exe|FLTMGR.SYS|ntoskrnl.exe",
},
}

for _, tt := range tests {
t.Run(tt.expectedUserSummary+"!"+tt.expectedKernelSummary, func(t *testing.T) {
assert.Equal(t, tt.expectedUserSummary, tt.callstack.Summary(UserSummary))
assert.Equal(t, tt.expectedKernelSummary, tt.callstack.Summary(KernelSummary))
})
}
}

func callstackFromFrames(frames ...Frame) Callstack {
var c Callstack
for _, frame := range frames {
Expand Down
5 changes: 4 additions & 1 deletion pkg/filter/accessor_windows.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"strings"
"time"

"github.com/rabbitstack/fibratus/pkg/callstack"
"github.com/rabbitstack/fibratus/pkg/fs"
"github.com/rabbitstack/fibratus/pkg/network"
psnap "github.com/rabbitstack/fibratus/pkg/ps"
Expand Down Expand Up @@ -527,7 +528,9 @@ func (t *threadAccessor) Get(f Field, e *event.Event) (params.Value, error) {
}
return e.GetParamAsString(params.NTStatus), nil
case fields.ThreadCallstackSummary:
return e.Callstack.Summary(), nil
return e.Callstack.Summary(callstack.UserSummary), nil
case fields.ThreadCallstackKernelSummary:
return e.Callstack.Summary(callstack.KernelSummary), nil
case fields.ThreadCallstackDetail:
return e.Callstack.String(), nil
case fields.ThreadCallstackModules:
Expand Down
3 changes: 3 additions & 0 deletions pkg/filter/fields/fields_windows.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,8 @@ const (
ThreadAccessStatus Field = "thread.access.status"
// ThreadCallstackSummary represents the thread callstack summary field
ThreadCallstackSummary Field = "thread.callstack.summary"
// ThreadCallstackKernelSummary represents the kernel thread callstack summary field
ThreadCallstackKernelSummary Field = "thread.callstack.kernel_summary"
// ThreadCallstackDetail represents the thread callstack detail field
ThreadCallstackDetail Field = "thread.callstack.detail"
// ThreadCallstackModules represents the callstack modules field
Expand Down Expand Up @@ -1089,6 +1091,7 @@ var fields = map[Field]FieldInfo{
ThreadAccessMaskNames: {ThreadAccessMaskNames, "thread desired access rights as a string list", params.Slice, []string{"thread.access.mask.names in ('IMPERSONATE')"}, nil, nil},
ThreadAccessStatus: {ThreadAccessStatus, "thread access status", params.UnicodeString, []string{"thread.access.status = 'success'"}, nil, nil},
ThreadCallstackSummary: {ThreadCallstackSummary, "callstack summary", params.UnicodeString, []string{"thread.callstack.summary contains 'ntdll.dll|KERNELBASE.dll'"}, nil, nil},
ThreadCallstackKernelSummary: {ThreadCallstackKernelSummary, "kernel thread callstack summary", params.UnicodeString, []string{"thread.callstack.kernel_summary contains 'fileinfo.sys|ntoskrnl.exe|FLTMGR.SYS|ntoskrnl.exe'"}, nil, nil},
ThreadCallstackDetail: {ThreadCallstackDetail, "detailed information of each stack frame", params.UnicodeString, []string{"thread.callstack.detail contains 'KERNELBASE.dll!CreateProcessW'"}, nil, nil},
ThreadCallstackModules: {ThreadCallstackModules, "list of modules comprising the callstack", params.Slice, []string{"thread.callstack.modules in ('C:\\WINDOWS\\System32\\KERNELBASE.dll')", "base(thread.callstack.modules[7]) = 'ntdll.dll'"}, nil, &Argument{Optional: true, Pattern: "[0-9]+", ValidationFunc: isNumber}},
ThreadCallstackSymbols: {ThreadCallstackSymbols, "list of symbols comprising the callstack", params.Slice, []string{"thread.callstack.symbols in ('ntdll.dll!NtCreateProcess')", "thread.callstack.symbols[3] = 'ntdll!NtCreateProcess'"}, nil, &Argument{Optional: true, Pattern: "[0-9]+", ValidationFunc: isNumber}},
Expand Down
14 changes: 8 additions & 6 deletions pkg/filter/filter_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -460,7 +460,7 @@ func TestThreadFilter(t *testing.T) {
}
require.NoError(t, windows.WriteProcessMemory(windows.CurrentProcess(), base, &insns[0], uintptr(len(insns)), nil))

evt.Callstack.Init(8)
evt.Callstack.Init(9)
evt.Callstack.PushFrame(callstack.Frame{PID: evt.PID, Addr: 0x2638e59e0a5, Offset: 0, Symbol: "?", Module: "unbacked"})
evt.Callstack.PushFrame(callstack.Frame{PID: evt.PID, Addr: va.Address(base), Offset: 0, Symbol: "?", Module: "unbacked"})
evt.Callstack.PushFrame(callstack.Frame{PID: evt.PID, Addr: 0x7ffb313853b2, Offset: 0x10a, Symbol: "Java_java_lang_ProcessImpl_create", Module: "C:\\Program Files\\JetBrains\\GoLand 2021.2.3\\jbr\\bin\\java.dll"})
Expand All @@ -469,6 +469,7 @@ func TestThreadFilter(t *testing.T) {
evt.Callstack.PushFrame(callstack.Frame{PID: evt.PID, Addr: 0x7ffb5c1d0396, Offset: 0x66, Symbol: "CreateProcessW", Module: "C:\\WINDOWS\\System32\\KERNELBASE.dll"})
evt.Callstack.PushFrame(callstack.Frame{PID: evt.PID, Addr: 0xfffff8072ebc1f6f, Offset: 0x4ef, Symbol: "FltRequestFileInfoOnCreateCompletion", Module: "C:\\WINDOWS\\System32\\drivers\\FLTMGR.SYS"})
evt.Callstack.PushFrame(callstack.Frame{PID: evt.PID, Addr: 0xfffff8072eb8961b, Offset: 0x20cb, Symbol: "FltGetStreamContext", Module: "C:\\WINDOWS\\System32\\drivers\\FLTMGR.SYS"})
evt.Callstack.PushFrame(callstack.Frame{PID: evt.PID, Addr: 0xfffff8172eb8961b, Offset: 0x20cb, Symbol: "IofCallDriver", Module: "C:\\WINDOWS\\system32\\ntoskrnl.exe"})

var tests = []struct {
filter string
Expand All @@ -484,14 +485,15 @@ func TestThreadFilter(t *testing.T) {
{`thread.start_address.symbol = 'LoadModule'`, true},
{`thread.start_address.module = 'C:\\Windows\\System32\\kernel32.dll'`, true},
{`thread.callstack.summary = 'KERNELBASE.dll|KERNEL32.DLL|java.dll|unbacked'`, true},
{`thread.callstack.kernel_summary = 'ntoskrnl.exe|FLTMGR.SYS'`, true},
{`thread.callstack.detail icontains 'C:\\WINDOWS\\System32\\KERNELBASE.dll!CreateProcessW+0x66'`, true},
{`thread.callstack.modules in ('C:\\WINDOWS\\System32\\KERNELBASE.dll', 'C:\\Program Files\\JetBrains\\GoLand 2021.2.3\\jbr\\bin\\java.dll')`, true},
{`thread.callstack.modules[5] = 'C:\\WINDOWS\\System32\\KERNELBASE.dll'`, true},
{`thread.callstack.modules[7] = 'C:\\WINDOWS\\System32\\drivers\\FLTMGR.SYS'`, true},
{`thread.callstack.modules[8] = ''`, true},
{`thread.callstack.modules[10] = ''`, true},
{`thread.callstack.symbols imatches ('KERNELBASE.dll!CreateProcess*', 'Java_java_lang_ProcessImpl_create')`, true},
{`thread.callstack.symbols[2] = 'Java_java_lang_ProcessImpl_create'`, true},
{`thread.callstack.symbols[8] = ''`, true},
{`thread.callstack.symbols[10] = ''`, true},
{`thread.callstack.protections in ('RWX')`, true},
{`thread.callstack.allocation_sizes > 0`, false},
{`length(thread.callstack.callsite_leading_assembly) > 0`, true},
Expand All @@ -501,9 +503,9 @@ func TestThreadFilter(t *testing.T) {
{`thread.callstack.final_user_module.name = 'java.dll'`, true},
{`thread.callstack.final_user_module.path = 'C:\\Program Files\\JetBrains\\GoLand 2021.2.3\\jbr\\bin\\java.dll'`, true},
{`thread.callstack.final_user_symbol.name = 'Java_java_lang_ProcessImpl_waitForTimeoutInterruptibly'`, true},
{`thread.callstack.final_kernel_module.name = 'FLTMGR.SYS'`, true},
{`thread.callstack.final_kernel_module.path = 'C:\\WINDOWS\\System32\\drivers\\FLTMGR.SYS'`, true},
{`thread.callstack.final_kernel_symbol.name = 'FltGetStreamContext'`, true},
{`thread.callstack.final_kernel_module.name = 'ntoskrnl.exe'`, true},
{`thread.callstack.final_kernel_module.path = 'C:\\WINDOWS\\system32\\ntoskrnl.exe'`, true},
{`thread.callstack.final_kernel_symbol.name = 'IofCallDriver'`, true},
{`thread.callstack.final_user_module.signature.exists = true`, true},
{`thread.callstack.final_user_module.signature.trusted = true`, true},
{`thread.callstack.final_user_module.signature.issuer imatches '*Microsoft Corporation*'`, true},
Expand Down
7 changes: 0 additions & 7 deletions pkg/symbolize/symbolizer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -305,13 +305,6 @@ func (s *Symbolizer) processCallstack(e *event.Event) error {
addrs := e.Params.MustGetSliceAddrs(params.Callstack)
e.Callstack.Init(len(addrs))

// skip stack enrichment for the events generated by the System process
// except the LoadModule event which may prove to be useful when the
// driver is loaded and the kernel address symbolization is enabled
if e.IsSystemPid() && !e.IsLoadModule() {
return nil
}

s.mu.Lock()
defer s.mu.Unlock()

Expand Down
3 changes: 2 additions & 1 deletion pkg/symbolize/symbolizer_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"testing"
"time"

"github.com/rabbitstack/fibratus/pkg/callstack"
"github.com/rabbitstack/fibratus/pkg/config"
"github.com/rabbitstack/fibratus/pkg/event"
"github.com/rabbitstack/fibratus/pkg/event/params"
Expand Down Expand Up @@ -187,7 +188,7 @@ func TestProcessCallstackPeExports(t *testing.T) {
assert.Equal(t, "user32.dll!LoadKeyboardLayoutW", e.Callstack.Symbols()[3])
assert.Equal(t, "kernel32.dll!?", e.Callstack.Symbols()[4]) // unexported symbol

assert.Equal(t, "kernel32.dll|user32.dll|ntdll.dll|unbacked", e.Callstack.Summary())
assert.Equal(t, "kernel32.dll|user32.dll|ntdll.dll|unbacked", e.Callstack.Summary(callstack.UserSummary))
assert.True(t, e.Callstack.ContainsUnbacked())

// check internal state
Expand Down
Loading