From 30f8c03b629afb9788fbb12f2bd6627c1d1e273d Mon Sep 17 00:00:00 2001 From: Dejan Zele Pejchev Date: Mon, 7 Sep 2026 11:54:25 +0200 Subject: [PATCH 1/7] feat(runner): add live-log streaming metrics --- pkg/controlplaneclient/metrics.go | 46 ++++++++ pkg/controlplaneclient/metrics_test.go | 117 +++++++++++++++++++ pkg/controlplaneclient/runner.go | 6 +- pkg/controlplaneclient/testworkflows_test.go | 16 ++- pkg/controlplaneclient/utils.go | 106 ++++++++++++++--- 5 files changed, 272 insertions(+), 19 deletions(-) create mode 100644 pkg/controlplaneclient/metrics.go create mode 100644 pkg/controlplaneclient/metrics_test.go diff --git a/pkg/controlplaneclient/metrics.go b/pkg/controlplaneclient/metrics.go new file mode 100644 index 00000000000..56adb0b7e8a --- /dev/null +++ b/pkg/controlplaneclient/metrics.go @@ -0,0 +1,46 @@ +package controlplaneclient + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +// Live-log streaming metrics. Registered on the default registry via promauto, +// which is what pkg/server/httpserver.go serves at /metrics. +var ( + liveLogSessions = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "testkube_live_log_sessions", + Help: "Current number of live-log streaming sessions by state", + }, []string{"kind", "state"}) + + liveLogReplayBytes = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "testkube_live_log_replay_bytes", + Help: "Approximate bytes held in live-log replay buffers", + }, []string{"kind"}) + + liveLogResumeTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "testkube_live_log_resume_total", + Help: "Total live-log resume attempts by result", + }, []string{"kind", "result"}) + + liveLogSessionsCreatedTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "testkube_live_log_sessions_created_total", + Help: "Total live-log streaming sessions created", + }, []string{"kind"}) + + liveLogSessionsEvictedTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "testkube_live_log_sessions_evicted_total", + Help: "Total live-log streaming sessions evicted by reason", + }, []string{"kind", "reason"}) + + liveLogSubscribers = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "testkube_live_log_subscribers", + Help: "Current number of live-log stream subscribers", + }, []string{"kind"}) + + liveLogSourceDurationSeconds = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "testkube_live_log_source_duration_seconds", + Help: "Duration of live-log source lifetimes in seconds", + Buckets: prometheus.DefBuckets, + }, []string{"kind"}) +) diff --git a/pkg/controlplaneclient/metrics_test.go b/pkg/controlplaneclient/metrics_test.go new file mode 100644 index 00000000000..825e656da34 --- /dev/null +++ b/pkg/controlplaneclient/metrics_test.go @@ -0,0 +1,117 @@ +package controlplaneclient + +import ( + "context" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/require" + + "github.com/kubeshop/testkube/pkg/api/v1/testkube" + "github.com/kubeshop/testkube/pkg/cloud" + "github.com/kubeshop/testkube/pkg/repository/channels" +) + +func TestLiveLogMetricsMoveOnAttachAndPublish(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + const kind = "metrics-unit-test" + + sourceReady := make(chan channels.WritableWatcher[*testkube.TestWorkflowExecutionNotification], 1) + manager := newNotificationStreamSessionManager( + ctx, + kind, + func(req *cloud.TestWorkflowNotificationsRequest) string { + return req.ExecutionId + }, + func(context.Context, *cloud.TestWorkflowNotificationsRequest) NotificationWatcher { + watcher := channels.NewWatcher[*testkube.TestWorkflowExecutionNotification]() + sourceReady <- watcher + return watcher + }, + ) + + createdBefore := testutil.ToFloat64(liveLogSessionsCreatedTotal.WithLabelValues(kind)) + + session, sub, _, _, _, _ := manager.attach(&cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1"}) + + require.Equal(t, createdBefore+1, testutil.ToFloat64(liveLogSessionsCreatedTotal.WithLabelValues(kind))) + require.Equal(t, float64(1), testutil.ToFloat64(liveLogSessions.WithLabelValues(kind, "active"))) + require.Equal(t, float64(1), testutil.ToFloat64(liveLogSubscribers.WithLabelValues(kind))) + + // A resumable notification adds to the replay buffer and the memory gauge. + session.publish(&testkube.TestWorkflowExecutionNotification{Log: "hello"}) + require.Greater(t, testutil.ToFloat64(liveLogReplayBytes.WithLabelValues(kind)), float64(0)) + + // Resuming the live session within replay range is available and counted. + availableBefore := testutil.ToFloat64(liveLogResumeTotal.WithLabelValues(kind, "available")) + _, sub2, _, available, _, _ := manager.attach(&cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1", ResumeAfterSeqNo: 1}) + require.True(t, available) + require.Equal(t, availableBefore+1, testutil.ToFloat64(liveLogResumeTotal.WithLabelValues(kind, "available"))) + + watcher := <-sourceReady + watcher.Close(nil) + + // The source ends: active drops, done rises, and a duration sample is recorded. + require.Eventually(t, func() bool { + return testutil.ToFloat64(liveLogSessions.WithLabelValues(kind, "active")) == 0 && + testutil.ToFloat64(liveLogSessions.WithLabelValues(kind, "done")) == 1 + }, 2*time.Second, 5*time.Millisecond) + require.GreaterOrEqual(t, testutil.CollectAndCount(liveLogSourceDurationSeconds), 1) + + manager.detach(session, sub) + manager.detach(session, sub2) + require.Equal(t, float64(0), testutil.ToFloat64(liveLogSubscribers.WithLabelValues(kind))) +} + +// Two managers of the same kind can overlap: on gRPC reconnect a fresh manager +// is built while the previous one's sessions keep draining bytes via stale TTL +// timers. The replay_bytes gauge must reflect the current live total across +// both, not be clobbered to a single manager's draining value. +func TestLiveLogReplayBytesAdditiveAcrossManagers(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + const kind = "metrics-overlap-test" + + newManager := func() *notificationStreamSessionManager[*cloud.TestWorkflowNotificationsRequest] { + return newNotificationStreamSessionManager( + ctx, + kind, + func(req *cloud.TestWorkflowNotificationsRequest) string { return req.ExecutionId }, + func(context.Context, *cloud.TestWorkflowNotificationsRequest) NotificationWatcher { + return channels.NewWatcher[*testkube.TestWorkflowExecutionNotification]() + }, + ) + } + + require.Equal(t, float64(0), testutil.ToFloat64(liveLogReplayBytes.WithLabelValues(kind))) + + // Old manager buffers some bytes. + oldManager := newManager() + oldSession := newNotificationStreamSession(oldManager.addReplayBytes) + oldSession.publish(&testkube.TestWorkflowExecutionNotification{Log: "old-manager-payload"}) + oldBytes := testutil.ToFloat64(liveLogReplayBytes.WithLabelValues(kind)) + require.Greater(t, oldBytes, float64(0)) + + // A reconnect builds a fresh manager whose session buffers more bytes. + newMgr := newManager() + newSession := newNotificationStreamSession(newMgr.addReplayBytes) + newSession.publish(&testkube.TestWorkflowExecutionNotification{Log: "new-manager-payload"}) + combined := testutil.ToFloat64(liveLogReplayBytes.WithLabelValues(kind)) + require.Greater(t, combined, oldBytes) + + newBytes := combined - oldBytes + + // The old manager's stale timer releases its session's bytes. The gauge must + // drop by exactly the old contribution and still hold the new manager's live + // total, not be reset to the old manager's (now zero) draining value. + oldSession.releaseReplayBytes() + require.Equal(t, newBytes, testutil.ToFloat64(liveLogReplayBytes.WithLabelValues(kind))) + + // Once the new manager's session drops too, the gauge nets back to zero. + newSession.releaseReplayBytes() + require.Equal(t, float64(0), testutil.ToFloat64(liveLogReplayBytes.WithLabelValues(kind))) +} diff --git a/pkg/controlplaneclient/runner.go b/pkg/controlplaneclient/runner.go index 129cdae02c6..b699b746720 100644 --- a/pkg/controlplaneclient/runner.go +++ b/pkg/controlplaneclient/runner.go @@ -197,7 +197,7 @@ func (c *client) ProcessExecutionNotificationRequests(ctx context.Context, proce // the agent lifetime today, but rebuilding it on a dead context avoids silently // running with sources and a sweeper that can never start. if c.workflowNotifManager == nil || c.workflowNotifManager.ctx.Err() != nil { - c.workflowNotifManager = newNotificationStreamSessionManager(ctx, workflowNotificationSessionKey, process) + c.workflowNotifManager = newNotificationStreamSessionManager(ctx, "workflow", workflowNotificationSessionKey, process) } manager := c.workflowNotifManager c.notifMu.Unlock() @@ -219,7 +219,7 @@ func (c *client) ProcessExecutionNotificationRequests(ctx context.Context, proce func (c *client) ProcessExecutionParallelWorkerNotificationRequests(ctx context.Context, process func(ctx context.Context, req *cloud.TestWorkflowParallelStepNotificationsRequest) NotificationWatcher) error { c.notifMu.Lock() if c.parallelNotifManager == nil || c.parallelNotifManager.ctx.Err() != nil { - c.parallelNotifManager = newNotificationStreamSessionManager(ctx, parallelWorkerNotificationSessionKey, process) + c.parallelNotifManager = newNotificationStreamSessionManager(ctx, "parallel", parallelWorkerNotificationSessionKey, process) } manager := c.parallelNotifManager c.notifMu.Unlock() @@ -241,7 +241,7 @@ func (c *client) ProcessExecutionParallelWorkerNotificationRequests(ctx context. func (c *client) ProcessExecutionServiceNotificationRequests(ctx context.Context, process func(ctx context.Context, req *cloud.TestWorkflowServiceNotificationsRequest) NotificationWatcher) error { c.notifMu.Lock() if c.serviceNotifManager == nil || c.serviceNotifManager.ctx.Err() != nil { - c.serviceNotifManager = newNotificationStreamSessionManager(ctx, serviceNotificationSessionKey, process) + c.serviceNotifManager = newNotificationStreamSessionManager(ctx, "service", serviceNotificationSessionKey, process) } manager := c.serviceNotifManager c.notifMu.Unlock() diff --git a/pkg/controlplaneclient/testworkflows_test.go b/pkg/controlplaneclient/testworkflows_test.go index 2e5ed2dea6b..898d6aa6cb5 100644 --- a/pkg/controlplaneclient/testworkflows_test.go +++ b/pkg/controlplaneclient/testworkflows_test.go @@ -79,6 +79,7 @@ func TestNotificationStreamSessionManagerReplaysAfterCursor(t *testing.T) { manager := newNotificationStreamSessionManager( ctx, + "workflow", func(req *cloud.TestWorkflowNotificationsRequest) string { return req.ExecutionId }, @@ -120,6 +121,7 @@ func TestNotificationStreamSessionSurvivesReaderDropAndResumesLive(t *testing.T) emit := make(chan string) manager := newNotificationStreamSessionManager( ctx, + "workflow", func(req *cloud.TestWorkflowNotificationsRequest) string { return req.ExecutionId }, @@ -172,7 +174,7 @@ func TestSendNotificationResponseReturnsContextErrorWhenCanceled(t *testing.T) { } func TestNotificationStreamSessionPublishDoesNotHoldLockForSlowSubscriber(t *testing.T) { - session := newNotificationStreamSession() + session := newNotificationStreamSession(nil) sub, _, _, _, _ := session.subscribe(0, 1) for i := 0; i < cap(sub.ch); i++ { sub.ch <- notificationStreamEvent{} @@ -208,7 +210,7 @@ func TestNotificationStreamSessionPublishDoesNotHoldLockForSlowSubscriber(t *tes } func TestWorkflowProtocolEventsDoNotAdvanceApplicationSeqNo(t *testing.T) { - session := newNotificationStreamSession() + session := newNotificationStreamSession(nil) ready := buildCloudProtocol("stream-1", session.currentSeqNo(), cloud.TestWorkflowNotificationType_WORKFLOW_STREAM_READY, "") require.Equal(t, uint32(0), ready.SeqNo) @@ -223,7 +225,7 @@ func TestWorkflowProtocolEventsDoNotAdvanceApplicationSeqNo(t *testing.T) { } func TestNotificationStreamSessionReplayUnavailableForTrimmedCursor(t *testing.T) { - session := newNotificationStreamSession() + session := newNotificationStreamSession(nil) for i := 0; i < workflowNotificationReplayMaxEvents+2; i++ { session.publish(&testkube.TestWorkflowExecutionNotification{Log: "log"}) } @@ -246,6 +248,7 @@ func TestNotificationStreamSessionManagerStartsFreshAfterDoneSessionWithoutResum var processCalls atomic.Int32 manager := newNotificationStreamSessionManager( ctx, + "workflow", func(req *cloud.TestWorkflowNotificationsRequest) string { return req.ExecutionId }, @@ -282,6 +285,7 @@ func TestNotificationStreamSessionManagerStartsFreshAfterErroredDoneSessionWithR var processCalls atomic.Int32 manager := newNotificationStreamSessionManager( ctx, + "workflow", func(req *cloud.TestWorkflowNotificationsRequest) string { return req.ExecutionId }, @@ -340,6 +344,7 @@ func TestNotificationStreamSessionManagerFreshResumeStartsFromLiveTail(t *testin release := make(chan struct{}) manager := newNotificationStreamSessionManager( ctx, + "workflow", func(req *cloud.TestWorkflowNotificationsRequest) string { return req.ExecutionId }, @@ -385,6 +390,7 @@ func TestNotificationStreamSessionManagerMarksResumeUnavailableForFreshSessionWi release := make(chan struct{}) manager := newNotificationStreamSessionManager( ctx, + "workflow", func(req *cloud.TestWorkflowNotificationsRequest) string { return req.ExecutionId }, @@ -417,6 +423,7 @@ func TestNotificationStreamSessionManagerStartsFreshForConcurrentViewersWithoutR var processCalls atomic.Int32 manager := newNotificationStreamSessionManager( ctx, + "workflow", func(req *cloud.TestWorkflowNotificationsRequest) string { return req.ExecutionId }, @@ -468,6 +475,7 @@ func TestNotificationStreamSessionManagerExpiresDoneSessionsWithoutAttach(t *tes release := make(chan struct{}) manager := newNotificationStreamSessionManager( ctx, + "workflow", func(req *cloud.TestWorkflowNotificationsRequest) string { return req.ExecutionId }, @@ -502,6 +510,7 @@ func TestNotificationStreamSessionManagerSweepExpiredRemovesDonePastTTLSession(t release := make(chan struct{}) manager := newNotificationStreamSessionManager( ctx, + "workflow", func(req *cloud.TestWorkflowNotificationsRequest) string { return req.ExecutionId }, @@ -597,6 +606,7 @@ func TestNotificationStreamSessionReplacementCancelsOrphanedSource(t *testing.T) manager := newNotificationStreamSessionManager( ctx, + "workflow", func(req *cloud.TestWorkflowNotificationsRequest) string { return req.ExecutionId }, func(sourceCtx context.Context, _ *cloud.TestWorkflowNotificationsRequest) NotificationWatcher { sourceCtxs <- sourceCtx diff --git a/pkg/controlplaneclient/utils.go b/pkg/controlplaneclient/utils.go index ec86db2d672..ef4e6a32f45 100644 --- a/pkg/controlplaneclient/utils.go +++ b/pkg/controlplaneclient/utils.go @@ -130,16 +130,36 @@ type notificationStreamSession struct { errored bool lastSeqNo uint32 lastActive time.Time + // onReplayBytes reports the signed change in this session's buffered bytes. + // Called under mu so a session's deltas are serialized. + onReplayBytes func(delta int) } -func newNotificationStreamSession() *notificationStreamSession { +func newNotificationStreamSession(onReplayBytes func(delta int)) *notificationStreamSession { return ¬ificationStreamSession{ - nextSeqNo: 1, - subscribers: make(map[uint64]*notificationStreamSubscription), - lastActive: time.Now(), + nextSeqNo: 1, + subscribers: make(map[uint64]*notificationStreamSubscription), + lastActive: time.Now(), + onReplayBytes: onReplayBytes, } } +func (s *notificationStreamSession) reportReplayBytes(delta int) { + if delta != 0 && s.onReplayBytes != nil { + s.onReplayBytes(delta) + } +} + +// releaseReplayBytes drops the session's remaining buffered bytes from the +// accounting, once, when the session is removed from its manager. +func (s *notificationStreamSession) releaseReplayBytes() { + s.mu.Lock() + remaining := s.replayBytes + s.replayBytes = 0 + s.reportReplayBytes(-remaining) + s.mu.Unlock() +} + func (s *notificationStreamSession) subscribe(resumeAfterSeqNo uint32, subscriptionID uint64) (*notificationStreamSubscription, []notificationStreamEvent, bool, uint32, bool) { s.mu.Lock() defer s.mu.Unlock() @@ -185,11 +205,16 @@ func (s *notificationStreamSession) publish(notification *testkube.TestWorkflowE seqNo: seqNo, notification: notification, } + bytesDelta := 0 if seqNo > 0 { + added := approximateNotificationBytes(notification) s.replay = append(s.replay, event) - s.replayBytes += approximateNotificationBytes(notification) + s.replayBytes += added + bytesDelta += added for len(s.replay) > workflowNotificationReplayMaxEvents || s.replayBytes > workflowNotificationReplayMaxBytes { - s.replayBytes -= approximateNotificationBytes(s.replay[0].notification) + evicted := approximateNotificationBytes(s.replay[0].notification) + s.replayBytes -= evicted + bytesDelta -= evicted s.replay[0].notification = nil s.replay = s.replay[1:] } @@ -198,6 +223,7 @@ func (s *notificationStreamSession) publish(notification *testkube.TestWorkflowE for _, sub := range s.subscribers { subscribers = append(subscribers, sub) } + s.reportReplayBytes(bytesDelta) s.mu.Unlock() for _, sub := range subscribers { @@ -304,12 +330,14 @@ type notificationStreamSessionManager[Request notificationRequest] struct { nextID atomic.Uint64 sessions map[string]*notificationStreamSession sessionIdleTTL time.Duration + kind string key func(Request) string process func(ctx context.Context, req Request) NotificationWatcher } func newNotificationStreamSessionManager[Request notificationRequest]( ctx context.Context, + kind string, key func(Request) string, process func(ctx context.Context, req Request) NotificationWatcher, ) *notificationStreamSessionManager[Request] { @@ -317,6 +345,7 @@ func newNotificationStreamSessionManager[Request notificationRequest]( ctx: ctx, sessions: make(map[string]*notificationStreamSession), sessionIdleTTL: workflowNotificationSessionIdleTTL, + kind: kind, key: key, process: process, } @@ -341,6 +370,17 @@ func (m *notificationStreamSessionManager[Request]) runSweeper(interval time.Dur } } +// addReplayBytes applies a signed byte delta to the gauge. Adds are matched by +// their own release deltas, so overlapping managers of the same kind (e.g. a +// new manager built on gRPC reconnect while an old one's sessions still drain) +// sum correctly and the gauge nets back to zero once every session drops. +func (m *notificationStreamSessionManager[Request]) addReplayBytes(delta int) { + if delta == 0 { + return + } + liveLogReplayBytes.WithLabelValues(m.kind).Add(float64(delta)) +} + func (m *notificationStreamSessionManager[Request]) sessionKey(req Request) string { key := m.key(req) if req.GetStreamId() == "" { @@ -351,13 +391,20 @@ func (m *notificationStreamSessionManager[Request]) sessionKey(req Request) stri // sweepExpired removes every session that is done and past the idle TTL as of now. func (m *notificationStreamSessionManager[Request]) sweepExpired(now time.Time) { + var releaseSessions []*notificationStreamSession m.mu.Lock() - defer m.mu.Unlock() for sessionKey, session := range m.sessions { if session.expired(now, m.sessionIdleTTL) { delete(m.sessions, sessionKey) + releaseSessions = append(releaseSessions, session) + liveLogSessionsEvictedTotal.WithLabelValues(m.kind, "ttl").Inc() + liveLogSessions.WithLabelValues(m.kind, "done").Dec() } } + m.mu.Unlock() + for _, s := range releaseSessions { + s.releaseReplayBytes() + } } func (m *notificationStreamSessionManager[Request]) attach(req Request) (*notificationStreamSession, *notificationStreamSubscription, []notificationStreamEvent, bool, uint32, bool) { @@ -372,6 +419,7 @@ func (m *notificationStreamSessionManager[Request]) attach(req Request) (*notifi // stopped: once replaced in the map it is unreachable and would otherwise run // orphaned until the execution ends. var replaced *notificationStreamSession + var releaseSessions []*notificationStreamSession session := m.sessions[key] if req.GetResumeAfterSeqNo() == 0 { replaced = session @@ -380,16 +428,20 @@ func (m *notificationStreamSessionManager[Request]) attach(req Request) (*notifi done, errored := session.status() if done && errored { delete(m.sessions, key) + releaseSessions = append(releaseSessions, session) + liveLogSessions.WithLabelValues(m.kind, "done").Dec() session = nil } } freshSession := false var sourceCtx context.Context if session == nil { - session = newNotificationStreamSession() + session = newNotificationStreamSession(m.addReplayBytes) sourceCtx, session.cancel = context.WithCancel(m.ctx) m.sessions[key] = session freshSession = true + liveLogSessionsCreatedTotal.WithLabelValues(m.kind).Inc() + liveLogSessions.WithLabelValues(m.kind, "active").Inc() } subscriptionID := m.nextID.Add(1) m.mu.Unlock() @@ -397,6 +449,9 @@ func (m *notificationStreamSessionManager[Request]) attach(req Request) (*notifi if replaced != nil { replaced.stopSource() } + for _, s := range releaseSessions { + s.releaseReplayBytes() + } subscribeAfterSeqNo := req.GetResumeAfterSeqNo() if freshSession && req.GetResumeAfterSeqNo() > 0 { @@ -413,20 +468,38 @@ func (m *notificationStreamSessionManager[Request]) attach(req Request) (*notifi available = false } } + if req.GetResumeAfterSeqNo() > 0 { + result := "unavailable" + if available { + result = "available" + } + liveLogResumeTotal.WithLabelValues(m.kind, result).Inc() + } + liveLogSubscribers.WithLabelValues(m.kind).Inc() return session, sub, replay, available, lastSeqNo, done } +func (m *notificationStreamSessionManager[Request]) detach(session *notificationStreamSession, sub *notificationStreamSubscription) { + session.unsubscribe(sub) + liveLogSubscribers.WithLabelValues(m.kind).Dec() +} + func (m *notificationStreamSessionManager[Request]) scheduleExpiration(key string, session *notificationStreamSession) { time.AfterFunc(m.sessionIdleTTL, func() { m.mu.Lock() - defer m.mu.Unlock() - if m.sessions[key] != session { + m.mu.Unlock() return } - if session.expired(time.Now(), m.sessionIdleTTL) { - delete(m.sessions, key) + if !session.expired(time.Now(), m.sessionIdleTTL) { + m.mu.Unlock() + return } + delete(m.sessions, key) + liveLogSessionsEvictedTotal.WithLabelValues(m.kind, "ttl").Inc() + liveLogSessions.WithLabelValues(m.kind, "done").Dec() + m.mu.Unlock() + session.releaseReplayBytes() }) } @@ -441,10 +514,17 @@ func shouldPublishLiveResumeNotification(notification *testkube.TestWorkflowExec } func (m *notificationStreamSessionManager[Request]) runSource(ctx context.Context, key string, session *notificationStreamSession, req Request, liveOnlyAfter time.Time) { + started := time.Now() var sourceErr error defer func() { session.close(sourceErr != nil) session.stopSource() + liveLogSessions.WithLabelValues(m.kind, "active").Dec() + liveLogSessions.WithLabelValues(m.kind, "done").Inc() + liveLogSourceDurationSeconds.WithLabelValues(m.kind).Observe(time.Since(started).Seconds()) + if sourceErr != nil { + liveLogSessionsEvictedTotal.WithLabelValues(m.kind, "error").Inc() + } m.scheduleExpiration(key, session) }() @@ -601,7 +681,7 @@ func processNotifications[Request notificationRequest, Response any, Srv notific }() session, sub, replay, resumeAvailable, lastSeqNo, done := sessionManager.attach(req) - defer session.unsubscribe(sub) + defer sessionManager.detach(session, sub) // READY means the agent accepted this request and attached it to a logical stream session. // It does not imply the Kubernetes log source has already produced data. From 11b44397e2ea37ddddf640b054e948c6d389b4f3 Mon Sep 17 00:00:00 2001 From: Dejan Zele Pejchev Date: Mon, 7 Sep 2026 12:12:54 +0200 Subject: [PATCH 2/7] test(runner): move live-log metrics tests next to the session manager tests --- pkg/controlplaneclient/metrics_test.go | 117 -------- pkg/controlplaneclient/testworkflows_test.go | 297 +++++++++++++++++++ pkg/controlplaneclient/utils.go | 8 +- 3 files changed, 301 insertions(+), 121 deletions(-) delete mode 100644 pkg/controlplaneclient/metrics_test.go diff --git a/pkg/controlplaneclient/metrics_test.go b/pkg/controlplaneclient/metrics_test.go deleted file mode 100644 index 825e656da34..00000000000 --- a/pkg/controlplaneclient/metrics_test.go +++ /dev/null @@ -1,117 +0,0 @@ -package controlplaneclient - -import ( - "context" - "testing" - "time" - - "github.com/prometheus/client_golang/prometheus/testutil" - "github.com/stretchr/testify/require" - - "github.com/kubeshop/testkube/pkg/api/v1/testkube" - "github.com/kubeshop/testkube/pkg/cloud" - "github.com/kubeshop/testkube/pkg/repository/channels" -) - -func TestLiveLogMetricsMoveOnAttachAndPublish(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - - const kind = "metrics-unit-test" - - sourceReady := make(chan channels.WritableWatcher[*testkube.TestWorkflowExecutionNotification], 1) - manager := newNotificationStreamSessionManager( - ctx, - kind, - func(req *cloud.TestWorkflowNotificationsRequest) string { - return req.ExecutionId - }, - func(context.Context, *cloud.TestWorkflowNotificationsRequest) NotificationWatcher { - watcher := channels.NewWatcher[*testkube.TestWorkflowExecutionNotification]() - sourceReady <- watcher - return watcher - }, - ) - - createdBefore := testutil.ToFloat64(liveLogSessionsCreatedTotal.WithLabelValues(kind)) - - session, sub, _, _, _, _ := manager.attach(&cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1"}) - - require.Equal(t, createdBefore+1, testutil.ToFloat64(liveLogSessionsCreatedTotal.WithLabelValues(kind))) - require.Equal(t, float64(1), testutil.ToFloat64(liveLogSessions.WithLabelValues(kind, "active"))) - require.Equal(t, float64(1), testutil.ToFloat64(liveLogSubscribers.WithLabelValues(kind))) - - // A resumable notification adds to the replay buffer and the memory gauge. - session.publish(&testkube.TestWorkflowExecutionNotification{Log: "hello"}) - require.Greater(t, testutil.ToFloat64(liveLogReplayBytes.WithLabelValues(kind)), float64(0)) - - // Resuming the live session within replay range is available and counted. - availableBefore := testutil.ToFloat64(liveLogResumeTotal.WithLabelValues(kind, "available")) - _, sub2, _, available, _, _ := manager.attach(&cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1", ResumeAfterSeqNo: 1}) - require.True(t, available) - require.Equal(t, availableBefore+1, testutil.ToFloat64(liveLogResumeTotal.WithLabelValues(kind, "available"))) - - watcher := <-sourceReady - watcher.Close(nil) - - // The source ends: active drops, done rises, and a duration sample is recorded. - require.Eventually(t, func() bool { - return testutil.ToFloat64(liveLogSessions.WithLabelValues(kind, "active")) == 0 && - testutil.ToFloat64(liveLogSessions.WithLabelValues(kind, "done")) == 1 - }, 2*time.Second, 5*time.Millisecond) - require.GreaterOrEqual(t, testutil.CollectAndCount(liveLogSourceDurationSeconds), 1) - - manager.detach(session, sub) - manager.detach(session, sub2) - require.Equal(t, float64(0), testutil.ToFloat64(liveLogSubscribers.WithLabelValues(kind))) -} - -// Two managers of the same kind can overlap: on gRPC reconnect a fresh manager -// is built while the previous one's sessions keep draining bytes via stale TTL -// timers. The replay_bytes gauge must reflect the current live total across -// both, not be clobbered to a single manager's draining value. -func TestLiveLogReplayBytesAdditiveAcrossManagers(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - - const kind = "metrics-overlap-test" - - newManager := func() *notificationStreamSessionManager[*cloud.TestWorkflowNotificationsRequest] { - return newNotificationStreamSessionManager( - ctx, - kind, - func(req *cloud.TestWorkflowNotificationsRequest) string { return req.ExecutionId }, - func(context.Context, *cloud.TestWorkflowNotificationsRequest) NotificationWatcher { - return channels.NewWatcher[*testkube.TestWorkflowExecutionNotification]() - }, - ) - } - - require.Equal(t, float64(0), testutil.ToFloat64(liveLogReplayBytes.WithLabelValues(kind))) - - // Old manager buffers some bytes. - oldManager := newManager() - oldSession := newNotificationStreamSession(oldManager.addReplayBytes) - oldSession.publish(&testkube.TestWorkflowExecutionNotification{Log: "old-manager-payload"}) - oldBytes := testutil.ToFloat64(liveLogReplayBytes.WithLabelValues(kind)) - require.Greater(t, oldBytes, float64(0)) - - // A reconnect builds a fresh manager whose session buffers more bytes. - newMgr := newManager() - newSession := newNotificationStreamSession(newMgr.addReplayBytes) - newSession.publish(&testkube.TestWorkflowExecutionNotification{Log: "new-manager-payload"}) - combined := testutil.ToFloat64(liveLogReplayBytes.WithLabelValues(kind)) - require.Greater(t, combined, oldBytes) - - newBytes := combined - oldBytes - - // The old manager's stale timer releases its session's bytes. The gauge must - // drop by exactly the old contribution and still hold the new manager's live - // total, not be reset to the old manager's (now zero) draining value. - oldSession.releaseReplayBytes() - require.Equal(t, newBytes, testutil.ToFloat64(liveLogReplayBytes.WithLabelValues(kind))) - - // Once the new manager's session drops too, the gauge nets back to zero. - newSession.releaseReplayBytes() - require.Equal(t, float64(0), testutil.ToFloat64(liveLogReplayBytes.WithLabelValues(kind))) -} diff --git a/pkg/controlplaneclient/testworkflows_test.go b/pkg/controlplaneclient/testworkflows_test.go index 898d6aa6cb5..f860a3e20cb 100644 --- a/pkg/controlplaneclient/testworkflows_test.go +++ b/pkg/controlplaneclient/testworkflows_test.go @@ -3,10 +3,14 @@ package controlplaneclient import ( "context" "errors" + "sync" "sync/atomic" "testing" "time" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + dto "github.com/prometheus/client_model/go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" gomock "go.uber.org/mock/gomock" @@ -649,3 +653,296 @@ func TestNotificationStreamSessionReplacementCancelsOrphanedSource(t *testing.T) t.Fatal("replaced session source context was not cancelled") } } + +// newMetricsTestSessionManager builds a manager whose kind label is the test name. +// The metrics live on the shared default registry, so a unique kind keeps each +// test's readings apart from the other tests in the package. +func newMetricsTestSessionManager(t *testing.T, process func(context.Context, *cloud.TestWorkflowNotificationsRequest) NotificationWatcher) *notificationStreamSessionManager[*cloud.TestWorkflowNotificationsRequest] { + t.Helper() + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + return newNotificationStreamSessionManager( + ctx, + t.Name(), + func(req *cloud.TestWorkflowNotificationsRequest) string { return req.ExecutionId }, + process, + ) +} + +// blockingNotificationSource returns a process function whose watcher sends one +// log line and then stays open until the test finishes the execution id, with +// the error the source should end with. +func blockingNotificationSource(t *testing.T) (func(context.Context, *cloud.TestWorkflowNotificationsRequest) NotificationWatcher, func(executionID string, err error)) { + t.Helper() + + type sourceEnd struct { + err error + done chan struct{} + } + var mu sync.Mutex + ends := make(map[string]*sourceEnd) + process := func(_ context.Context, req *cloud.TestWorkflowNotificationsRequest) NotificationWatcher { + end := &sourceEnd{done: make(chan struct{})} + mu.Lock() + ends[req.ExecutionId] = end + mu.Unlock() + watcher := channels.NewWatcher[*testkube.TestWorkflowExecutionNotification]() + go func() { + watcher.Send(&testkube.TestWorkflowExecutionNotification{Log: "line"}) + <-end.done + watcher.Close(end.err) + }() + return watcher + } + // finish waits for the source, because runSource starts it on its own goroutine. + finish := func(executionID string, err error) { + var end *sourceEnd + require.Eventually(t, func() bool { + mu.Lock() + defer mu.Unlock() + end = ends[executionID] + return end != nil + }, 2*time.Second, time.Millisecond, "no source was started for %q", executionID) + end.err = err + close(end.done) + } + t.Cleanup(func() { + mu.Lock() + defer mu.Unlock() + for _, end := range ends { + select { + case <-end.done: + default: + close(end.done) + } + } + }) + return process, finish +} + +func metricValue(t *testing.T, collector prometheus.Collector) float64 { + t.Helper() + return testutil.ToFloat64(collector) +} + +// histogramSampleCount reads one label set's sample count. testutil.CollectAndCount +// on a curried vector counts every label set in the vector, so it cannot isolate a +// single kind on the shared registry. +func histogramSampleCount(t *testing.T, observer prometheus.Observer) uint64 { + t.Helper() + + var metric dto.Metric + require.NoError(t, observer.(prometheus.Metric).Write(&metric)) + return metric.GetHistogram().GetSampleCount() +} + +func sessionReplayBytes(session *notificationStreamSession) int { + session.mu.Lock() + defer session.mu.Unlock() + return session.replayBytes +} + +func waitForNotificationSessionDone(t *testing.T, session *notificationStreamSession) { + t.Helper() + + require.Eventually(t, func() bool { + done, _ := session.status() + return done + }, 2*time.Second, time.Millisecond) +} + +func TestNotificationStreamSessionManagerMetricsFollowSessionLifecycle(t *testing.T) { + process, finish := blockingNotificationSource(t) + manager := newMetricsTestSessionManager(t, process) + manager.sessionIdleTTL = time.Minute + kind := manager.kind + + // A fresh attach creates one active session with one subscriber. + session, sub, _, _, _, _ := manager.attach(&cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1"}) + assert.Equal(t, float64(1), metricValue(t, liveLogSessionsCreatedTotal.WithLabelValues(kind))) + assert.Equal(t, float64(1), metricValue(t, liveLogSessions.WithLabelValues(kind, "active"))) + assert.Equal(t, float64(0), metricValue(t, liveLogSessions.WithLabelValues(kind, "done"))) + assert.Equal(t, float64(1), metricValue(t, liveLogSubscribers.WithLabelValues(kind))) + + // A second viewer that resumes the same stream shares the session. + _, sub2, _, _, _, _ := manager.attach(&cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1", ResumeAfterSeqNo: 1}) + assert.Equal(t, float64(1), metricValue(t, liveLogSessionsCreatedTotal.WithLabelValues(kind))) + assert.Equal(t, float64(2), metricValue(t, liveLogSubscribers.WithLabelValues(kind))) + + // The source ends: the session moves from active to done and records its duration. + finish("exec-1", nil) + waitForNotificationSessionDone(t, session) + require.Eventually(t, func() bool { + return metricValue(t, liveLogSessions.WithLabelValues(kind, "active")) == 0 && + metricValue(t, liveLogSessions.WithLabelValues(kind, "done")) == 1 && + histogramSampleCount(t, liveLogSourceDurationSeconds.WithLabelValues(kind)) == 1 + }, 2*time.Second, time.Millisecond) + assert.Greater(t, metricValue(t, liveLogReplayBytes.WithLabelValues(kind)), float64(0)) + + // Viewers leave: the subscriber gauge returns to zero. + manager.detach(session, sub) + manager.detach(session, sub2) + assert.Equal(t, float64(0), metricValue(t, liveLogSubscribers.WithLabelValues(kind))) + + // The idle TTL passes: the sweep evicts the done session and frees its buffer. + manager.sweepExpired(time.Now().Add(2 * manager.sessionIdleTTL)) + assert.Equal(t, float64(0), metricValue(t, liveLogSessions.WithLabelValues(kind, "done"))) + assert.Equal(t, float64(1), metricValue(t, liveLogSessionsEvictedTotal.WithLabelValues(kind, "ttl"))) + assert.Equal(t, float64(0), metricValue(t, liveLogReplayBytes.WithLabelValues(kind))) +} + +func TestNotificationStreamSessionManagerReplayBytesGaugeTracksBufferedBytes(t *testing.T) { + process, finish := blockingNotificationSource(t) + manager := newMetricsTestSessionManager(t, process) + manager.sessionIdleTTL = time.Minute + kind := manager.kind + gauge := func() float64 { return metricValue(t, liveLogReplayBytes.WithLabelValues(kind)) } + + // The viewers leave at once: this test only reads the replay buffer, and an + // unread subscription would block publish once its channel fills. + first, firstSub, _, _, _, _ := manager.attach(&cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1"}) + manager.detach(first, firstSub) + + // The source's first line lands in the buffer before the test publishes more, + // so every later reading compares a settled session against the gauge. + require.Eventually(t, func() bool { return sessionReplayBytes(first) > 0 }, 2*time.Second, time.Millisecond) + assert.Equal(t, float64(sessionReplayBytes(first)), gauge()) + + // Every buffered notification adds its size to the gauge. + first.publish(&testkube.TestWorkflowExecutionNotification{Log: "hello"}) + assert.Equal(t, float64(sessionReplayBytes(first)), gauge()) + + // Once the buffer trims old events, the gauge follows the trimmed total. + for i := 0; i < workflowNotificationReplayMaxEvents; i++ { + first.publish(&testkube.TestWorkflowExecutionNotification{Log: "fill"}) + } + first.mu.Lock() + replayLen := len(first.replay) + first.mu.Unlock() + require.Equal(t, workflowNotificationReplayMaxEvents, replayLen) + firstBytes := sessionReplayBytes(first) + assert.Equal(t, float64(firstBytes), gauge()) + + // Sessions of one kind share the gauge, so a second session adds to it. + second, secondSub, _, _, _, _ := manager.attach(&cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-2", StreamId: "stream-2"}) + manager.detach(second, secondSub) + require.Eventually(t, func() bool { return sessionReplayBytes(second) > 0 }, 2*time.Second, time.Millisecond) + second.publish(&testkube.TestWorkflowExecutionNotification{Log: "second"}) + secondBytes := sessionReplayBytes(second) + assert.Equal(t, float64(firstBytes+secondBytes), gauge()) + + // When the sweep evicts one session, the gauge drops by that session's bytes only. + finish("exec-1", nil) + waitForNotificationSessionDone(t, first) + manager.sweepExpired(time.Now().Add(2 * manager.sessionIdleTTL)) + assert.Equal(t, float64(secondBytes), gauge()) + + // When the sweep evicts the last session, the gauge returns to zero. + finish("exec-2", nil) + waitForNotificationSessionDone(t, second) + manager.sweepExpired(time.Now().Add(2 * manager.sessionIdleTTL)) + assert.Equal(t, float64(0), gauge()) +} + +func TestNotificationStreamSessionManagerResumeMetricsCountResult(t *testing.T) { + tests := []struct { + name string + prepare func(t *testing.T, manager *notificationStreamSessionManager[*cloud.TestWorkflowNotificationsRequest]) + req *cloud.TestWorkflowNotificationsRequest + available float64 + unavailable float64 + }{ + { + name: "resume inside the replay buffer is available", + prepare: func(t *testing.T, manager *notificationStreamSessionManager[*cloud.TestWorkflowNotificationsRequest]) { + session, sub, _, _, _, _ := manager.attach(&cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1"}) + t.Cleanup(func() { manager.detach(session, sub) }) + session.publish(&testkube.TestWorkflowExecutionNotification{Log: "two"}) + }, + req: &cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1", ResumeAfterSeqNo: 1}, + available: 1, + unavailable: 0, + }, + { + name: "resume without a session is unavailable", + prepare: func(*testing.T, *notificationStreamSessionManager[*cloud.TestWorkflowNotificationsRequest]) {}, + req: &cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1", ResumeAfterSeqNo: 5}, + available: 0, + unavailable: 1, + }, + { + name: "start from zero is not a resume", + prepare: func(*testing.T, *notificationStreamSessionManager[*cloud.TestWorkflowNotificationsRequest]) {}, + req: &cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1"}, + available: 0, + unavailable: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + process, _ := blockingNotificationSource(t) + manager := newMetricsTestSessionManager(t, process) + kind := manager.kind + tt.prepare(t, manager) + + session, sub, _, _, _, _ := manager.attach(tt.req) + t.Cleanup(func() { manager.detach(session, sub) }) + + assert.Equal(t, tt.available, metricValue(t, liveLogResumeTotal.WithLabelValues(kind, "available"))) + assert.Equal(t, tt.unavailable, metricValue(t, liveLogResumeTotal.WithLabelValues(kind, "unavailable"))) + }) + } +} + +func TestNotificationStreamSessionManagerEvictionMetricsCountReason(t *testing.T) { + tests := []struct { + name string + sourceErr error + evict func(t *testing.T, manager *notificationStreamSessionManager[*cloud.TestWorkflowNotificationsRequest]) + reason string + }{ + { + name: "idle TTL sweep evicts a done session", + sourceErr: nil, + evict: func(t *testing.T, manager *notificationStreamSessionManager[*cloud.TestWorkflowNotificationsRequest]) { + manager.sweepExpired(time.Now().Add(2 * manager.sessionIdleTTL)) + }, + reason: "ttl", + }, + { + name: "resume after a failed source evicts the errored session", + sourceErr: errors.New("source failed"), + evict: func(t *testing.T, manager *notificationStreamSessionManager[*cloud.TestWorkflowNotificationsRequest]) { + session, sub, _, _, _, _ := manager.attach(&cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1", ResumeAfterSeqNo: 1}) + t.Cleanup(func() { manager.detach(session, sub) }) + }, + reason: "error", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + process, finish := blockingNotificationSource(t) + manager := newMetricsTestSessionManager(t, process) + manager.sessionIdleTTL = time.Minute + kind := manager.kind + + session, sub, _, _, _, _ := manager.attach(&cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1"}) + finish("exec-1", tt.sourceErr) + waitForNotificationSessionDone(t, session) + manager.detach(session, sub) + require.Eventually(t, func() bool { + return metricValue(t, liveLogSessions.WithLabelValues(kind, "done")) == 1 + }, 2*time.Second, time.Millisecond) + + tt.evict(t, manager) + + assert.Equal(t, float64(1), metricValue(t, liveLogSessionsEvictedTotal.WithLabelValues(kind, tt.reason))) + assert.Equal(t, float64(0), metricValue(t, liveLogSessions.WithLabelValues(kind, "done"))) + manager.mu.Lock() + current := manager.sessions["exec-1:stream-1"] + manager.mu.Unlock() + assert.NotSame(t, session, current, "evicted session must leave the manager") + }) + } +} diff --git a/pkg/controlplaneclient/utils.go b/pkg/controlplaneclient/utils.go index ef4e6a32f45..f767b8f65b6 100644 --- a/pkg/controlplaneclient/utils.go +++ b/pkg/controlplaneclient/utils.go @@ -370,10 +370,10 @@ func (m *notificationStreamSessionManager[Request]) runSweeper(interval time.Dur } } -// addReplayBytes applies a signed byte delta to the gauge. Adds are matched by -// their own release deltas, so overlapping managers of the same kind (e.g. a -// new manager built on gRPC reconnect while an old one's sessions still drain) -// sum correctly and the gauge nets back to zero once every session drops. +// addReplayBytes applies a signed byte delta to the gauge. Every session of one +// kind shares the gauge, and each add is matched by that session's own release +// delta, so the gauge is the live total across sessions and returns to zero +// once every session drops, with no manager-wide recount. func (m *notificationStreamSessionManager[Request]) addReplayBytes(delta int) { if delta == 0 { return From 5d772a8bf5773665ce81eb0ab66e45ff7c0d363f Mon Sep 17 00:00:00 2001 From: Dejan Zele Pejchev Date: Mon, 7 Sep 2026 12:22:41 +0200 Subject: [PATCH 3/7] chore(deps): require godebug for the prometheus testutil import --- go.mod | 1 + 1 file changed, 1 insertion(+) diff --git a/go.mod b/go.mod index 6074c5fa4a5..2be4d359552 100644 --- a/go.mod +++ b/go.mod @@ -216,6 +216,7 @@ require ( github.com/klauspost/compress v1.19.2 // indirect github.com/klauspost/cpuid/v2 v2.4.0 // indirect github.com/klauspost/crc32 v1.3.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/lithammer/fuzzysearch v1.1.8 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect From b21abd713623443e6b433ab9377d171cb71ac76e Mon Sep 17 00:00:00 2001 From: Dejan Zele Pejchev Date: Mon, 7 Sep 2026 12:47:32 +0200 Subject: [PATCH 4/7] chore(deps): run go mod tidy --- go.sum | 124 ++++++++------------------------------------------------- 1 file changed, 16 insertions(+), 108 deletions(-) diff --git a/go.sum b/go.sum index 717895fba27..72f38f5780f 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,6 @@ github.com/alecthomas/participle/v2 v2.1.4 h1:W/H79S8Sat/krZ3el6sQMvMaahJ+XcM9WS github.com/alecthomas/participle/v2 v2.1.4/go.mod h1:8tqVbpTX20Ru4NfYQgZf4mP18eXPTBViyMWiArNEgGI= github.com/alecthomas/repr v0.5.4 h1:OVP7JEcuzU9CCDsT6STCr3rg17oQfWILtPWd2EG0uN4= github.com/alecthomas/repr v0.5.4/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= -github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM= -github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/andybalholm/brotli v1.2.3 h1:8H1qwOkl2LPfjf3YezB90JnCliZb6SInJ/OJkEbA5NQ= github.com/andybalholm/brotli v1.2.3/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= @@ -173,16 +171,12 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dlclark/regexp2 v1.12.0 h1:0j4c5qQmnC6XOWNjP3PIXURXN2gWx76rd3KvgdPkCz8= github.com/dlclark/regexp2 v1.12.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/docker/cli v29.6.2+incompatible h1:/bjePvcbbFTnRrMfWJBY7AjfICdsiLVgHn6LwTVOcqw= -github.com/docker/cli v29.6.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/cli v29.7.2+incompatible h1:dlkwallR8XqfeVnA2ELEhdwvb4lsSwuB4IgsG8Q9cLY= github.com/docker/cli v29.7.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM= github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.9.5 h1:EFNN8DHvaiK8zVqFA2DT6BjXE0GzfLOZ38ggPTKePkY= github.com/docker/docker-credential-helpers v0.9.5/go.mod h1:v1S+hepowrQXITkEfw6o4+BMbGot02wiKpzWhGUZK6c= -github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c= -github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= github.com/docker/go-connections v0.8.1 h1:JibmG5hULs5qXSr/cp/w3Pw5fZuStt4MOHMUExb29/M= github.com/docker/go-connections v0.8.1/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -249,54 +243,30 @@ github.com/go-openapi/jsonpointer v1.0.0 h1:kR9tHqY0CtZaOPVFm622dPVNhrvYpwr4uCxg github.com/go-openapi/jsonpointer v1.0.0/go.mod h1:Z3rw7dWu1p9IgitXCFamSlA5lmDiklEB6vkaxcNZW5Y= github.com/go-openapi/jsonreference v1.0.0 h1:jlmTr6torcd1YgDQvSfNmRtKzYDO4FGBkrAdlAVWnpY= github.com/go-openapi/jsonreference v1.0.0/go.mod h1:jtwdyGbJk0Xhe5Y+rwtglQP6Sb1WZST4rT32LWB+sv0= -github.com/go-openapi/swag v0.27.1 h1:VotvOLWW8q/EAxB0YdsBBGC8XYyeL1YwBj2ungAGPNg= -github.com/go-openapi/swag v0.27.1/go.mod h1:GTkJPwHfhJp6MWr4/rCh64HVI3Ofu+tcsbfjfHmTxpE= github.com/go-openapi/swag v0.28.0 h1:xkgbOSKj6DZziNpyqRRAOt3GJGtgjgsd2RoyT30VWuw= github.com/go-openapi/swag v0.28.0/go.mod h1:4qYnT3Cqr1p1VknOdPo70evN4rgQnAg6jwApHyxSGIg= -github.com/go-openapi/swag/cmdutils v0.27.1 h1:I7sYqaWVl5mq0NEmNQkAmFDyNin9ufvMX/p2zwtQaOE= -github.com/go-openapi/swag/cmdutils v0.27.1/go.mod h1:Sm1MVFMkF6guJJ+pQqHnQA3N0j9qALV3NxzDSv6bETM= github.com/go-openapi/swag/cmdutils v0.28.0 h1:7TOeNtkYru1SG8Y34tDh9WBbLsMqGnptuxWiHREPZ4Q= github.com/go-openapi/swag/cmdutils v0.28.0/go.mod h1:Sm1MVFMkF6guJJ+pQqHnQA3N0j9qALV3NxzDSv6bETM= -github.com/go-openapi/swag/conv v0.27.1 h1:8wi9ZG+olmY1wXphl93EWniPtbSPkXM/feH7FgjsvrU= -github.com/go-openapi/swag/conv v0.27.1/go.mod h1:QbqMivkpKhC3g1B1GGGOJ6ANewI3S62dbzYu3Duowqs= github.com/go-openapi/swag/conv v0.28.0 h1:GtqqbyFe7vR5Y7ehxG9W6/OvrSFdf1OLeTGp40TqxH8= github.com/go-openapi/swag/conv v0.28.0/go.mod h1:mbUE+mzctnhxi864m0Q07SpN8OowD9JhxmxuYvZZD/k= -github.com/go-openapi/swag/fileutils v0.27.1 h1:QQqBSoi5mW4XpU85nS0mLcA+zAE6vLzrb0QkmLKf9oM= -github.com/go-openapi/swag/fileutils v0.27.1/go.mod h1:VvJFZLTZS0AI854gEQz5tk7dBESdLjiNUMSZ/th2ry8= github.com/go-openapi/swag/fileutils v0.28.0 h1:Z04XWQD7R8Eq+7GnOrjovBxPPmZzsS4gt2H2GPGIViU= github.com/go-openapi/swag/fileutils v0.28.0/go.mod h1:VvJFZLTZS0AI854gEQz5tk7dBESdLjiNUMSZ/th2ry8= -github.com/go-openapi/swag/jsonutils v0.27.1 h1:SVgK3i4USzCU5mibOOS/l4ea2h9UQXy7J7RNLTjuXjU= -github.com/go-openapi/swag/jsonutils v0.27.1/go.mod h1:tdlEpZqdcQ17uj6J4YdK9vd8It5qWMwjWXOs0tjpRlk= github.com/go-openapi/swag/jsonutils v0.28.0 h1:YIch6FwO7RXzeAnbO8Tu7dWBZeUEH+4nA0HXltVTnv4= github.com/go-openapi/swag/jsonutils v0.28.0/go.mod h1:CYM3WlTUcagR2ZoHdz54di/cbBqt82tuxuXgAjxw+mg= -github.com/go-openapi/swag/jsonutils/fixtures_test v0.27.1 h1:mJu3COL9WEaZVp/Kf2PRMi7tPszPEJfSr/OO75ynCs8= -github.com/go-openapi/swag/jsonutils/fixtures_test v0.27.1/go.mod h1:mofwUWx70wvskwESqRJ//k/9kURmCgyJl5m5Ppoh5kY= -github.com/go-openapi/swag/loading v0.27.1 h1:/DxUgDXKbBX4bcn7r9uEXfJyzN5XpiJmZplzQTjrRCY= -github.com/go-openapi/swag/loading v0.27.1/go.mod h1:jvGh3iA2+zyUUycB5fgJWzeHnhrpvGnJJM0RVE9ZShE= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.28.0 h1:qV+VVUAx5Oro8WjVWpZeql7YReTKhT4smR4zhcOQZr0= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.28.0/go.mod h1:mofwUWx70wvskwESqRJ//k/9kURmCgyJl5m5Ppoh5kY= github.com/go-openapi/swag/loading v0.28.0 h1:td8QZdZC9MIYGGSnSPKShKiK22I2tU5UQvuUhIBPRLU= github.com/go-openapi/swag/loading v0.28.0/go.mod h1:rXB0QiQX5mMveXEA7ouM4KiiM9jVJe4K6BVbwhD1M4k= -github.com/go-openapi/swag/mangling v0.27.1 h1:yC9D0HyUE8gbP+BfmGx9+AA89ikwZTMjESK3OnnoaqA= -github.com/go-openapi/swag/mangling v0.27.1/go.mod h1:jtBE2+V+3pILxOR7Vgce+Cwp6A2PgZbvVqfNntbVs0w= github.com/go-openapi/swag/mangling v0.28.0 h1:pH8eyeNO9SLYsTMWJrurnNfKmDa28XrlA+HePVD53VM= github.com/go-openapi/swag/mangling v0.28.0/go.mod h1:jtBE2+V+3pILxOR7Vgce+Cwp6A2PgZbvVqfNntbVs0w= -github.com/go-openapi/swag/netutils v0.27.1 h1:mICMFoS82F5TZ4Zy3cqmcQk+BFeCp3Uyq3Np7GI0/qU= -github.com/go-openapi/swag/netutils v0.27.1/go.mod h1:J+WYyFMLtvtCGqa6jLv+YNUmIKI3ZRQRrvfNDMoQoEQ= github.com/go-openapi/swag/netutils v0.28.0 h1:YXN6TALEi2pzts8/8GNm6T61HTAZsieukGZidap989k= github.com/go-openapi/swag/netutils v0.28.0/go.mod h1:J+WYyFMLtvtCGqa6jLv+YNUmIKI3ZRQRrvfNDMoQoEQ= -github.com/go-openapi/swag/pools v0.27.1 h1:9LeadcMyb2GJCbXX5hVQDbZ2Lq9TL4dCs/nx1j5DO0E= -github.com/go-openapi/swag/pools v0.27.1/go.mod h1:kVQefhSK5RWuRe7BXsL8htgBPAMpN7HDGpGEknqugeE= github.com/go-openapi/swag/pools v0.28.0 h1:HPMZWSAfce3rdVTFcjFiCIBtDg9h4x2QlRrHipwhxeU= github.com/go-openapi/swag/pools v0.28.0/go.mod h1:kVQefhSK5RWuRe7BXsL8htgBPAMpN7HDGpGEknqugeE= -github.com/go-openapi/swag/stringutils v0.27.1 h1:ZXePZ0r2p1qSjo8tD3Un4vFj8+FqlCkczxDrJIhYUp8= -github.com/go-openapi/swag/stringutils v0.27.1/go.mod h1:lzRN95CxXmA03XcDWHLOb6nOMcxCqR5rGY0lOgsfRoM= github.com/go-openapi/swag/stringutils v0.28.0 h1:ixsc9iYgDPubHL/8nSkbnryEHpD2VRlBMLKpQyPXcDU= github.com/go-openapi/swag/stringutils v0.28.0/go.mod h1:lzRN95CxXmA03XcDWHLOb6nOMcxCqR5rGY0lOgsfRoM= -github.com/go-openapi/swag/typeutils v0.27.1 h1:KSTdFlfnse4r6dP9IrEnwMldjE+zs71UeEB3//PtVXc= -github.com/go-openapi/swag/typeutils v0.27.1/go.mod h1:Srm0xFNRZ1Y+vCxJclo5qzx8aj+1pAKda/YfFPrG0dQ= github.com/go-openapi/swag/typeutils v0.28.0 h1:nRBKSBXjDgf01VDPB3fWeD9nQuhCOVeIYAkUx2tbkyY= github.com/go-openapi/swag/typeutils v0.28.0/go.mod h1:Srm0xFNRZ1Y+vCxJclo5qzx8aj+1pAKda/YfFPrG0dQ= -github.com/go-openapi/swag/yamlutils v0.27.1 h1:ftxv6xvXb1E3zohUc+okZ9nSqNb9StQX/FXnKZ98sQA= -github.com/go-openapi/swag/yamlutils v0.27.1/go.mod h1:bnxFIB1qewGRiZHypXGZ3fNgf13/0HfRgnS/iZBDrOo= github.com/go-openapi/swag/yamlutils v0.28.0 h1:TV3JXH6DS46KUroDtMLAYHGkdWf5VDq3wVWFirmzROY= github.com/go-openapi/swag/yamlutils v0.28.0/go.mod h1:x0q/yndZHEgk9Rx3DyDqzFUmHy55KTvIZldvF2dTJXs= github.com/go-openapi/testify/enable/yaml/v2 v2.6.0 h1:gGHwAJ0R/5jU8BEGDbfRNR3hL68dAVi84WuOApp29B0= @@ -336,18 +306,14 @@ github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/cel-go v0.26.0 h1:DPGjXackMpJWH680oGY4lZhYjIameYmR+/6RBdDGmaI= -github.com/google/cel-go v0.26.0/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= +github.com/google/cel-go v0.26.1 h1:iPbVVEdkhTX++hpe3lzSk7D3G3QSYqLGoHOcEio+UXQ= +github.com/google/cel-go v0.26.1/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= github.com/google/gnostic-models v0.7.1 h1:SisTfuFKJSKM5CPZkffwi6coztzzeYUhc3v4yxLWH8c= github.com/google/gnostic-models v0.7.1/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/go-containerregistry v0.21.9 h1:F+D4uZ3iA3DLMJLfhaqMdHJbzeqm/216WGQq2dokuLs= -github.com/google/go-containerregistry v0.21.9/go.mod h1:dP5XNKcL7kMFF/TB3LfvWmVhAcv7iqkHb3oDK8aauTo= -github.com/google/go-containerregistry v0.22.0 h1:eGbCiPeYxAH/7WLLq6zTBALP0tUIFsoyRauhxXDJ53I= -github.com/google/go-containerregistry v0.22.0/go.mod h1:bJR35SK8XgisYmhg/FMQ/5RK0S/XrOAqLBV5/LR2XE0= github.com/google/go-containerregistry v0.22.1 h1:RZuuSYhTvlDvtsK+NkutoCZ//C0X2ebLK8X8l3ULs84= github.com/google/go-containerregistry v0.22.1/go.mod h1:bJR35SK8XgisYmhg/FMQ/5RK0S/XrOAqLBV5/LR2XE0= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= @@ -371,12 +337,8 @@ github.com/gookit/color v1.6.1 h1:KoTnDxJPRgrL0SoX0f8rCFg2zI0t4E3GZZBMo2nN8LU= github.com/gookit/color v1.6.1/go.mod h1:9ACFc7/1IpHGBW8RwuDm/0YEnhg3dwwXpoMsmtyHfjs= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= -github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 h1:B+8ClL/kCQkRiU82d9xajRPKYMrB7E0MbtzWVi1K4ns= -github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3/go.mod h1:NbCUVmiS4foBGBHOYlCT25+YmGpJ32dZPi75pGEUpj4= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.4 h1:9ZJYjPEJpcleIKcqysXOo14NLQYwu23fzmKRFpIjPjc= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.4/go.mod h1:0xydIeg2omQ9DH6zYMP24Kk57793rnLUvP8fwdbFvz8= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY= github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= @@ -455,8 +417,6 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magiconair/properties v1.18.11 h1:j5ozYZl0zCjG7ahMDH0GWIobOvvUzT0BdAguG0ViKy0= github.com/magiconair/properties v1.18.11/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/mark3labs/mcp-go v0.58.0 h1:AWfBk8lgRR0KZYve7PaLbR2MIjpw1oK2eGpBApaNS+Q= -github.com/mark3labs/mcp-go v0.58.0/go.mod h1:+8WclSK1ZUweCP3hvktSji8n8ABG/95QaEkeVE/Uwas= github.com/mark3labs/mcp-go v1.0.0 h1:CZqCzXwUiTOstkIdW1MyOZGuM+LaKMEJSu+ZYluN4DU= github.com/mark3labs/mcp-go v1.0.0/go.mod h1:r2fW4o3wsoJ7IMsx1Wuq5xeP8PRGXPDfNveoGAYbb/s= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= @@ -512,8 +472,6 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc= github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc= -github.com/nats-io/nats-server/v2 v2.14.5 h1:M6yeo/Xb7khi97RSEVELof3DForDqmYza3P4tHCPFWw= -github.com/nats-io/nats-server/v2 v2.14.5/go.mod h1:1D3iocrisKvWaD1B/imqarTqmaGrWMqALMLbEDo3v7Q= github.com/nats-io/nats-server/v2 v2.14.6 h1:dHjEHa49lUTxIb5jrsk5kz/AmOyOQ4T9ti8djcBumDQ= github.com/nats-io/nats-server/v2 v2.14.6/go.mod h1:u/0bNBzN/m5orQGz7q2NNzxCQ6PWdwaOSwPNDADZn/A= github.com/nats-io/nats.go v1.53.1 h1:Otsq3uLc/kLdjmkNHkXH0jBqwUquwdKFoe3fq6/3/Xo= @@ -528,8 +486,6 @@ github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oasisprotocol/curve25519-voi v0.0.0-20230110094441-db37f07504ce h1:/pEpMk55wH0X+E5zedGEMOdLuWmV8P4+4W3+LZaM6kg= github.com/oasisprotocol/curve25519-voi v0.0.0-20230110094441-db37f07504ce/go.mod h1:hVoHR2EVESiICEMbg137etN/Lx+lSrHPTD39Z/uE+2s= -github.com/ohler55/ojg v1.28.4 h1:KVmO+KWnm4IU3bgKsp3pGUkyepoHnon5TwrvnlGx5Ek= -github.com/ohler55/ojg v1.28.4/go.mod h1:/Y5dGWkekv9ocnUixuETqiL58f+5pAsUfg5P8e7Pa2o= github.com/ohler55/ojg v1.28.5 h1:KlNeyCDlwt6CDlv7VP6f9sAe9w4t5trxJCo64vO0/kc= github.com/ohler55/ojg v1.28.5/go.mod h1:/Y5dGWkekv9ocnUixuETqiL58f+5pAsUfg5P8e7Pa2o= github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 h1:zrbMGy9YXpIeTnGj4EljqMiZsIcE09mmF8XsD5AYOJc= @@ -548,8 +504,6 @@ github.com/onsi/ginkgo/v2 v2.32.1 h1:6tlvcDm/3sE8lGJbZ4+d4mO3RLy24/tQWOFzVSQNIfw github.com/onsi/ginkgo/v2 v2.32.1/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= -github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I= -github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg= github.com/onsi/gomega v1.43.0 h1:VlG/1FxqNxhSO+lq/OHBNaaqwiBK/mO8JbVkX9Y+FeU= github.com/onsi/gomega v1.43.0/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -581,20 +535,14 @@ github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/pquerna/cachecontrol v0.2.0 h1:vBXSNuE5MYP9IJ5kjsdo8uq+w41jSPgvba2DEnkRx9k= github.com/pquerna/cachecontrol v0.2.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI= -github.com/pressly/goose/v3 v3.27.3 h1:pIglVHjw99r4e/hDHHwbl9vfOsDMqUokfkXo6+n/RxA= -github.com/pressly/goose/v3 v3.27.3/go.mod h1:Dag+xpV6o20HR2LFY1j0q6MDwc3f7vPUFDA77R+0yGY= github.com/pressly/goose/v3 v3.28.0 h1:D2M+iL31GmpZxSHOhX8mqyqAT3CXnokUmm0eKoSP+Vc= github.com/pressly/goose/v3 v3.28.0/go.mod h1:v26MOuB8bL3kzzrt3Vqhb3R0PRVsl8hFQKdrht/L6Rk= github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= -github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= -github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo= github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM= github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY= github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= -github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= -github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= github.com/prometheus/procfs v0.22.0 h1:6q9+/JL9IKAPbCmBrv9n5O5Ty3NKnciV5X7YGw0oics= github.com/prometheus/procfs v0.22.0/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58= github.com/pterm/pterm v0.12.27/go.mod h1:PhQ89w4i95rhgE+xedAoqous6K9X+r6aSOI2eFF7DZI= @@ -632,8 +580,6 @@ github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/sethvargo/go-retry v0.4.0 h1:9qy1OoIAxBL+gBYnkTnTnWle5wlfsXQlwRzIbbpdqPw= github.com/sethvargo/go-retry v0.4.0/go.mod h1:tvsjdKG6xfiCx4LSiUZ06kcv38xvdVQwv8R6/VnnVWg= -github.com/shirou/gopsutil/v4 v4.26.7 h1:IXzpHz/dkMRYAhKkOXr1HB6SuzWU3eoyyeWe7g3bNZc= -github.com/shirou/gopsutil/v4 v4.26.7/go.mod h1:5O9FjBiXoTDFatIWjZZosqj4pV0DRtLx598xGbBehzM= github.com/shirou/gopsutil/v4 v4.26.8 h1:YQMTF/1J50B5+Y0vlo1eDRf5DoR7Gk69hY+8wjYkQeo= github.com/shirou/gopsutil/v4 v4.26.8/go.mod h1:5O9FjBiXoTDFatIWjZZosqj4pV0DRtLx598xGbBehzM= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= @@ -651,8 +597,8 @@ github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiT github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= -github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= +github.com/stoewer/go-strcase v1.3.1 h1:iS0MdW+kVTxgMoE1LAZyMiYJFKlOzLooE4MxjirtkAs= +github.com/stoewer/go-strcase v1.3.1/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -722,54 +668,30 @@ github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= -go.mongodb.org/mongo-driver/v2 v2.8.0 h1:CxWDGQYY8QQwNjAl/aq2sfWakdnWZynnqJ9F4DhHbP8= -go.mongodb.org/mongo-driver/v2 v2.8.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= -go.mongodb.org/mongo-driver/v2 v2.8.1 h1:kJNOCrvRN6rVqMO3AonIoD7Z3yjBBHKIc1SSlZcC/xM= -go.mongodb.org/mongo-driver/v2 v2.8.1/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= -go.mongodb.org/mongo-driver/v2 v2.8.2 h1:b6o2m7zL8g2URuO8urBedAylxojybKXNZTxgkOcl+2w= -go.mongodb.org/mongo-driver/v2 v2.8.2/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= go.mongodb.org/mongo-driver/v2 v2.9.0 h1:e2mQdOmbkiYz+dj3faM7lVDwl7WdnRD+g5VicafMhL0= go.mongodb.org/mongo-driver/v2 v2.9.0/go.mod h1:SHKN0IWkKmEVGHLjXnni6s4wPKX4v86FTgOeJJFuXcA= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib v1.20.0 h1:oXUiIQLlkbi9uZB/bt5B1WRLsrTKqb7bPpAQ+6htn2w= go.opentelemetry.io/contrib v1.20.0/go.mod h1:gIzjwWFoGazJmtCaDgViqOSJPde2mCWzv60o0bWPcZs= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0 h1:oECp5f+hN7nkwjU/8BxQ/q23bGPb8FIrD839owX222E= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0/go.mod h1:DqEFwLumhzMBDQv9PcWbyoDxHI/4lAk6CM4nJBH39sc= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0 h1:B2h3uqicet1CT2N5TOFhS+Gq++9i0/CLmaxvhmhtP5s= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0/go.mod h1:dylvB+ZiiwMvsDij9O84Uy7SijLgHMX4mbkncds+4Sw= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 h1:3g7B90UzBltIDKq1/5mrTGxTnOFDV0ICOhLoxiZ8jlg= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0/go.mod h1:Ef8SuTh59BT7+ofpDxN9z+yOlc4t2GjLmKDgYNJL/NU= -go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= -go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc= go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 h1:QRefszxJmfPdjXUUm3j6iDzY03mTPXMjqErFqQ67vUg= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0/go.mod h1:Tiz03lTBVBrm7eWZBOidzEaYaJa8tjwGUGv6d8mlTyk= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 h1:OFnwLJr+pF3iHrlGSzbxyuo6/6HyBlnlN1CWEJmBVcw= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0/go.mod h1:716wFneO0ov19A2beH5hjfh9AK5z/VWNAtDijp1Y0/g= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 h1:DvJDOPmSWQHWywQS6lKL+pb8s3gBLOZUtw4N+mavW1I= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0/go.mod h1:EtekO9DEJb4/jRyN4v4Qjc2yA7AtfCBuz2FynRUWTXs= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0 h1:QBajQ2SrwQijzHyZbQlPsuIzpl/ll8DY6wPWsajeGcI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0/go.mod h1:08ZQLjrPLQ6R4kAXvuOvODEer5Yh4CoFvll5qB2BCI8= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.46.0 h1:KrC1YrQeSt46ITMWAbgQx1M1eV1/1TKzttrBzymPmss= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.46.0/go.mod h1:zDSEzoEqsOrgBeGvH66KRgxh90VonFyJqBHA0Pk3+rM= -go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= -go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8= go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o= -go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw= -go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI= go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM= -go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o= -go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA= -go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag= -go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= +go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE= +go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4= go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c= go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI= go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= @@ -793,12 +715,10 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= -golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y= golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I= -golang.org/x/exp v0.0.0-20260718201538-764159d718ef h1:LkZ48HFgy/TvhTI0bcWkjgFkgLyKUwcTbDjS0DUjw+A= -golang.org/x/exp v0.0.0-20260718201538-764159d718ef/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q= +golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa h1:QSyA8ishJCyT21kER9KwNt0b7BM3iRK4x9QXhjN5Fdk= +golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa/go.mod h1:zeBbvyFKDaLwa7CH/zI8KXt7gTl14SF7sO08Pl5jBCM= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= @@ -876,20 +796,10 @@ gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d h1:FarXi840EJWSHYTN3ERkADbPWjl307+FGrA22KAVjjc= -google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d/go.mod h1:K/+WGbmBY7aNW1HDw1fJnKYo10i0DkAX6pows00dLig= google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 h1:ax2KzoSRIZU/M0cIxri3pKxy99vniH1PVxWC6si/eZI= google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688/go.mod h1:1RJ9BQGyNdZwkGc1eTqkErfRZ6RJyYPHZo73BZ1vQqI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 h1:cYNAzI2sUwhmCcoj9TxvihSrqsxt6uIkj3rDRhSDmW4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260825221802-da73d73af1c5 h1:1VUiZAXyC+zmiFYi+WLtBzr68Cj8wOofHjjrA/kkizc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260825221802-da73d73af1c5/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA= google.golang.org/genproto/googleapis/rpc v0.0.0-20260831171406-18b4a7587f8a h1:3Dnd1cDaZlB68lziofO+bJXpjOy8UfRv8Unt+yH8tQ4= google.golang.org/genproto/googleapis/rpc v0.0.0-20260831171406-18b4a7587f8a/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA= -google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y= -google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= @@ -938,22 +848,20 @@ k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b h1:gMplByicHV/TJBizHd9aVEsTYo k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b/go.mod h1:CgujABENc3KuTrcsdpGmrrASjtQsWCT7R99mEV4U/fM= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= -k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad h1:oXImqH8mQNk7PmvzKhmN3ddJoY6OnyM225MXwGHPm0A= -k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad/go.mod h1:0/mqHCVhlumdJ3BhCfnjSZQE037nAhNodh1/hK0T8/I= k8s.io/kube-openapi v0.0.0-20260821135717-be32def86098 h1:z5+pcu1jTyKK5mNTe2/+x+U6Uuv9jRVOJQLaBJJMpeI= k8s.io/kube-openapi v0.0.0-20260821135717-be32def86098/go.mod h1:0/mqHCVhlumdJ3BhCfnjSZQE037nAhNodh1/hK0T8/I= k8s.io/streaming v0.36.4 h1:RS5YlhrdBN2pKGVjgygGntdu6SNdsduyjGWGe3cX0vo= k8s.io/streaming v0.36.4/go.mod h1:tJ6S2bZa2HxIBauguBbCWSCYyd93Grfz1+z3tcOvlDE= k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3 h1:jVkFFVfXdXP74B/zbO3hM3hpSFD0xvhQ5U686DPurkE= k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3/go.mod h1:M2s5JB1lIYP3jzZdorPLHXIPJzt9vv2muW5a6L9DtNM= -modernc.org/libc v1.74.3 h1:a4J+Z8aVaxPyjyxRAdJzw246PqpcFGvVPnfT/AuM5Ws= -modernc.org/libc v1.74.3/go.mod h1:4H7h/MJ8wnjL8RAbp9v3OXgnk22X7MouHIhDbvP3gj4= +modernc.org/libc v1.75.6 h1:yKk8qo+Di4gkmvRboK8ocCqH22FiUCR6jRy2OwtCRus= +modernc.org/libc v1.75.6/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= -modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= -modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= -modernc.org/sqlite v1.54.0 h1:JCxR4qwkJvOaqAoYcgDoO25Nc+ROg6EJ2LfBVzdrgog= -modernc.org/sqlite v1.54.0/go.mod h1:4ntCLuNmnH8+GNqjka1wNg7KJd5/Hi5FYp8K+XQ7GZw= +modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= +modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/sqlite v1.57.0 h1:qNQP6xnx5M0ISNtlnxoOX0+cD5bJ0/gr9aMmndFczzg= +modernc.org/sqlite v1.57.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 h1:hSfpvjjTQXQY2Fol2CS0QHMNs/WI1MOSGzCm1KhM5ec= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/controller-runtime v0.24.1 h1:miPEwrmirImAvgME1L9qebGHrOnGJoVmVdtOU9fRfo4= From 16561041233fb282dce1f7a85a57af39cfee59ad Mon Sep 17 00:00:00 2001 From: Dejan Zele Pejchev Date: Mon, 7 Sep 2026 12:57:09 +0200 Subject: [PATCH 5/7] refactor(runner): read live-log gauges at scrape time and count evictions once --- pkg/controlplaneclient/metrics.go | 164 ++++++++-- pkg/controlplaneclient/testworkflows_test.go | 321 +++++++++++++------ pkg/controlplaneclient/utils.go | 151 +++++---- 3 files changed, 427 insertions(+), 209 deletions(-) diff --git a/pkg/controlplaneclient/metrics.go b/pkg/controlplaneclient/metrics.go index 56adb0b7e8a..ded4b3853c8 100644 --- a/pkg/controlplaneclient/metrics.go +++ b/pkg/controlplaneclient/metrics.go @@ -1,28 +1,21 @@ package controlplaneclient import ( + "sort" + "sync" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" ) -// Live-log streaming metrics. Registered on the default registry via promauto, -// which is what pkg/server/httpserver.go serves at /metrics. +// Live-log streaming metrics, registered on the default registry, which is what +// pkg/server/httpserver.go serves at /metrics. +// +// Counters and the histogram record events at the point where they happen. +// Gauges are read at scrape time from the live session managers through +// liveLogCollector, so they always equal the state the managers hold and no +// code path has to keep them in step. var ( - liveLogSessions = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "testkube_live_log_sessions", - Help: "Current number of live-log streaming sessions by state", - }, []string{"kind", "state"}) - - liveLogReplayBytes = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "testkube_live_log_replay_bytes", - Help: "Approximate bytes held in live-log replay buffers", - }, []string{"kind"}) - - liveLogResumeTotal = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "testkube_live_log_resume_total", - Help: "Total live-log resume attempts by result", - }, []string{"kind", "result"}) - liveLogSessionsCreatedTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "testkube_live_log_sessions_created_total", Help: "Total live-log streaming sessions created", @@ -30,17 +23,138 @@ var ( liveLogSessionsEvictedTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "testkube_live_log_sessions_evicted_total", - Help: "Total live-log streaming sessions evicted by reason", + Help: "Total live-log streaming sessions removed from their manager by reason", }, []string{"kind", "reason"}) - liveLogSubscribers = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "testkube_live_log_subscribers", - Help: "Current number of live-log stream subscribers", - }, []string{"kind"}) + liveLogResumeTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "testkube_live_log_resume_total", + Help: "Total live-log resume attempts by result", + }, []string{"kind", "result"}) + // A source lives as long as the execution it follows, from seconds to hours, + // so the buckets cover that range instead of the sub-second defaults. liveLogSourceDurationSeconds = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "testkube_live_log_source_duration_seconds", - Help: "Duration of live-log source lifetimes in seconds", - Buckets: prometheus.DefBuckets, - }, []string{"kind"}) + Help: "Duration of live-log source lifetimes in seconds by result", + Buckets: []float64{1, 5, 15, 30, 60, 120, 300, 600, 1800, 3600, 7200}, + }, []string{"kind", "result"}) + + liveLogSessionsDesc = prometheus.NewDesc( + "testkube_live_log_sessions", + "Current number of live-log streaming sessions by state", + []string{"kind", "state"}, nil, + ) + liveLogReplayBytesDesc = prometheus.NewDesc( + "testkube_live_log_replay_bytes", + "Approximate bytes held in live-log replay buffers", + []string{"kind"}, nil, + ) + liveLogSubscribersDesc = prometheus.NewDesc( + "testkube_live_log_subscribers", + "Current number of live-log stream subscribers", + []string{"kind"}, nil, + ) + + // liveLogMetrics is the collector every session manager reports to. + liveLogMetrics = newLiveLogCollector() +) + +func init() { + prometheus.MustRegister(liveLogMetrics) +} + +const ( + liveLogEvictionReasonTTL = "ttl" + liveLogEvictionReasonErrored = "errored" + liveLogEvictionReasonReplaced = "replaced" + + liveLogResultOK = "ok" + liveLogResultError = "error" ) + +// liveLogStats is a point-in-time reading of one session manager. +type liveLogStats struct { + activeSessions int + doneSessions int + subscribers int + replayBytes int +} + +func (s liveLogStats) add(other liveLogStats) liveLogStats { + return liveLogStats{ + activeSessions: s.activeSessions + other.activeSessions, + doneSessions: s.doneSessions + other.doneSessions, + subscribers: s.subscribers + other.subscribers, + replayBytes: s.replayBytes + other.replayBytes, + } +} + +// liveLogStatsSource is a session manager as seen by the collector. +type liveLogStatsSource interface { + liveLogKind() string + liveLogStats() liveLogStats +} + +// liveLogCollector exposes the gauges of every registered session manager, +// summed by kind. Managers register on creation and leave when their context +// ends. +type liveLogCollector struct { + mu sync.Mutex + sources map[liveLogStatsSource]struct{} +} + +func newLiveLogCollector() *liveLogCollector { + return &liveLogCollector{sources: make(map[liveLogStatsSource]struct{})} +} + +func (c *liveLogCollector) add(source liveLogStatsSource) { + c.mu.Lock() + defer c.mu.Unlock() + c.sources[source] = struct{}{} +} + +func (c *liveLogCollector) remove(source liveLogStatsSource) { + c.mu.Lock() + defer c.mu.Unlock() + delete(c.sources, source) +} + +// totalsByKind reads every source outside the collector lock, because a +// source's stats take its own manager lock. +func (c *liveLogCollector) totalsByKind() map[string]liveLogStats { + c.mu.Lock() + sources := make([]liveLogStatsSource, 0, len(c.sources)) + for source := range c.sources { + sources = append(sources, source) + } + c.mu.Unlock() + + totals := make(map[string]liveLogStats) + for _, source := range sources { + kind := source.liveLogKind() + totals[kind] = totals[kind].add(source.liveLogStats()) + } + return totals +} + +func (c *liveLogCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- liveLogSessionsDesc + ch <- liveLogReplayBytesDesc + ch <- liveLogSubscribersDesc +} + +func (c *liveLogCollector) Collect(ch chan<- prometheus.Metric) { + totals := c.totalsByKind() + kinds := make([]string, 0, len(totals)) + for kind := range totals { + kinds = append(kinds, kind) + } + sort.Strings(kinds) + for _, kind := range kinds { + stats := totals[kind] + ch <- prometheus.MustNewConstMetric(liveLogSessionsDesc, prometheus.GaugeValue, float64(stats.activeSessions), kind, "active") + ch <- prometheus.MustNewConstMetric(liveLogSessionsDesc, prometheus.GaugeValue, float64(stats.doneSessions), kind, "done") + ch <- prometheus.MustNewConstMetric(liveLogReplayBytesDesc, prometheus.GaugeValue, float64(stats.replayBytes), kind) + ch <- prometheus.MustNewConstMetric(liveLogSubscribersDesc, prometheus.GaugeValue, float64(stats.subscribers), kind) + } +} diff --git a/pkg/controlplaneclient/testworkflows_test.go b/pkg/controlplaneclient/testworkflows_test.go index f860a3e20cb..f19767a59e0 100644 --- a/pkg/controlplaneclient/testworkflows_test.go +++ b/pkg/controlplaneclient/testworkflows_test.go @@ -3,6 +3,7 @@ package controlplaneclient import ( "context" "errors" + "strings" "sync" "sync/atomic" "testing" @@ -178,7 +179,7 @@ func TestSendNotificationResponseReturnsContextErrorWhenCanceled(t *testing.T) { } func TestNotificationStreamSessionPublishDoesNotHoldLockForSlowSubscriber(t *testing.T) { - session := newNotificationStreamSession(nil) + session := newNotificationStreamSession(defaultNotificationReplayLimits()) sub, _, _, _, _ := session.subscribe(0, 1) for i := 0; i < cap(sub.ch); i++ { sub.ch <- notificationStreamEvent{} @@ -214,7 +215,7 @@ func TestNotificationStreamSessionPublishDoesNotHoldLockForSlowSubscriber(t *tes } func TestWorkflowProtocolEventsDoNotAdvanceApplicationSeqNo(t *testing.T) { - session := newNotificationStreamSession(nil) + session := newNotificationStreamSession(defaultNotificationReplayLimits()) ready := buildCloudProtocol("stream-1", session.currentSeqNo(), cloud.TestWorkflowNotificationType_WORKFLOW_STREAM_READY, "") require.Equal(t, uint32(0), ready.SeqNo) @@ -229,7 +230,7 @@ func TestWorkflowProtocolEventsDoNotAdvanceApplicationSeqNo(t *testing.T) { } func TestNotificationStreamSessionReplayUnavailableForTrimmedCursor(t *testing.T) { - session := newNotificationStreamSession(nil) + session := newNotificationStreamSession(defaultNotificationReplayLimits()) for i := 0; i < workflowNotificationReplayMaxEvents+2; i++ { session.publish(&testkube.TestWorkflowExecutionNotification{Log: "log"}) } @@ -654,9 +655,9 @@ func TestNotificationStreamSessionReplacementCancelsOrphanedSource(t *testing.T) } } -// newMetricsTestSessionManager builds a manager whose kind label is the test name. -// The metrics live on the shared default registry, so a unique kind keeps each -// test's readings apart from the other tests in the package. +// newMetricsTestSessionManager builds a manager whose kind label is the test name, +// so the counter series it raises on the shared default registry are easy to tell +// apart. The gauges are read through liveLogStats and touch no shared state. func newMetricsTestSessionManager(t *testing.T, process func(context.Context, *cloud.TestWorkflowNotificationsRequest) NotificationWatcher) *notificationStreamSessionManager[*cloud.TestWorkflowNotificationsRequest] { t.Helper() @@ -682,7 +683,9 @@ func blockingNotificationSource(t *testing.T) (func(context.Context, *cloud.Test } var mu sync.Mutex ends := make(map[string]*sourceEnd) - process := func(_ context.Context, req *cloud.TestWorkflowNotificationsRequest) NotificationWatcher { + // The watcher also ends when the manager cancels the source context, the way + // a replaced session's source ends in production. + process := func(sourceCtx context.Context, req *cloud.TestWorkflowNotificationsRequest) NotificationWatcher { end := &sourceEnd{done: make(chan struct{})} mu.Lock() ends[req.ExecutionId] = end @@ -690,8 +693,12 @@ func blockingNotificationSource(t *testing.T) (func(context.Context, *cloud.Test watcher := channels.NewWatcher[*testkube.TestWorkflowExecutionNotification]() go func() { watcher.Send(&testkube.TestWorkflowExecutionNotification{Log: "line"}) - <-end.done - watcher.Close(end.err) + select { + case <-end.done: + watcher.Close(end.err) + case <-sourceCtx.Done(): + watcher.Close(nil) + } }() return watcher } @@ -721,26 +728,29 @@ func blockingNotificationSource(t *testing.T) (func(context.Context, *cloud.Test return process, finish } -func metricValue(t *testing.T, collector prometheus.Collector) float64 { +// counterSince returns the counter's growth since the call. The counters live on +// the shared default registry, so a test asserts what it caused and not an +// absolute value that another run with the same label may have raised. +func counterSince(t *testing.T, counter prometheus.Counter) func() float64 { t.Helper() - return testutil.ToFloat64(collector) + + base := testutil.ToFloat64(counter) + return func() float64 { return testutil.ToFloat64(counter) - base } } -// histogramSampleCount reads one label set's sample count. testutil.CollectAndCount -// on a curried vector counts every label set in the vector, so it cannot isolate a -// single kind on the shared registry. -func histogramSampleCount(t *testing.T, observer prometheus.Observer) uint64 { +// histogramSamplesSince is counterSince for one label set of a histogram. +// testutil.CollectAndCount on a curried vector counts every label set in the +// vector, so the sample count is read from the metric itself. +func histogramSamplesSince(t *testing.T, observer prometheus.Observer) func() uint64 { t.Helper() - var metric dto.Metric - require.NoError(t, observer.(prometheus.Metric).Write(&metric)) - return metric.GetHistogram().GetSampleCount() -} - -func sessionReplayBytes(session *notificationStreamSession) int { - session.mu.Lock() - defer session.mu.Unlock() - return session.replayBytes + read := func() uint64 { + var metric dto.Metric + require.NoError(t, observer.(prometheus.Metric).Write(&metric)) + return metric.GetHistogram().GetSampleCount() + } + base := read() + return func() uint64 { return read() - base } } func waitForNotificationSessionDone(t *testing.T, session *notificationStreamSession) { @@ -752,97 +762,95 @@ func waitForNotificationSessionDone(t *testing.T, session *notificationStreamSes }, 2*time.Second, time.Millisecond) } -func TestNotificationStreamSessionManagerMetricsFollowSessionLifecycle(t *testing.T) { +func waitForLiveLogStats(t *testing.T, manager *notificationStreamSessionManager[*cloud.TestWorkflowNotificationsRequest], want liveLogStats) { + t.Helper() + + require.Eventually(t, func() bool { + return manager.liveLogStats() == want + }, 2*time.Second, time.Millisecond, "stats did not settle at %+v, last %+v", want, manager.liveLogStats()) +} + +func TestNotificationStreamSessionManagerStatsFollowSessionLifecycle(t *testing.T) { process, finish := blockingNotificationSource(t) manager := newMetricsTestSessionManager(t, process) manager.sessionIdleTTL = time.Minute kind := manager.kind + created := counterSince(t, liveLogSessionsCreatedTotal.WithLabelValues(kind)) + evictedByTTL := counterSince(t, liveLogSessionsEvictedTotal.WithLabelValues(kind, liveLogEvictionReasonTTL)) + sourcesEndedOK := histogramSamplesSince(t, liveLogSourceDurationSeconds.WithLabelValues(kind, liveLogResultOK)) - // A fresh attach creates one active session with one subscriber. + // A fresh attach creates one active session with one subscriber. The source's + // first line lands in the replay buffer shortly after. session, sub, _, _, _, _ := manager.attach(&cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1"}) - assert.Equal(t, float64(1), metricValue(t, liveLogSessionsCreatedTotal.WithLabelValues(kind))) - assert.Equal(t, float64(1), metricValue(t, liveLogSessions.WithLabelValues(kind, "active"))) - assert.Equal(t, float64(0), metricValue(t, liveLogSessions.WithLabelValues(kind, "done"))) - assert.Equal(t, float64(1), metricValue(t, liveLogSubscribers.WithLabelValues(kind))) + assert.Equal(t, float64(1), created()) + lineBytes := approximateNotificationBytes(&testkube.TestWorkflowExecutionNotification{Log: "line"}) + waitForLiveLogStats(t, manager, liveLogStats{activeSessions: 1, subscribers: 1, replayBytes: lineBytes}) // A second viewer that resumes the same stream shares the session. _, sub2, _, _, _, _ := manager.attach(&cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1", ResumeAfterSeqNo: 1}) - assert.Equal(t, float64(1), metricValue(t, liveLogSessionsCreatedTotal.WithLabelValues(kind))) - assert.Equal(t, float64(2), metricValue(t, liveLogSubscribers.WithLabelValues(kind))) + assert.Equal(t, float64(1), created()) + assert.Equal(t, liveLogStats{activeSessions: 1, subscribers: 2, replayBytes: lineBytes}, manager.liveLogStats()) - // The source ends: the session moves from active to done and records its duration. + // The source ends: the session is done, its subscribers are closed, and its + // duration is recorded as a success. The replay buffer stays for late viewers. finish("exec-1", nil) waitForNotificationSessionDone(t, session) - require.Eventually(t, func() bool { - return metricValue(t, liveLogSessions.WithLabelValues(kind, "active")) == 0 && - metricValue(t, liveLogSessions.WithLabelValues(kind, "done")) == 1 && - histogramSampleCount(t, liveLogSourceDurationSeconds.WithLabelValues(kind)) == 1 - }, 2*time.Second, time.Millisecond) - assert.Greater(t, metricValue(t, liveLogReplayBytes.WithLabelValues(kind)), float64(0)) - - // Viewers leave: the subscriber gauge returns to zero. - manager.detach(session, sub) - manager.detach(session, sub2) - assert.Equal(t, float64(0), metricValue(t, liveLogSubscribers.WithLabelValues(kind))) + waitForLiveLogStats(t, manager, liveLogStats{doneSessions: 1, replayBytes: lineBytes}) + require.Eventually(t, func() bool { return sourcesEndedOK() == 1 }, 2*time.Second, time.Millisecond) + session.unsubscribe(sub) + session.unsubscribe(sub2) // The idle TTL passes: the sweep evicts the done session and frees its buffer. manager.sweepExpired(time.Now().Add(2 * manager.sessionIdleTTL)) - assert.Equal(t, float64(0), metricValue(t, liveLogSessions.WithLabelValues(kind, "done"))) - assert.Equal(t, float64(1), metricValue(t, liveLogSessionsEvictedTotal.WithLabelValues(kind, "ttl"))) - assert.Equal(t, float64(0), metricValue(t, liveLogReplayBytes.WithLabelValues(kind))) + assert.Equal(t, liveLogStats{}, manager.liveLogStats()) + assert.Equal(t, float64(1), evictedByTTL()) } -func TestNotificationStreamSessionManagerReplayBytesGaugeTracksBufferedBytes(t *testing.T) { +func TestNotificationStreamSessionManagerStatsReplayBytesFollowBuffer(t *testing.T) { process, finish := blockingNotificationSource(t) manager := newMetricsTestSessionManager(t, process) manager.sessionIdleTTL = time.Minute - kind := manager.kind - gauge := func() float64 { return metricValue(t, liveLogReplayBytes.WithLabelValues(kind)) } + manager.replayLimits = notificationReplayLimits{maxEvents: 3, maxBytes: workflowNotificationReplayMaxBytes} + bytesOf := func(logs ...string) int { + total := 0 + for _, log := range logs { + total += approximateNotificationBytes(&testkube.TestWorkflowExecutionNotification{Log: log}) + } + return total + } + replayBytes := func() int { return manager.liveLogStats().replayBytes } // The viewers leave at once: this test only reads the replay buffer, and an // unread subscription would block publish once its channel fills. first, firstSub, _, _, _, _ := manager.attach(&cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1"}) - manager.detach(first, firstSub) + first.unsubscribe(firstSub) + require.Eventually(t, func() bool { return replayBytes() == bytesOf("line") }, 2*time.Second, time.Millisecond) - // The source's first line lands in the buffer before the test publishes more, - // so every later reading compares a settled session against the gauge. - require.Eventually(t, func() bool { return sessionReplayBytes(first) > 0 }, 2*time.Second, time.Millisecond) - assert.Equal(t, float64(sessionReplayBytes(first)), gauge()) + // Every buffered notification adds its size. + first.publish(&testkube.TestWorkflowExecutionNotification{Log: "two"}) + first.publish(&testkube.TestWorkflowExecutionNotification{Log: "three"}) + assert.Equal(t, bytesOf("line", "two", "three"), replayBytes()) - // Every buffered notification adds its size to the gauge. - first.publish(&testkube.TestWorkflowExecutionNotification{Log: "hello"}) - assert.Equal(t, float64(sessionReplayBytes(first)), gauge()) + // Past the event limit the oldest event is dropped, and the bytes follow. + first.publish(&testkube.TestWorkflowExecutionNotification{Log: "four"}) + assert.Equal(t, bytesOf("two", "three", "four"), replayBytes()) - // Once the buffer trims old events, the gauge follows the trimmed total. - for i := 0; i < workflowNotificationReplayMaxEvents; i++ { - first.publish(&testkube.TestWorkflowExecutionNotification{Log: "fill"}) - } - first.mu.Lock() - replayLen := len(first.replay) - first.mu.Unlock() - require.Equal(t, workflowNotificationReplayMaxEvents, replayLen) - firstBytes := sessionReplayBytes(first) - assert.Equal(t, float64(firstBytes), gauge()) - - // Sessions of one kind share the gauge, so a second session adds to it. + // Sessions of one manager add up. second, secondSub, _, _, _, _ := manager.attach(&cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-2", StreamId: "stream-2"}) - manager.detach(second, secondSub) - require.Eventually(t, func() bool { return sessionReplayBytes(second) > 0 }, 2*time.Second, time.Millisecond) - second.publish(&testkube.TestWorkflowExecutionNotification{Log: "second"}) - secondBytes := sessionReplayBytes(second) - assert.Equal(t, float64(firstBytes+secondBytes), gauge()) + second.unsubscribe(secondSub) + require.Eventually(t, func() bool { return replayBytes() == bytesOf("two", "three", "four", "line") }, 2*time.Second, time.Millisecond) - // When the sweep evicts one session, the gauge drops by that session's bytes only. + // When the sweep evicts one session, only that session's bytes leave. finish("exec-1", nil) waitForNotificationSessionDone(t, first) manager.sweepExpired(time.Now().Add(2 * manager.sessionIdleTTL)) - assert.Equal(t, float64(secondBytes), gauge()) + assert.Equal(t, bytesOf("line"), replayBytes()) - // When the sweep evicts the last session, the gauge returns to zero. + // When the sweep evicts the last session, nothing is left. finish("exec-2", nil) waitForNotificationSessionDone(t, second) manager.sweepExpired(time.Now().Add(2 * manager.sessionIdleTTL)) - assert.Equal(t, float64(0), gauge()) + assert.Equal(t, 0, replayBytes()) } func TestNotificationStreamSessionManagerResumeMetricsCountResult(t *testing.T) { @@ -857,7 +865,7 @@ func TestNotificationStreamSessionManagerResumeMetricsCountResult(t *testing.T) name: "resume inside the replay buffer is available", prepare: func(t *testing.T, manager *notificationStreamSessionManager[*cloud.TestWorkflowNotificationsRequest]) { session, sub, _, _, _, _ := manager.attach(&cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1"}) - t.Cleanup(func() { manager.detach(session, sub) }) + t.Cleanup(func() { session.unsubscribe(sub) }) session.publish(&testkube.TestWorkflowExecutionNotification{Log: "two"}) }, req: &cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1", ResumeAfterSeqNo: 1}, @@ -885,39 +893,74 @@ func TestNotificationStreamSessionManagerResumeMetricsCountResult(t *testing.T) manager := newMetricsTestSessionManager(t, process) kind := manager.kind tt.prepare(t, manager) + available := counterSince(t, liveLogResumeTotal.WithLabelValues(kind, "available")) + unavailable := counterSince(t, liveLogResumeTotal.WithLabelValues(kind, "unavailable")) session, sub, _, _, _, _ := manager.attach(tt.req) - t.Cleanup(func() { manager.detach(session, sub) }) + t.Cleanup(func() { session.unsubscribe(sub) }) - assert.Equal(t, tt.available, metricValue(t, liveLogResumeTotal.WithLabelValues(kind, "available"))) - assert.Equal(t, tt.unavailable, metricValue(t, liveLogResumeTotal.WithLabelValues(kind, "unavailable"))) + assert.Equal(t, tt.available, available()) + assert.Equal(t, tt.unavailable, unavailable()) }) } } func TestNotificationStreamSessionManagerEvictionMetricsCountReason(t *testing.T) { + stream := &cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1"} tests := []struct { - name string - sourceErr error - evict func(t *testing.T, manager *notificationStreamSessionManager[*cloud.TestWorkflowNotificationsRequest]) - reason string + name string + // sourceErr ends the first session's source before the eviction, when set. + // finishFirst is false when the first source must still run at eviction time. + finishFirst bool + sourceErr error + evict func(t *testing.T, manager *notificationStreamSessionManager[*cloud.TestWorkflowNotificationsRequest]) + reason string + sourceResult string + want liveLogStats }{ { - name: "idle TTL sweep evicts a done session", - sourceErr: nil, + name: "idle TTL sweep evicts a done session", + finishFirst: true, evict: func(t *testing.T, manager *notificationStreamSessionManager[*cloud.TestWorkflowNotificationsRequest]) { manager.sweepExpired(time.Now().Add(2 * manager.sessionIdleTTL)) }, - reason: "ttl", + reason: liveLogEvictionReasonTTL, + sourceResult: liveLogResultOK, + want: liveLogStats{}, }, { - name: "resume after a failed source evicts the errored session", - sourceErr: errors.New("source failed"), + name: "resume after a failed source evicts the errored session", + finishFirst: true, + sourceErr: errors.New("source failed"), evict: func(t *testing.T, manager *notificationStreamSessionManager[*cloud.TestWorkflowNotificationsRequest]) { session, sub, _, _, _, _ := manager.attach(&cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1", ResumeAfterSeqNo: 1}) - t.Cleanup(func() { manager.detach(session, sub) }) + t.Cleanup(func() { session.unsubscribe(sub) }) + }, + reason: liveLogEvictionReasonErrored, + sourceResult: liveLogResultError, + want: liveLogStats{activeSessions: 1, subscribers: 1}, + }, + { + name: "start from zero replaces a done session", + finishFirst: true, + evict: func(t *testing.T, manager *notificationStreamSessionManager[*cloud.TestWorkflowNotificationsRequest]) { + session, sub, _, _, _, _ := manager.attach(stream) + t.Cleanup(func() { session.unsubscribe(sub) }) }, - reason: "error", + reason: liveLogEvictionReasonReplaced, + sourceResult: liveLogResultOK, + want: liveLogStats{activeSessions: 1, subscribers: 1}, + }, + { + name: "start from zero replaces a running session", + finishFirst: false, + evict: func(t *testing.T, manager *notificationStreamSessionManager[*cloud.TestWorkflowNotificationsRequest]) { + session, sub, _, _, _, _ := manager.attach(stream) + t.Cleanup(func() { session.unsubscribe(sub) }) + }, + reason: liveLogEvictionReasonReplaced, + sourceResult: liveLogResultOK, + want: liveLogStats{activeSessions: 1, subscribers: 1}, }, } for _, tt := range tests { @@ -926,23 +969,91 @@ func TestNotificationStreamSessionManagerEvictionMetricsCountReason(t *testing.T manager := newMetricsTestSessionManager(t, process) manager.sessionIdleTTL = time.Minute kind := manager.kind - - session, sub, _, _, _, _ := manager.attach(&cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1"}) - finish("exec-1", tt.sourceErr) - waitForNotificationSessionDone(t, session) - manager.detach(session, sub) - require.Eventually(t, func() bool { - return metricValue(t, liveLogSessions.WithLabelValues(kind, "done")) == 1 - }, 2*time.Second, time.Millisecond) + evicted := counterSince(t, liveLogSessionsEvictedTotal.WithLabelValues(kind, tt.reason)) + sourcesEnded := histogramSamplesSince(t, liveLogSourceDurationSeconds.WithLabelValues(kind, tt.sourceResult)) + + first, sub, _, _, _, _ := manager.attach(stream) + if tt.finishFirst { + finish("exec-1", tt.sourceErr) + waitForNotificationSessionDone(t, first) + first.unsubscribe(sub) + assert.Equal(t, liveLogStats{doneSessions: 1, replayBytes: manager.liveLogStats().replayBytes}, manager.liveLogStats()) + } tt.evict(t, manager) - assert.Equal(t, float64(1), metricValue(t, liveLogSessionsEvictedTotal.WithLabelValues(kind, tt.reason))) - assert.Equal(t, float64(0), metricValue(t, liveLogSessions.WithLabelValues(kind, "done"))) + assert.Equal(t, float64(1), evicted()) manager.mu.Lock() current := manager.sessions["exec-1:stream-1"] manager.mu.Unlock() - assert.NotSame(t, session, current, "evicted session must leave the manager") + assert.NotSame(t, first, current, "evicted session must leave the manager") + + // A replaced running source is cancelled and ends on its own. The stats + // settle once the new session's first line lands in the buffer. + require.Eventually(t, func() bool { + stats := manager.liveLogStats() + stats.replayBytes = 0 + return stats == tt.want + }, 2*time.Second, time.Millisecond, "last stats %+v", manager.liveLogStats()) + require.Eventually(t, func() bool { return sourcesEnded() >= 1 }, 2*time.Second, time.Millisecond) }) } } + +type fakeLiveLogStatsSource struct { + kind string + stats liveLogStats +} + +func (s fakeLiveLogStatsSource) liveLogKind() string { return s.kind } +func (s fakeLiveLogStatsSource) liveLogStats() liveLogStats { return s.stats } + +func TestLiveLogCollectorSumsSourcesByKind(t *testing.T) { + collector := newLiveLogCollector() + collector.add(fakeLiveLogStatsSource{kind: "workflow", stats: liveLogStats{activeSessions: 2, doneSessions: 1, subscribers: 3, replayBytes: 100}}) + collector.add(fakeLiveLogStatsSource{kind: "workflow", stats: liveLogStats{activeSessions: 1, subscribers: 1, replayBytes: 50}}) + collector.add(fakeLiveLogStatsSource{kind: "service", stats: liveLogStats{doneSessions: 4, replayBytes: 7}}) + gone := fakeLiveLogStatsSource{kind: "parallel", stats: liveLogStats{activeSessions: 9}} + collector.add(gone) + collector.remove(gone) + + expected := ` +# HELP testkube_live_log_replay_bytes Approximate bytes held in live-log replay buffers +# TYPE testkube_live_log_replay_bytes gauge +testkube_live_log_replay_bytes{kind="service"} 7 +testkube_live_log_replay_bytes{kind="workflow"} 150 +# HELP testkube_live_log_sessions Current number of live-log streaming sessions by state +# TYPE testkube_live_log_sessions gauge +testkube_live_log_sessions{kind="service",state="active"} 0 +testkube_live_log_sessions{kind="service",state="done"} 4 +testkube_live_log_sessions{kind="workflow",state="active"} 3 +testkube_live_log_sessions{kind="workflow",state="done"} 1 +# HELP testkube_live_log_subscribers Current number of live-log stream subscribers +# TYPE testkube_live_log_subscribers gauge +testkube_live_log_subscribers{kind="service"} 0 +testkube_live_log_subscribers{kind="workflow"} 4 +` + require.NoError(t, testutil.CollectAndCompare(collector, strings.NewReader(expected))) +} + +func TestNotificationStreamSessionManagerLeavesCollectorWhenContextEnds(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + manager := newNotificationStreamSessionManager( + ctx, + t.Name(), + func(req *cloud.TestWorkflowNotificationsRequest) string { return req.ExecutionId }, + func(context.Context, *cloud.TestWorkflowNotificationsRequest) NotificationWatcher { + return channels.NewWatcher[*testkube.TestWorkflowExecutionNotification]() + }, + ) + registered := func() bool { + liveLogMetrics.mu.Lock() + defer liveLogMetrics.mu.Unlock() + _, ok := liveLogMetrics.sources[manager] + return ok + } + require.True(t, registered()) + + cancel() + require.Eventually(t, func() bool { return !registered() }, 2*time.Second, time.Millisecond) +} diff --git a/pkg/controlplaneclient/utils.go b/pkg/controlplaneclient/utils.go index f767b8f65b6..0ffe30b0188 100644 --- a/pkg/controlplaneclient/utils.go +++ b/pkg/controlplaneclient/utils.go @@ -130,34 +130,30 @@ type notificationStreamSession struct { errored bool lastSeqNo uint32 lastActive time.Time - // onReplayBytes reports the signed change in this session's buffered bytes. - // Called under mu so a session's deltas are serialized. - onReplayBytes func(delta int) + limits notificationReplayLimits } -func newNotificationStreamSession(onReplayBytes func(delta int)) *notificationStreamSession { - return ¬ificationStreamSession{ - nextSeqNo: 1, - subscribers: make(map[uint64]*notificationStreamSubscription), - lastActive: time.Now(), - onReplayBytes: onReplayBytes, - } +// notificationReplayLimits bounds a session's replay buffer. The oldest events +// are dropped once either bound is exceeded. +type notificationReplayLimits struct { + maxEvents int + maxBytes int } -func (s *notificationStreamSession) reportReplayBytes(delta int) { - if delta != 0 && s.onReplayBytes != nil { - s.onReplayBytes(delta) +func defaultNotificationReplayLimits() notificationReplayLimits { + return notificationReplayLimits{ + maxEvents: workflowNotificationReplayMaxEvents, + maxBytes: workflowNotificationReplayMaxBytes, } } -// releaseReplayBytes drops the session's remaining buffered bytes from the -// accounting, once, when the session is removed from its manager. -func (s *notificationStreamSession) releaseReplayBytes() { - s.mu.Lock() - remaining := s.replayBytes - s.replayBytes = 0 - s.reportReplayBytes(-remaining) - s.mu.Unlock() +func newNotificationStreamSession(limits notificationReplayLimits) *notificationStreamSession { + return ¬ificationStreamSession{ + nextSeqNo: 1, + subscribers: make(map[uint64]*notificationStreamSubscription), + lastActive: time.Now(), + limits: limits, + } } func (s *notificationStreamSession) subscribe(resumeAfterSeqNo uint32, subscriptionID uint64) (*notificationStreamSubscription, []notificationStreamEvent, bool, uint32, bool) { @@ -205,16 +201,11 @@ func (s *notificationStreamSession) publish(notification *testkube.TestWorkflowE seqNo: seqNo, notification: notification, } - bytesDelta := 0 if seqNo > 0 { - added := approximateNotificationBytes(notification) s.replay = append(s.replay, event) - s.replayBytes += added - bytesDelta += added - for len(s.replay) > workflowNotificationReplayMaxEvents || s.replayBytes > workflowNotificationReplayMaxBytes { - evicted := approximateNotificationBytes(s.replay[0].notification) - s.replayBytes -= evicted - bytesDelta -= evicted + s.replayBytes += approximateNotificationBytes(notification) + for len(s.replay) > s.limits.maxEvents || s.replayBytes > s.limits.maxBytes { + s.replayBytes -= approximateNotificationBytes(s.replay[0].notification) s.replay[0].notification = nil s.replay = s.replay[1:] } @@ -223,7 +214,6 @@ func (s *notificationStreamSession) publish(notification *testkube.TestWorkflowE for _, sub := range s.subscribers { subscribers = append(subscribers, sub) } - s.reportReplayBytes(bytesDelta) s.mu.Unlock() for _, sub := range subscribers { @@ -330,6 +320,7 @@ type notificationStreamSessionManager[Request notificationRequest] struct { nextID atomic.Uint64 sessions map[string]*notificationStreamSession sessionIdleTTL time.Duration + replayLimits notificationReplayLimits kind string key func(Request) string process func(ctx context.Context, req Request) NotificationWatcher @@ -345,10 +336,12 @@ func newNotificationStreamSessionManager[Request notificationRequest]( ctx: ctx, sessions: make(map[string]*notificationStreamSession), sessionIdleTTL: workflowNotificationSessionIdleTTL, + replayLimits: defaultNotificationReplayLimits(), kind: kind, key: key, process: process, } + liveLogMetrics.add(m) go m.runSweeper(workflowNotificationSweepInterval) return m } @@ -356,13 +349,15 @@ func newNotificationStreamSessionManager[Request notificationRequest]( // runSweeper reclaims expired sessions on a fixed interval regardless of attach // traffic. This covers a manager that goes idle (attach never runs to sweep) and a // done session re-attached near its TTL boundary that escapes scheduleExpiration's -// single AfterFunc. It stops when the manager's context is done. +// single AfterFunc. It stops when the manager's context is done, and the manager +// leaves the metrics collector at that point. func (m *notificationStreamSessionManager[Request]) runSweeper(interval time.Duration) { ticker := time.NewTicker(interval) defer ticker.Stop() for { select { case <-m.ctx.Done(): + liveLogMetrics.remove(m) return case <-ticker.C: m.sweepExpired(time.Now()) @@ -370,15 +365,39 @@ func (m *notificationStreamSessionManager[Request]) runSweeper(interval time.Dur } } -// addReplayBytes applies a signed byte delta to the gauge. Every session of one -// kind shares the gauge, and each add is matched by that session's own release -// delta, so the gauge is the live total across sessions and returns to zero -// once every session drops, with no manager-wide recount. -func (m *notificationStreamSessionManager[Request]) addReplayBytes(delta int) { - if delta == 0 { +func (m *notificationStreamSessionManager[Request]) liveLogKind() string { + return m.kind +} + +// liveLogStats reads the manager's gauges from its sessions. The collector calls +// it at scrape time, so the gauges cannot drift from the sessions map. +func (m *notificationStreamSessionManager[Request]) liveLogStats() liveLogStats { + m.mu.Lock() + defer m.mu.Unlock() + + var stats liveLogStats + for _, session := range m.sessions { + session.mu.Lock() + if session.done { + stats.doneSessions++ + } else { + stats.activeSessions++ + } + stats.subscribers += len(session.subscribers) + stats.replayBytes += session.replayBytes + session.mu.Unlock() + } + return stats +} + +// removeSessionLocked is the only way a session leaves the sessions map, so every +// eviction is counted once with its reason. The caller holds m.mu. +func (m *notificationStreamSessionManager[Request]) removeSessionLocked(key string, session *notificationStreamSession, reason string) { + if m.sessions[key] != session { return } - liveLogReplayBytes.WithLabelValues(m.kind).Add(float64(delta)) + delete(m.sessions, key) + liveLogSessionsEvictedTotal.WithLabelValues(m.kind, reason).Inc() } func (m *notificationStreamSessionManager[Request]) sessionKey(req Request) string { @@ -391,20 +410,13 @@ func (m *notificationStreamSessionManager[Request]) sessionKey(req Request) stri // sweepExpired removes every session that is done and past the idle TTL as of now. func (m *notificationStreamSessionManager[Request]) sweepExpired(now time.Time) { - var releaseSessions []*notificationStreamSession m.mu.Lock() + defer m.mu.Unlock() for sessionKey, session := range m.sessions { if session.expired(now, m.sessionIdleTTL) { - delete(m.sessions, sessionKey) - releaseSessions = append(releaseSessions, session) - liveLogSessionsEvictedTotal.WithLabelValues(m.kind, "ttl").Inc() - liveLogSessions.WithLabelValues(m.kind, "done").Dec() + m.removeSessionLocked(sessionKey, session, liveLogEvictionReasonTTL) } } - m.mu.Unlock() - for _, s := range releaseSessions { - s.releaseReplayBytes() - } } func (m *notificationStreamSessionManager[Request]) attach(req Request) (*notificationStreamSession, *notificationStreamSubscription, []notificationStreamEvent, bool, uint32, bool) { @@ -419,29 +431,28 @@ func (m *notificationStreamSessionManager[Request]) attach(req Request) (*notifi // stopped: once replaced in the map it is unreachable and would otherwise run // orphaned until the execution ends. var replaced *notificationStreamSession - var releaseSessions []*notificationStreamSession session := m.sessions[key] if req.GetResumeAfterSeqNo() == 0 { + if session != nil { + m.removeSessionLocked(key, session, liveLogEvictionReasonReplaced) + } replaced = session session = nil } else if session != nil { done, errored := session.status() if done && errored { - delete(m.sessions, key) - releaseSessions = append(releaseSessions, session) - liveLogSessions.WithLabelValues(m.kind, "done").Dec() + m.removeSessionLocked(key, session, liveLogEvictionReasonErrored) session = nil } } freshSession := false var sourceCtx context.Context if session == nil { - session = newNotificationStreamSession(m.addReplayBytes) + session = newNotificationStreamSession(m.replayLimits) sourceCtx, session.cancel = context.WithCancel(m.ctx) m.sessions[key] = session freshSession = true liveLogSessionsCreatedTotal.WithLabelValues(m.kind).Inc() - liveLogSessions.WithLabelValues(m.kind, "active").Inc() } subscriptionID := m.nextID.Add(1) m.mu.Unlock() @@ -449,9 +460,6 @@ func (m *notificationStreamSessionManager[Request]) attach(req Request) (*notifi if replaced != nil { replaced.stopSource() } - for _, s := range releaseSessions { - s.releaseReplayBytes() - } subscribeAfterSeqNo := req.GetResumeAfterSeqNo() if freshSession && req.GetResumeAfterSeqNo() > 0 { @@ -475,31 +483,17 @@ func (m *notificationStreamSessionManager[Request]) attach(req Request) (*notifi } liveLogResumeTotal.WithLabelValues(m.kind, result).Inc() } - liveLogSubscribers.WithLabelValues(m.kind).Inc() return session, sub, replay, available, lastSeqNo, done } -func (m *notificationStreamSessionManager[Request]) detach(session *notificationStreamSession, sub *notificationStreamSubscription) { - session.unsubscribe(sub) - liveLogSubscribers.WithLabelValues(m.kind).Dec() -} - func (m *notificationStreamSessionManager[Request]) scheduleExpiration(key string, session *notificationStreamSession) { time.AfterFunc(m.sessionIdleTTL, func() { m.mu.Lock() - if m.sessions[key] != session { - m.mu.Unlock() - return - } - if !session.expired(time.Now(), m.sessionIdleTTL) { - m.mu.Unlock() - return + defer m.mu.Unlock() + + if session.expired(time.Now(), m.sessionIdleTTL) { + m.removeSessionLocked(key, session, liveLogEvictionReasonTTL) } - delete(m.sessions, key) - liveLogSessionsEvictedTotal.WithLabelValues(m.kind, "ttl").Inc() - liveLogSessions.WithLabelValues(m.kind, "done").Dec() - m.mu.Unlock() - session.releaseReplayBytes() }) } @@ -519,12 +513,11 @@ func (m *notificationStreamSessionManager[Request]) runSource(ctx context.Contex defer func() { session.close(sourceErr != nil) session.stopSource() - liveLogSessions.WithLabelValues(m.kind, "active").Dec() - liveLogSessions.WithLabelValues(m.kind, "done").Inc() - liveLogSourceDurationSeconds.WithLabelValues(m.kind).Observe(time.Since(started).Seconds()) + result := liveLogResultOK if sourceErr != nil { - liveLogSessionsEvictedTotal.WithLabelValues(m.kind, "error").Inc() + result = liveLogResultError } + liveLogSourceDurationSeconds.WithLabelValues(m.kind, result).Observe(time.Since(started).Seconds()) m.scheduleExpiration(key, session) }() @@ -681,7 +674,7 @@ func processNotifications[Request notificationRequest, Response any, Srv notific }() session, sub, replay, resumeAvailable, lastSeqNo, done := sessionManager.attach(req) - defer sessionManager.detach(session, sub) + defer session.unsubscribe(sub) // READY means the agent accepted this request and attached it to a logical stream session. // It does not imply the Kubernetes log source has already produced data. From d081be0799e366264a2c99e53b477e680a7fe0aa Mon Sep 17 00:00:00 2001 From: Dejan Zele Pejchev Date: Mon, 7 Sep 2026 13:37:31 +0200 Subject: [PATCH 6/7] test(runner): trim the live-log metrics tests --- pkg/controlplaneclient/testworkflows_test.go | 201 ++++++++----------- 1 file changed, 82 insertions(+), 119 deletions(-) diff --git a/pkg/controlplaneclient/testworkflows_test.go b/pkg/controlplaneclient/testworkflows_test.go index f19767a59e0..a35a5c77154 100644 --- a/pkg/controlplaneclient/testworkflows_test.go +++ b/pkg/controlplaneclient/testworkflows_test.go @@ -655,10 +655,12 @@ func TestNotificationStreamSessionReplacementCancelsOrphanedSource(t *testing.T) } } +type workflowSessionManager = notificationStreamSessionManager[*cloud.TestWorkflowNotificationsRequest] + // newMetricsTestSessionManager builds a manager whose kind label is the test name, // so the counter series it raises on the shared default registry are easy to tell // apart. The gauges are read through liveLogStats and touch no shared state. -func newMetricsTestSessionManager(t *testing.T, process func(context.Context, *cloud.TestWorkflowNotificationsRequest) NotificationWatcher) *notificationStreamSessionManager[*cloud.TestWorkflowNotificationsRequest] { +func newMetricsTestSessionManager(t *testing.T, process func(context.Context, *cloud.TestWorkflowNotificationsRequest) NotificationWatcher) *workflowSessionManager { t.Helper() ctx, cancel := context.WithCancel(context.Background()) @@ -671,10 +673,10 @@ func newMetricsTestSessionManager(t *testing.T, process func(context.Context, *c ) } -// blockingNotificationSource returns a process function whose watcher sends one -// log line and then stays open until the test finishes the execution id, with -// the error the source should end with. -func blockingNotificationSource(t *testing.T) (func(context.Context, *cloud.TestWorkflowNotificationsRequest) NotificationWatcher, func(executionID string, err error)) { +// silentNotificationSource returns a process function whose watcher sends nothing +// and ends when the test finishes the execution id with an error, or when the +// manager cancels the source, the way a replaced session's source ends. +func silentNotificationSource(t *testing.T) (func(context.Context, *cloud.TestWorkflowNotificationsRequest) NotificationWatcher, func(executionID string, err error)) { t.Helper() type sourceEnd struct { @@ -683,8 +685,6 @@ func blockingNotificationSource(t *testing.T) (func(context.Context, *cloud.Test } var mu sync.Mutex ends := make(map[string]*sourceEnd) - // The watcher also ends when the manager cancels the source context, the way - // a replaced session's source ends in production. process := func(sourceCtx context.Context, req *cloud.TestWorkflowNotificationsRequest) NotificationWatcher { end := &sourceEnd{done: make(chan struct{})} mu.Lock() @@ -692,7 +692,6 @@ func blockingNotificationSource(t *testing.T) (func(context.Context, *cloud.Test mu.Unlock() watcher := channels.NewWatcher[*testkube.TestWorkflowExecutionNotification]() go func() { - watcher.Send(&testkube.TestWorkflowExecutionNotification{Log: "line"}) select { case <-end.done: watcher.Close(end.err) @@ -714,17 +713,6 @@ func blockingNotificationSource(t *testing.T) (func(context.Context, *cloud.Test end.err = err close(end.done) } - t.Cleanup(func() { - mu.Lock() - defer mu.Unlock() - for _, end := range ends { - select { - case <-end.done: - default: - close(end.done) - } - } - }) return process, finish } @@ -753,6 +741,14 @@ func histogramSamplesSince(t *testing.T, observer prometheus.Observer) func() ui return func() uint64 { return read() - base } } +func notificationBytes(logs ...string) int { + total := 0 + for _, log := range logs { + total += approximateNotificationBytes(&testkube.TestWorkflowExecutionNotification{Log: log}) + } + return total +} + func waitForNotificationSessionDone(t *testing.T, session *notificationStreamSession) { t.Helper() @@ -762,7 +758,7 @@ func waitForNotificationSessionDone(t *testing.T, session *notificationStreamSes }, 2*time.Second, time.Millisecond) } -func waitForLiveLogStats(t *testing.T, manager *notificationStreamSessionManager[*cloud.TestWorkflowNotificationsRequest], want liveLogStats) { +func waitForLiveLogStats(t *testing.T, manager *workflowSessionManager, want liveLogStats) { t.Helper() require.Eventually(t, func() bool { @@ -771,31 +767,28 @@ func waitForLiveLogStats(t *testing.T, manager *notificationStreamSessionManager } func TestNotificationStreamSessionManagerStatsFollowSessionLifecycle(t *testing.T) { - process, finish := blockingNotificationSource(t) + process, finish := silentNotificationSource(t) manager := newMetricsTestSessionManager(t, process) manager.sessionIdleTTL = time.Minute - kind := manager.kind - created := counterSince(t, liveLogSessionsCreatedTotal.WithLabelValues(kind)) - evictedByTTL := counterSince(t, liveLogSessionsEvictedTotal.WithLabelValues(kind, liveLogEvictionReasonTTL)) - sourcesEndedOK := histogramSamplesSince(t, liveLogSourceDurationSeconds.WithLabelValues(kind, liveLogResultOK)) + created := counterSince(t, liveLogSessionsCreatedTotal.WithLabelValues(manager.kind)) + evictedByTTL := counterSince(t, liveLogSessionsEvictedTotal.WithLabelValues(manager.kind, liveLogEvictionReasonTTL)) + sourcesEndedOK := histogramSamplesSince(t, liveLogSourceDurationSeconds.WithLabelValues(manager.kind, liveLogResultOK)) - // A fresh attach creates one active session with one subscriber. The source's - // first line lands in the replay buffer shortly after. + // A fresh attach creates one active session with one subscriber. session, sub, _, _, _, _ := manager.attach(&cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1"}) assert.Equal(t, float64(1), created()) - lineBytes := approximateNotificationBytes(&testkube.TestWorkflowExecutionNotification{Log: "line"}) - waitForLiveLogStats(t, manager, liveLogStats{activeSessions: 1, subscribers: 1, replayBytes: lineBytes}) + assert.Equal(t, liveLogStats{activeSessions: 1, subscribers: 1}, manager.liveLogStats()) - // A second viewer that resumes the same stream shares the session. + // A second viewer that resumes the same stream shares the session and its buffer. + session.publish(&testkube.TestWorkflowExecutionNotification{Log: "one"}) _, sub2, _, _, _, _ := manager.attach(&cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1", ResumeAfterSeqNo: 1}) assert.Equal(t, float64(1), created()) - assert.Equal(t, liveLogStats{activeSessions: 1, subscribers: 2, replayBytes: lineBytes}, manager.liveLogStats()) + assert.Equal(t, liveLogStats{activeSessions: 1, subscribers: 2, replayBytes: notificationBytes("one")}, manager.liveLogStats()) // The source ends: the session is done, its subscribers are closed, and its // duration is recorded as a success. The replay buffer stays for late viewers. finish("exec-1", nil) - waitForNotificationSessionDone(t, session) - waitForLiveLogStats(t, manager, liveLogStats{doneSessions: 1, replayBytes: lineBytes}) + waitForLiveLogStats(t, manager, liveLogStats{doneSessions: 1, replayBytes: notificationBytes("one")}) require.Eventually(t, func() bool { return sourcesEndedOK() == 1 }, 2*time.Second, time.Millisecond) session.unsubscribe(sub) session.unsubscribe(sub2) @@ -807,44 +800,36 @@ func TestNotificationStreamSessionManagerStatsFollowSessionLifecycle(t *testing. } func TestNotificationStreamSessionManagerStatsReplayBytesFollowBuffer(t *testing.T) { - process, finish := blockingNotificationSource(t) + process, finish := silentNotificationSource(t) manager := newMetricsTestSessionManager(t, process) manager.sessionIdleTTL = time.Minute manager.replayLimits = notificationReplayLimits{maxEvents: 3, maxBytes: workflowNotificationReplayMaxBytes} - bytesOf := func(logs ...string) int { - total := 0 - for _, log := range logs { - total += approximateNotificationBytes(&testkube.TestWorkflowExecutionNotification{Log: log}) - } - return total - } replayBytes := func() int { return manager.liveLogStats().replayBytes } - // The viewers leave at once: this test only reads the replay buffer, and an - // unread subscription would block publish once its channel fills. + // Only the buffers matter here, so the viewers leave at once: an unread + // subscription would block publish once its channel fills. first, firstSub, _, _, _, _ := manager.attach(&cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1"}) first.unsubscribe(firstSub) - require.Eventually(t, func() bool { return replayBytes() == bytesOf("line") }, 2*time.Second, time.Millisecond) - - // Every buffered notification adds its size. - first.publish(&testkube.TestWorkflowExecutionNotification{Log: "two"}) - first.publish(&testkube.TestWorkflowExecutionNotification{Log: "three"}) - assert.Equal(t, bytesOf("line", "two", "three"), replayBytes()) + for _, log := range []string{"one", "two", "three"} { + first.publish(&testkube.TestWorkflowExecutionNotification{Log: log}) + } + assert.Equal(t, notificationBytes("one", "two", "three"), replayBytes()) // Past the event limit the oldest event is dropped, and the bytes follow. first.publish(&testkube.TestWorkflowExecutionNotification{Log: "four"}) - assert.Equal(t, bytesOf("two", "three", "four"), replayBytes()) + assert.Equal(t, notificationBytes("two", "three", "four"), replayBytes()) // Sessions of one manager add up. second, secondSub, _, _, _, _ := manager.attach(&cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-2", StreamId: "stream-2"}) second.unsubscribe(secondSub) - require.Eventually(t, func() bool { return replayBytes() == bytesOf("two", "three", "four", "line") }, 2*time.Second, time.Millisecond) + second.publish(&testkube.TestWorkflowExecutionNotification{Log: "five"}) + assert.Equal(t, notificationBytes("two", "three", "four", "five"), replayBytes()) // When the sweep evicts one session, only that session's bytes leave. finish("exec-1", nil) waitForNotificationSessionDone(t, first) manager.sweepExpired(time.Now().Add(2 * manager.sessionIdleTTL)) - assert.Equal(t, bytesOf("line"), replayBytes()) + assert.Equal(t, notificationBytes("five"), replayBytes()) // When the sweep evicts the last session, nothing is left. finish("exec-2", nil) @@ -856,45 +841,40 @@ func TestNotificationStreamSessionManagerStatsReplayBytesFollowBuffer(t *testing func TestNotificationStreamSessionManagerResumeMetricsCountResult(t *testing.T) { tests := []struct { name string - prepare func(t *testing.T, manager *notificationStreamSessionManager[*cloud.TestWorkflowNotificationsRequest]) + prepare func(t *testing.T, manager *workflowSessionManager) req *cloud.TestWorkflowNotificationsRequest available float64 unavailable float64 }{ { name: "resume inside the replay buffer is available", - prepare: func(t *testing.T, manager *notificationStreamSessionManager[*cloud.TestWorkflowNotificationsRequest]) { + prepare: func(t *testing.T, manager *workflowSessionManager) { session, sub, _, _, _, _ := manager.attach(&cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1"}) t.Cleanup(func() { session.unsubscribe(sub) }) - session.publish(&testkube.TestWorkflowExecutionNotification{Log: "two"}) + session.publish(&testkube.TestWorkflowExecutionNotification{Log: "one"}) }, - req: &cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1", ResumeAfterSeqNo: 1}, - available: 1, - unavailable: 0, + req: &cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1", ResumeAfterSeqNo: 1}, + available: 1, }, { name: "resume without a session is unavailable", - prepare: func(*testing.T, *notificationStreamSessionManager[*cloud.TestWorkflowNotificationsRequest]) {}, req: &cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1", ResumeAfterSeqNo: 5}, - available: 0, unavailable: 1, }, { - name: "start from zero is not a resume", - prepare: func(*testing.T, *notificationStreamSessionManager[*cloud.TestWorkflowNotificationsRequest]) {}, - req: &cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1"}, - available: 0, - unavailable: 0, + name: "start from zero is not a resume", + req: &cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1"}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - process, _ := blockingNotificationSource(t) + process, _ := silentNotificationSource(t) manager := newMetricsTestSessionManager(t, process) - kind := manager.kind - tt.prepare(t, manager) - available := counterSince(t, liveLogResumeTotal.WithLabelValues(kind, "available")) - unavailable := counterSince(t, liveLogResumeTotal.WithLabelValues(kind, "unavailable")) + if tt.prepare != nil { + tt.prepare(t, manager) + } + available := counterSince(t, liveLogResumeTotal.WithLabelValues(manager.kind, "available")) + unavailable := counterSince(t, liveLogResumeTotal.WithLabelValues(manager.kind, "unavailable")) session, sub, _, _, _, _ := manager.attach(tt.req) t.Cleanup(func() { session.unsubscribe(sub) }) @@ -907,57 +887,53 @@ func TestNotificationStreamSessionManagerResumeMetricsCountResult(t *testing.T) func TestNotificationStreamSessionManagerEvictionMetricsCountReason(t *testing.T) { stream := &cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1"} + attach := func(req *cloud.TestWorkflowNotificationsRequest) func(t *testing.T, manager *workflowSessionManager) { + return func(t *testing.T, manager *workflowSessionManager) { + session, sub, _, _, _, _ := manager.attach(req) + t.Cleanup(func() { session.unsubscribe(sub) }) + } + } + sweep := func(t *testing.T, manager *workflowSessionManager) { + manager.sweepExpired(time.Now().Add(2 * manager.sessionIdleTTL)) + } tests := []struct { name string - // sourceErr ends the first session's source before the eviction, when set. - // finishFirst is false when the first source must still run at eviction time. + // finishFirst ends the first source with sourceErr before the eviction. + // When false, the first source is still running at eviction time. finishFirst bool sourceErr error - evict func(t *testing.T, manager *notificationStreamSessionManager[*cloud.TestWorkflowNotificationsRequest]) + evict func(t *testing.T, manager *workflowSessionManager) reason string sourceResult string want liveLogStats }{ { - name: "idle TTL sweep evicts a done session", - finishFirst: true, - evict: func(t *testing.T, manager *notificationStreamSessionManager[*cloud.TestWorkflowNotificationsRequest]) { - manager.sweepExpired(time.Now().Add(2 * manager.sessionIdleTTL)) - }, + name: "idle TTL sweep evicts a done session", + finishFirst: true, + evict: sweep, reason: liveLogEvictionReasonTTL, sourceResult: liveLogResultOK, - want: liveLogStats{}, }, { - name: "resume after a failed source evicts the errored session", - finishFirst: true, - sourceErr: errors.New("source failed"), - evict: func(t *testing.T, manager *notificationStreamSessionManager[*cloud.TestWorkflowNotificationsRequest]) { - session, sub, _, _, _, _ := manager.attach(&cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1", ResumeAfterSeqNo: 1}) - t.Cleanup(func() { session.unsubscribe(sub) }) - }, + name: "resume after a failed source evicts the errored session", + finishFirst: true, + sourceErr: errors.New("source failed"), + evict: attach(&cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1", ResumeAfterSeqNo: 1}), reason: liveLogEvictionReasonErrored, sourceResult: liveLogResultError, want: liveLogStats{activeSessions: 1, subscribers: 1}, }, { - name: "start from zero replaces a done session", - finishFirst: true, - evict: func(t *testing.T, manager *notificationStreamSessionManager[*cloud.TestWorkflowNotificationsRequest]) { - session, sub, _, _, _, _ := manager.attach(stream) - t.Cleanup(func() { session.unsubscribe(sub) }) - }, + name: "start from zero replaces a done session", + finishFirst: true, + evict: attach(stream), reason: liveLogEvictionReasonReplaced, sourceResult: liveLogResultOK, want: liveLogStats{activeSessions: 1, subscribers: 1}, }, { - name: "start from zero replaces a running session", - finishFirst: false, - evict: func(t *testing.T, manager *notificationStreamSessionManager[*cloud.TestWorkflowNotificationsRequest]) { - session, sub, _, _, _, _ := manager.attach(stream) - t.Cleanup(func() { session.unsubscribe(sub) }) - }, + name: "start from zero replaces a running session", + evict: attach(stream), reason: liveLogEvictionReasonReplaced, sourceResult: liveLogResultOK, want: liveLogStats{activeSessions: 1, subscribers: 1}, @@ -965,19 +941,17 @@ func TestNotificationStreamSessionManagerEvictionMetricsCountReason(t *testing.T } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - process, finish := blockingNotificationSource(t) + process, finish := silentNotificationSource(t) manager := newMetricsTestSessionManager(t, process) manager.sessionIdleTTL = time.Minute - kind := manager.kind - evicted := counterSince(t, liveLogSessionsEvictedTotal.WithLabelValues(kind, tt.reason)) - sourcesEnded := histogramSamplesSince(t, liveLogSourceDurationSeconds.WithLabelValues(kind, tt.sourceResult)) + evicted := counterSince(t, liveLogSessionsEvictedTotal.WithLabelValues(manager.kind, tt.reason)) + sourcesEnded := histogramSamplesSince(t, liveLogSourceDurationSeconds.WithLabelValues(manager.kind, tt.sourceResult)) first, sub, _, _, _, _ := manager.attach(stream) if tt.finishFirst { finish("exec-1", tt.sourceErr) waitForNotificationSessionDone(t, first) first.unsubscribe(sub) - assert.Equal(t, liveLogStats{doneSessions: 1, replayBytes: manager.liveLogStats().replayBytes}, manager.liveLogStats()) } tt.evict(t, manager) @@ -988,14 +962,9 @@ func TestNotificationStreamSessionManagerEvictionMetricsCountReason(t *testing.T manager.mu.Unlock() assert.NotSame(t, first, current, "evicted session must leave the manager") - // A replaced running source is cancelled and ends on its own. The stats - // settle once the new session's first line lands in the buffer. - require.Eventually(t, func() bool { - stats := manager.liveLogStats() - stats.replayBytes = 0 - return stats == tt.want - }, 2*time.Second, time.Millisecond, "last stats %+v", manager.liveLogStats()) - require.Eventually(t, func() bool { return sourcesEnded() >= 1 }, 2*time.Second, time.Millisecond) + // A replaced running source ends through cancellation on its own. + waitForLiveLogStats(t, manager, tt.want) + require.Eventually(t, func() bool { return sourcesEnded() == 1 }, 2*time.Second, time.Millisecond) }) } } @@ -1038,14 +1007,8 @@ testkube_live_log_subscribers{kind="workflow"} 4 func TestNotificationStreamSessionManagerLeavesCollectorWhenContextEnds(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) - manager := newNotificationStreamSessionManager( - ctx, - t.Name(), - func(req *cloud.TestWorkflowNotificationsRequest) string { return req.ExecutionId }, - func(context.Context, *cloud.TestWorkflowNotificationsRequest) NotificationWatcher { - return channels.NewWatcher[*testkube.TestWorkflowExecutionNotification]() - }, - ) + process, _ := silentNotificationSource(t) + manager := newNotificationStreamSessionManager(ctx, t.Name(), func(req *cloud.TestWorkflowNotificationsRequest) string { return req.ExecutionId }, process) registered := func() bool { liveLogMetrics.mu.Lock() defer liveLogMetrics.mu.Unlock() From 4ed9810f2da889680c41a65c007459c7766a6ff6 Mon Sep 17 00:00:00 2001 From: Dejan Zele Pejchev Date: Mon, 7 Sep 2026 13:55:37 +0200 Subject: [PATCH 7/7] test(runner): merge the live-log metrics tests into a lifecycle and an attach table --- pkg/controlplaneclient/testworkflows_test.go | 348 +++++++++---------- 1 file changed, 161 insertions(+), 187 deletions(-) diff --git a/pkg/controlplaneclient/testworkflows_test.go b/pkg/controlplaneclient/testworkflows_test.go index a35a5c77154..0dbc5b8290e 100644 --- a/pkg/controlplaneclient/testworkflows_test.go +++ b/pkg/controlplaneclient/testworkflows_test.go @@ -660,17 +660,17 @@ type workflowSessionManager = notificationStreamSessionManager[*cloud.TestWorkfl // newMetricsTestSessionManager builds a manager whose kind label is the test name, // so the counter series it raises on the shared default registry are easy to tell // apart. The gauges are read through liveLogStats and touch no shared state. -func newMetricsTestSessionManager(t *testing.T, process func(context.Context, *cloud.TestWorkflowNotificationsRequest) NotificationWatcher) *workflowSessionManager { +func newMetricsTestSessionManager(t *testing.T, ctx context.Context, process func(context.Context, *cloud.TestWorkflowNotificationsRequest) NotificationWatcher) *workflowSessionManager { t.Helper() - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - return newNotificationStreamSessionManager( + manager := newNotificationStreamSessionManager( ctx, t.Name(), func(req *cloud.TestWorkflowNotificationsRequest) string { return req.ExecutionId }, process, ) + manager.sessionIdleTTL = time.Minute + return manager } // silentNotificationSource returns a process function whose watcher sends nothing @@ -716,29 +716,55 @@ func silentNotificationSource(t *testing.T) (func(context.Context, *cloud.TestWo return process, finish } -// counterSince returns the counter's growth since the call. The counters live on -// the shared default registry, so a test asserts what it caused and not an -// absolute value that another run with the same label may have raised. -func counterSince(t *testing.T, counter prometheus.Counter) func() float64 { - t.Helper() - - base := testutil.ToFloat64(counter) - return func() float64 { return testutil.ToFloat64(counter) - base } +// liveLogCounters is the growth of every live-log counter series of one kind. +type liveLogCounters struct { + created float64 + resumeAvailable float64 + resumeUnavailable float64 + evictedTTL float64 + evictedErrored float64 + evictedReplaced float64 + sourcesEndedOK float64 + sourcesEndedError float64 } -// histogramSamplesSince is counterSince for one label set of a histogram. -// testutil.CollectAndCount on a curried vector counts every label set in the -// vector, so the sample count is read from the metric itself. -func histogramSamplesSince(t *testing.T, observer prometheus.Observer) func() uint64 { +// liveLogCountersSince returns the counters' growth since the call. The counters +// live on the shared default registry, so a test asserts what it caused and not +// an absolute value that another run with the same label may have raised. +func liveLogCountersSince(t *testing.T, kind string) func() liveLogCounters { t.Helper() - read := func() uint64 { + sampleCount := func(result string) float64 { var metric dto.Metric - require.NoError(t, observer.(prometheus.Metric).Write(&metric)) - return metric.GetHistogram().GetSampleCount() + require.NoError(t, liveLogSourceDurationSeconds.WithLabelValues(kind, result).(prometheus.Metric).Write(&metric)) + return float64(metric.GetHistogram().GetSampleCount()) + } + read := func() liveLogCounters { + return liveLogCounters{ + created: testutil.ToFloat64(liveLogSessionsCreatedTotal.WithLabelValues(kind)), + resumeAvailable: testutil.ToFloat64(liveLogResumeTotal.WithLabelValues(kind, "available")), + resumeUnavailable: testutil.ToFloat64(liveLogResumeTotal.WithLabelValues(kind, "unavailable")), + evictedTTL: testutil.ToFloat64(liveLogSessionsEvictedTotal.WithLabelValues(kind, liveLogEvictionReasonTTL)), + evictedErrored: testutil.ToFloat64(liveLogSessionsEvictedTotal.WithLabelValues(kind, liveLogEvictionReasonErrored)), + evictedReplaced: testutil.ToFloat64(liveLogSessionsEvictedTotal.WithLabelValues(kind, liveLogEvictionReasonReplaced)), + sourcesEndedOK: sampleCount(liveLogResultOK), + sourcesEndedError: sampleCount(liveLogResultError), + } } base := read() - return func() uint64 { return read() - base } + return func() liveLogCounters { + now := read() + return liveLogCounters{ + created: now.created - base.created, + resumeAvailable: now.resumeAvailable - base.resumeAvailable, + resumeUnavailable: now.resumeUnavailable - base.resumeUnavailable, + evictedTTL: now.evictedTTL - base.evictedTTL, + evictedErrored: now.evictedErrored - base.evictedErrored, + evictedReplaced: now.evictedReplaced - base.evictedReplaced, + sourcesEndedOK: now.sourcesEndedOK - base.sourcesEndedOK, + sourcesEndedError: now.sourcesEndedError - base.sourcesEndedError, + } + } } func notificationBytes(logs ...string) int { @@ -758,217 +784,181 @@ func waitForNotificationSessionDone(t *testing.T, session *notificationStreamSes }, 2*time.Second, time.Millisecond) } -func waitForLiveLogStats(t *testing.T, manager *workflowSessionManager, want liveLogStats) { +// waitForLiveLogMetrics waits for the counters and stats to settle, because a +// source ends on its own goroutine after the test's last call. +func waitForLiveLogMetrics(t *testing.T, manager *workflowSessionManager, counters func() liveLogCounters, wantCounters liveLogCounters, wantStats liveLogStats) { t.Helper() require.Eventually(t, func() bool { - return manager.liveLogStats() == want - }, 2*time.Second, time.Millisecond, "stats did not settle at %+v, last %+v", want, manager.liveLogStats()) + return counters() == wantCounters && manager.liveLogStats() == wantStats + }, 2*time.Second, time.Millisecond, "metrics did not settle: counters %+v stats %+v", counters(), manager.liveLogStats()) } -func TestNotificationStreamSessionManagerStatsFollowSessionLifecycle(t *testing.T) { +func TestNotificationStreamSessionManagerMetricsFollowSessionLifecycle(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) process, finish := silentNotificationSource(t) - manager := newMetricsTestSessionManager(t, process) - manager.sessionIdleTTL = time.Minute - created := counterSince(t, liveLogSessionsCreatedTotal.WithLabelValues(manager.kind)) - evictedByTTL := counterSince(t, liveLogSessionsEvictedTotal.WithLabelValues(manager.kind, liveLogEvictionReasonTTL)) - sourcesEndedOK := histogramSamplesSince(t, liveLogSourceDurationSeconds.WithLabelValues(manager.kind, liveLogResultOK)) + manager := newMetricsTestSessionManager(t, ctx, process) + counters := liveLogCountersSince(t, manager.kind) // A fresh attach creates one active session with one subscriber. session, sub, _, _, _, _ := manager.attach(&cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1"}) - assert.Equal(t, float64(1), created()) + want := liveLogCounters{created: 1} + assert.Equal(t, want, counters()) assert.Equal(t, liveLogStats{activeSessions: 1, subscribers: 1}, manager.liveLogStats()) // A second viewer that resumes the same stream shares the session and its buffer. session.publish(&testkube.TestWorkflowExecutionNotification{Log: "one"}) _, sub2, _, _, _, _ := manager.attach(&cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1", ResumeAfterSeqNo: 1}) - assert.Equal(t, float64(1), created()) + want.resumeAvailable = 1 + assert.Equal(t, want, counters()) assert.Equal(t, liveLogStats{activeSessions: 1, subscribers: 2, replayBytes: notificationBytes("one")}, manager.liveLogStats()) // The source ends: the session is done, its subscribers are closed, and its // duration is recorded as a success. The replay buffer stays for late viewers. finish("exec-1", nil) - waitForLiveLogStats(t, manager, liveLogStats{doneSessions: 1, replayBytes: notificationBytes("one")}) - require.Eventually(t, func() bool { return sourcesEndedOK() == 1 }, 2*time.Second, time.Millisecond) + want.sourcesEndedOK = 1 + waitForLiveLogMetrics(t, manager, counters, want, liveLogStats{doneSessions: 1, replayBytes: notificationBytes("one")}) session.unsubscribe(sub) session.unsubscribe(sub2) // The idle TTL passes: the sweep evicts the done session and frees its buffer. manager.sweepExpired(time.Now().Add(2 * manager.sessionIdleTTL)) + want.evictedTTL = 1 + assert.Equal(t, want, counters()) assert.Equal(t, liveLogStats{}, manager.liveLogStats()) - assert.Equal(t, float64(1), evictedByTTL()) -} -func TestNotificationStreamSessionManagerStatsReplayBytesFollowBuffer(t *testing.T) { - process, finish := silentNotificationSource(t) - manager := newMetricsTestSessionManager(t, process) - manager.sessionIdleTTL = time.Minute - manager.replayLimits = notificationReplayLimits{maxEvents: 3, maxBytes: workflowNotificationReplayMaxBytes} - replayBytes := func() int { return manager.liveLogStats().replayBytes } - - // Only the buffers matter here, so the viewers leave at once: an unread - // subscription would block publish once its channel fills. - first, firstSub, _, _, _, _ := manager.attach(&cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1"}) - first.unsubscribe(firstSub) - for _, log := range []string{"one", "two", "three"} { - first.publish(&testkube.TestWorkflowExecutionNotification{Log: log}) + // The manager's context ends: it stops reporting to the collector. + registered := func() bool { + liveLogMetrics.mu.Lock() + defer liveLogMetrics.mu.Unlock() + _, ok := liveLogMetrics.sources[manager] + return ok } - assert.Equal(t, notificationBytes("one", "two", "three"), replayBytes()) - - // Past the event limit the oldest event is dropped, and the bytes follow. - first.publish(&testkube.TestWorkflowExecutionNotification{Log: "four"}) - assert.Equal(t, notificationBytes("two", "three", "four"), replayBytes()) - - // Sessions of one manager add up. - second, secondSub, _, _, _, _ := manager.attach(&cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-2", StreamId: "stream-2"}) - second.unsubscribe(secondSub) - second.publish(&testkube.TestWorkflowExecutionNotification{Log: "five"}) - assert.Equal(t, notificationBytes("two", "three", "four", "five"), replayBytes()) + require.True(t, registered()) + cancel() + require.Eventually(t, func() bool { return !registered() }, 2*time.Second, time.Millisecond) +} - // When the sweep evicts one session, only that session's bytes leave. - finish("exec-1", nil) - waitForNotificationSessionDone(t, first) - manager.sweepExpired(time.Now().Add(2 * manager.sessionIdleTTL)) - assert.Equal(t, notificationBytes("five"), replayBytes()) +// priorSessionState is the state of the session a request finds under its key. +type priorSessionState int - // When the sweep evicts the last session, nothing is left. - finish("exec-2", nil) - waitForNotificationSessionDone(t, second) - manager.sweepExpired(time.Now().Add(2 * manager.sessionIdleTTL)) - assert.Equal(t, 0, replayBytes()) -} +const ( + noPriorSession priorSessionState = iota + runningPriorSession + donePriorSession + erroredPriorSession +) -func TestNotificationStreamSessionManagerResumeMetricsCountResult(t *testing.T) { +func TestNotificationStreamSessionManagerMetricsFollowAttach(t *testing.T) { + start := &cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1"} + resume := &cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1", ResumeAfterSeqNo: 1} tests := []struct { - name string - prepare func(t *testing.T, manager *workflowSessionManager) - req *cloud.TestWorkflowNotificationsRequest - available float64 - unavailable float64 + name string + prior priorSessionState + req *cloud.TestWorkflowNotificationsRequest + wantCounters liveLogCounters + wantStats liveLogStats + // joinsPrior is true when the request must attach to the prior session + // instead of creating a new one. + joinsPrior bool }{ { - name: "resume inside the replay buffer is available", - prepare: func(t *testing.T, manager *workflowSessionManager) { - session, sub, _, _, _, _ := manager.attach(&cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1"}) - t.Cleanup(func() { session.unsubscribe(sub) }) - session.publish(&testkube.TestWorkflowExecutionNotification{Log: "one"}) - }, - req: &cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1", ResumeAfterSeqNo: 1}, - available: 1, + name: "fresh attach creates a session", + prior: noPriorSession, + req: start, + wantCounters: liveLogCounters{created: 1}, + wantStats: liveLogStats{activeSessions: 1, subscribers: 1}, }, { - name: "resume without a session is unavailable", - req: &cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1", ResumeAfterSeqNo: 5}, - unavailable: 1, + name: "resume inside the replay buffer joins the session", + prior: runningPriorSession, + req: resume, + wantCounters: liveLogCounters{resumeAvailable: 1}, + wantStats: liveLogStats{activeSessions: 1, subscribers: 2, replayBytes: notificationBytes("one")}, + joinsPrior: true, }, { - name: "start from zero is not a resume", - req: &cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1"}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - process, _ := silentNotificationSource(t) - manager := newMetricsTestSessionManager(t, process) - if tt.prepare != nil { - tt.prepare(t, manager) - } - available := counterSince(t, liveLogResumeTotal.WithLabelValues(manager.kind, "available")) - unavailable := counterSince(t, liveLogResumeTotal.WithLabelValues(manager.kind, "unavailable")) - - session, sub, _, _, _, _ := manager.attach(tt.req) - t.Cleanup(func() { session.unsubscribe(sub) }) - - assert.Equal(t, tt.available, available()) - assert.Equal(t, tt.unavailable, unavailable()) - }) - } -} - -func TestNotificationStreamSessionManagerEvictionMetricsCountReason(t *testing.T) { - stream := &cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1"} - attach := func(req *cloud.TestWorkflowNotificationsRequest) func(t *testing.T, manager *workflowSessionManager) { - return func(t *testing.T, manager *workflowSessionManager) { - session, sub, _, _, _, _ := manager.attach(req) - t.Cleanup(func() { session.unsubscribe(sub) }) - } - } - sweep := func(t *testing.T, manager *workflowSessionManager) { - manager.sweepExpired(time.Now().Add(2 * manager.sessionIdleTTL)) - } - tests := []struct { - name string - // finishFirst ends the first source with sourceErr before the eviction. - // When false, the first source is still running at eviction time. - finishFirst bool - sourceErr error - evict func(t *testing.T, manager *workflowSessionManager) - reason string - sourceResult string - want liveLogStats - }{ - { - name: "idle TTL sweep evicts a done session", - finishFirst: true, - evict: sweep, - reason: liveLogEvictionReasonTTL, - sourceResult: liveLogResultOK, + name: "resume without a session starts fresh and is unavailable", + prior: noPriorSession, + req: resume, + wantCounters: liveLogCounters{created: 1, resumeUnavailable: 1}, + wantStats: liveLogStats{activeSessions: 1, subscribers: 1}, }, { name: "resume after a failed source evicts the errored session", - finishFirst: true, - sourceErr: errors.New("source failed"), - evict: attach(&cloud.TestWorkflowNotificationsRequest{ExecutionId: "exec-1", StreamId: "stream-1", ResumeAfterSeqNo: 1}), - reason: liveLogEvictionReasonErrored, - sourceResult: liveLogResultError, - want: liveLogStats{activeSessions: 1, subscribers: 1}, + prior: erroredPriorSession, + req: resume, + wantCounters: liveLogCounters{created: 1, resumeUnavailable: 1, evictedErrored: 1}, + wantStats: liveLogStats{activeSessions: 1, subscribers: 1}, }, { name: "start from zero replaces a done session", - finishFirst: true, - evict: attach(stream), - reason: liveLogEvictionReasonReplaced, - sourceResult: liveLogResultOK, - want: liveLogStats{activeSessions: 1, subscribers: 1}, + prior: donePriorSession, + req: start, + wantCounters: liveLogCounters{created: 1, evictedReplaced: 1}, + wantStats: liveLogStats{activeSessions: 1, subscribers: 1}, }, { - name: "start from zero replaces a running session", - evict: attach(stream), - reason: liveLogEvictionReasonReplaced, - sourceResult: liveLogResultOK, - want: liveLogStats{activeSessions: 1, subscribers: 1}, + name: "start from zero replaces a running session and cancels its source", + prior: runningPriorSession, + req: start, + wantCounters: liveLogCounters{created: 1, evictedReplaced: 1, sourcesEndedOK: 1}, + wantStats: liveLogStats{activeSessions: 1, subscribers: 1}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) process, finish := silentNotificationSource(t) - manager := newMetricsTestSessionManager(t, process) - manager.sessionIdleTTL = time.Minute - evicted := counterSince(t, liveLogSessionsEvictedTotal.WithLabelValues(manager.kind, tt.reason)) - sourcesEnded := histogramSamplesSince(t, liveLogSourceDurationSeconds.WithLabelValues(manager.kind, tt.sourceResult)) - - first, sub, _, _, _, _ := manager.attach(stream) - if tt.finishFirst { - finish("exec-1", tt.sourceErr) - waitForNotificationSessionDone(t, first) - first.unsubscribe(sub) + manager := newMetricsTestSessionManager(t, ctx, process) + + var prior *notificationStreamSession + if tt.prior != noPriorSession { + var sub *notificationStreamSubscription + prior, sub, _, _, _, _ = manager.attach(start) + t.Cleanup(func() { prior.unsubscribe(sub) }) + prior.publish(&testkube.TestWorkflowExecutionNotification{Log: "one"}) } + switch tt.prior { + case donePriorSession: + finish("exec-1", nil) + waitForNotificationSessionDone(t, prior) + case erroredPriorSession: + finish("exec-1", errors.New("source failed")) + waitForNotificationSessionDone(t, prior) + } + counters := liveLogCountersSince(t, manager.kind) - tt.evict(t, manager) + session, sub, _, _, _, _ := manager.attach(tt.req) + t.Cleanup(func() { session.unsubscribe(sub) }) - assert.Equal(t, float64(1), evicted()) + waitForLiveLogMetrics(t, manager, counters, tt.wantCounters, tt.wantStats) manager.mu.Lock() current := manager.sessions["exec-1:stream-1"] manager.mu.Unlock() - assert.NotSame(t, first, current, "evicted session must leave the manager") - - // A replaced running source ends through cancellation on its own. - waitForLiveLogStats(t, manager, tt.want) - require.Eventually(t, func() bool { return sourcesEnded() == 1 }, 2*time.Second, time.Millisecond) + if tt.joinsPrior { + assert.Same(t, prior, current, "the request must join the prior session") + } else { + assert.NotSame(t, prior, current, "the prior session must leave the manager") + } }) } } +func TestNotificationStreamSessionReplayBytesFollowTrimming(t *testing.T) { + session := newNotificationStreamSession(notificationReplayLimits{maxEvents: 3, maxBytes: workflowNotificationReplayMaxBytes}) + for _, log := range []string{"one", "two", "three", "four"} { + session.publish(&testkube.TestWorkflowExecutionNotification{Log: log}) + } + + session.mu.Lock() + defer session.mu.Unlock() + require.Len(t, session.replay, 3) + require.Equal(t, notificationBytes("two", "three", "four"), session.replayBytes) +} + type fakeLiveLogStatsSource struct { kind string stats liveLogStats @@ -1004,19 +994,3 @@ testkube_live_log_subscribers{kind="workflow"} 4 ` require.NoError(t, testutil.CollectAndCompare(collector, strings.NewReader(expected))) } - -func TestNotificationStreamSessionManagerLeavesCollectorWhenContextEnds(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - process, _ := silentNotificationSource(t) - manager := newNotificationStreamSessionManager(ctx, t.Name(), func(req *cloud.TestWorkflowNotificationsRequest) string { return req.ExecutionId }, process) - registered := func() bool { - liveLogMetrics.mu.Lock() - defer liveLogMetrics.mu.Unlock() - _, ok := liveLogMetrics.sources[manager] - return ok - } - require.True(t, registered()) - - cancel() - require.Eventually(t, func() bool { return !registered() }, 2*time.Second, time.Millisecond) -}