Cap append-only JSONL logs via a shared dream-driven retention pass

Directory.Build.props

@@ -9,7 +9,7 @@
     <!-- Version can be overridden from the command line: -p:Version=0.3.1
          AssemblyVersion and FileVersion are derived automatically by the SDK
          (prerelease suffixes like -beta001 are stripped for assembly versions). -->
-<Version>0.12.26</Version>
+<Version>0.12.27</Version>
   </PropertyGroup>
 
   <!-- NuGet package metadata (shared across all packable projects) -->

deploy/helm/rockbot/templates/configmap.yaml

@@ -28,6 +28,16 @@ data:
   AgentProfile__BasePath: "/data/agent"
   AgentProfile__TierRoutingLogMaxEntries: {{ .Values.agent.tierRoutingLogMaxEntries | default 1500 | quote }}
   Memory__BasePath: "/data/agent/memory"
+
+  # ── Append-only JSONL log retention (applied once per dream cycle) ─────────
+  # `dig` (not `default`) so an explicit false / 0 — which disable a dimension —
+  # are honoured rather than swallowed, while a wholly-absent block (e.g. a
+  # --reuse-values upgrade) still falls back to the chart defaults below.
+  {{- $logRetention := .Values.agent.logRetention | default dict }}
+  Dream__LogRetentionEnabled: {{ dig "enabled" true $logRetention | quote }}
+  Dream__LogRetentionMaxFileAge: {{ dig "maxFileAge" "30.00:00:00" $logRetention | quote }}
+  Dream__LogRetentionMaxFilesPerDirectory: {{ dig "maxFilesPerDirectory" 1000 $logRetention | quote }}
+  Dream__LogRetentionMaxLinesPerFile: {{ dig "maxLinesPerFile" 10000 $logRetention | quote }}
   Skill__BasePath: "/data/agent/skills"
   McpBridge__ConfigPath: "/data/agent/mcp.json"
   LlmPricing__ConfigPath: "/data/agent/llm-pricing.json"

deploy/helm/rockbot/values.yaml

@@ -20,6 +20,28 @@ agent:
   # get_routing_summary tool). Once the cap is reached the logger trims the oldest
   # entries on every append. ~500 bytes per entry, so 1500 ≈ 750 KB on disk.
   tierRoutingLogMaxEntries: 1500
+  # Append-only JSONL log retention, applied once per dream cycle (see
+  # docs/dream-service.md → "Log retention"). Covers skill-usage, tool-call,
+  # feedback (per-session directories) and skill-resource-usage,
+  # wisp-executions (single files). Values below are sized from observed live
+  # traffic; raise them if you query these logs further back in time.
+  logRetention:
+    enabled: true
+    # Per-session directory logs: delete {sessionId}.jsonl files whose last-write
+    # time is older than this. Floor it at the widest dream query window (skill
+    # usage looks back 30 days) so pruning never starves a pass. TimeSpan format
+    # "d.hh:mm:ss". Set to "0" to disable age pruning.
+    maxFileAge: "30.00:00:00"
+    # Per-session directory logs: backstop cap on file count after age pruning
+    # (oldest dropped first). Age pruning is the primary control; this only bites
+    # during a session storm. Set to 0 to disable count pruning.
+    maxFilesPerDirectory: 1000
+    # Per-file line cap. Applies to the single-file logs (wisp-executions.jsonl,
+    # skill-resource-usage.jsonl) AND to each per-session file — this is what bounds
+    # a persistent UI/CLI session's {id}.jsonl, which age/count pruning never reaps.
+    # At ~1.1 KB/line for wisp records, 10000 ≈ 11 MB, well above the 14-day window
+    # the wisp dream pass reads. Set to 0 to disable line trimming.
+    maxLinesPerFile: 10000
   resources:
     requests:
       cpu: 500m

docs/agent-host.md

@@ -155,6 +155,11 @@ One JSON object per line. Per-session semaphores prevent concurrent write races.
 `QueryRecentAsync` scans all JSONL files to find entries since a given timestamp — used by the
 dream cycle to gather quality signals for memory consolidation and skill optimization.
 
+These per-session files (along with the skill-usage and tool-call logs) are append-only and are
+capped by the dream cycle's [log-retention pass](dream-service.md#pass-0--log-retention) — aged
+files are deleted, the directory is held under a file-count cap, and each surviving file is
+line-trimmed (so a persistent UI/CLI session that never ages out is still bounded).
+
 ### `SessionSummaryService`
 
 Background hosted service that evaluates completed sessions:

docs/dream-service.md

@@ -33,9 +33,40 @@ Startup
 
 ## Passes
 
-Each dream cycle runs five passes in sequence. Passes that depend on optional services
-(`IConversationLog`, `IFeedbackStore`, `ISkillUsageStore`) are skipped when those services are
-not registered.
+Each dream cycle runs a log-retention pass followed by five knowledge passes in sequence.
+Passes that depend on optional services (`IConversationLog`, `IFeedbackStore`,
+`ISkillUsageStore`) are skipped when those services are not registered.
+
+### Pass 0 — Log retention
+
+Runs **first and unconditionally**, before the knowledge passes — and crucially before the
+"fewer than two memories → early return" guard, so the append-only logs are capped on every
+cycle even when there is nothing to consolidate.
+
+The agent's append-only JSONL telemetry logs (skill-usage, tool-call, feedback,
+skill-resource-usage, wisp-executions) have no rotation of their own, so without this pass
+they grow forever. The pass resolves every registered `IPrunableLog` and applies the
+configured `LogRetentionPolicy`. Each log knows its own on-disk shape and delegates the file
+work to the shared `JsonlLogRetention` helper:
+
+| Log | Shape | Retention applied |
+|---|---|---|
+| skill-usage, tool-call, feedback | per-session directory of `{sessionId}.jsonl` | delete files older than `LogRetentionMaxFileAge` (by last-write time); cap the directory at `LogRetentionMaxFilesPerDirectory` (oldest dropped first); then line-trim each surviving file to `LogRetentionMaxLinesPerFile` under that session's write lock |
+| skill-resource-usage, wisp-executions | single append-only file | trim to the last `LogRetentionMaxLinesPerFile` lines (atomic temp-file rewrite, serialized against the writer) |
+
+Retention is best-effort: a failure pruning one log is logged and does not abort the sweep or
+the rest of the dream cycle. A non-positive value disables the corresponding dimension.
+
+The per-session line-trim is what bounds a *persistent* session file — `blazor-session.jsonl`,
+`cli-session.jsonl` — that age/count pruning alone never reaps, because such a file is written
+continuously (never aged out by last-write time) and is never the oldest file (never
+count-pruned). On a long-running deployment the UI session's tool-call log is the largest single
+file; line-trimming holds it to `LogRetentionMaxLinesPerFile`. Trimming reuses the store's own
+per-session semaphore, so it can never race a concurrent append. (Scope matches age/count
+pruning — top-level `{sessionId}.jsonl` files only; namespaced session files in subdirectories
+are not swept.)
+
+**Enabled/disabled by:** `DreamOptions.LogRetentionEnabled` (default `true`).
 
 ### Pass 1 — Memory consolidation
 
@@ -269,9 +300,23 @@ public sealed class DreamOptions
     // Feature flags
     public bool PreferenceInferenceEnabled { get; set; } = true;
     public bool SkillGapEnabled { get; set; } = true;
+
+    // Append-only JSONL log retention (Pass 0)
+    public bool LogRetentionEnabled { get; set; } = true;
+    public TimeSpan LogRetentionMaxFileAge { get; set; } = TimeSpan.FromDays(30);  // per-session dirs
+    public int LogRetentionMaxFilesPerDirectory { get; set; } = 1000;              // per-session dirs
+    public int LogRetentionMaxLinesPerFile { get; set; } = 50_000;                 // single-file logs
 }
 ```
 
+In Kubernetes these are bound from the `Dream` configuration section via the agent ConfigMap
+(`Dream__LogRetentionEnabled`, `Dream__LogRetentionMaxFileAge`,
+`Dream__LogRetentionMaxFilesPerDirectory`, `Dream__LogRetentionMaxLinesPerFile`), driven by the
+`agent.logRetention.*` Helm values. The Helm chart ships tighter, traffic-sized values than the
+code defaults (`maxLinesPerFile: 10000` ≈ 11 MB for the wisp log at ~1.1 KB/line). Floor
+`maxFileAge` at the widest dream query window (skill usage looks back 30 days) so age pruning
+never starves a downstream pass.
+
 ---
 
 ## DI registration

src/RockBot.Agent/Program.cs

@@ -276,7 +276,7 @@ async Task<IChatClient> BuildClientForTierAsync(LlmTierConfig config, string tie
     agent.WithKnowledgeGraph();
     agent.WithFailureClusterStore();
     agent.WithRepairTickets();
-    agent.WithDreaming();
+    agent.WithDreaming(opts => builder.Configuration.GetSection("Dream").Bind(opts));
     agent.AddToolHandler();
     agent.AddMcpToolProxy();
     agent.AddFileSystemTools(opts => builder.Configuration.GetSection("FileSystem").Bind(opts));

src/RockBot.Host.Abstractions/DreamOptions.cs

@@ -278,4 +278,39 @@ public sealed class DreamOptions
     /// discoverable via keyword search.
     /// </summary>
     public float ImportanceDecayFloor { get; set; } = 0.10f;
+
+    /// <summary>
+    /// Whether the log-retention pass runs each dream cycle. When enabled, the dream
+    /// prunes the append-only JSONL logs (skill-usage, tool-call, feedback,
+    /// skill-resource-usage, wisp-executions) so they don't grow without bound.
+    /// Disable to retain every log line/file indefinitely. Default: true.
+    /// </summary>
+    public bool LogRetentionEnabled { get; set; } = true;
+
+    /// <summary>
+    /// Per-session JSONL log files (one <c>{sessionId}.jsonl</c> per session for the
+    /// skill-usage, tool-call, and feedback logs) older than this — by last-write
+    /// time — are deleted by the retention pass. Set to <see cref="TimeSpan.Zero"/>
+    /// or negative to disable age-based pruning. Default: 30 days.
+    /// </summary>
+    public TimeSpan LogRetentionMaxFileAge { get; set; } = TimeSpan.FromDays(30);
+
+    /// <summary>
+    /// Ceiling on the number of per-session JSONL files kept in each session-log
+    /// directory. After age pruning, if more remain, the oldest are deleted until the
+    /// count is within this cap. Set to zero or negative to disable count-based
+    /// pruning. Default: 1000.
+    /// </summary>
+    public int LogRetentionMaxFilesPerDirectory { get; set; } = 1000;
+
+    /// <summary>
+    /// Ceiling on the number of lines retained in any single JSONL log file. Applies to
+    /// the single-file append-only logs (skill-resource-usage.jsonl,
+    /// wisp-executions.jsonl) and to each individual per-session file (e.g. a persistent
+    /// UI/CLI session's <c>{sessionId}.jsonl</c> that age/count pruning never reaps
+    /// because it is continuously written). When a file exceeds this, the retention pass
+    /// rewrites it keeping only the most recent lines. Set to zero or negative to disable
+    /// trimming. Default: 50,000.
+    /// </summary>
+    public int LogRetentionMaxLinesPerFile { get; set; } = 50_000;
 }

src/RockBot.Host/AgentMemoryExtensions.cs

@@ -106,9 +106,12 @@ public static AgentHostBuilder WithSkills(
         builder.Services.TryAddSingleton<EmbeddingTextPreparer>();
         builder.Services.AddSingleton<ISkillStore, FileSkillStore>();
         builder.Services.AddSingleton<ISkillUsageStore, FileSkillUsageStore>();
+        builder.Services.AddSingleton<IPrunableLog>(sp => (IPrunableLog)sp.GetRequiredService<ISkillUsageStore>());
         builder.Services.AddSingleton<ISkillResourceUsageStore, FileSkillResourceUsageStore>();
+        builder.Services.AddSingleton<IPrunableLog>(sp => (IPrunableLog)sp.GetRequiredService<ISkillResourceUsageStore>());
         builder.Services.Configure<ToolCallLogOptions>(_ => { });
         builder.Services.AddSingleton<IToolCallLog, FileToolCallLog>();
+        builder.Services.AddSingleton<IPrunableLog>(sp => (IPrunableLog)sp.GetRequiredService<IToolCallLog>());
         builder.Services.AddSingleton<IHostedService, StarterSkillService>();
 
         return builder;
@@ -191,6 +194,7 @@ public static AgentHostBuilder WithFeedback(
             builder.Services.Configure<FeedbackOptions>(_ => { });
 
         builder.Services.AddSingleton<IFeedbackStore, FileFeedbackStore>();
+        builder.Services.AddSingleton<IPrunableLog>(sp => (IPrunableLog)sp.GetRequiredService<IFeedbackStore>());
         builder.Services.AddSingleton<SessionSummaryService>();
         builder.Services.AddSingleton<IHostedService>(sp => sp.GetRequiredService<SessionSummaryService>());
 

src/RockBot.Host/DreamService.cs

@@ -51,6 +51,7 @@ internal sealed class DreamService : IHostedService, IDisposable
     private readonly IRepairTicketVerifier? _repairTicketVerifier;
     private readonly RepairTicketOptions? _repairOptions;
     private readonly IReadOnlyList<IToolSkillProvider> _toolSkillProviders;
+    private readonly IReadOnlyList<IPrunableLog> _prunableLogs;
     private Timer? _timer;
     private CronExpression? _cron;
     private string? _dreamDirective;
@@ -102,7 +103,8 @@ public DreamService(
         IRepairTicketVerifier? repairTicketVerifier = null,
         IOptions<RepairTicketOptions>? repairOptions = null,
         IEnumerable<IToolSkillProvider>? toolSkillProviders = null,
-        TieredChatClientRegistry? tieredRegistry = null)
+        TieredChatClientRegistry? tieredRegistry = null,
+        IEnumerable<IPrunableLog>? prunableLogs = null)
     {
         _memory = memory;
         _skillStore = skillStores.FirstOrDefault();
@@ -139,6 +141,7 @@ public DreamService(
             _repairAppliers = map;
         }
         _toolSkillProviders = toolSkillProviders?.ToList() ?? (IReadOnlyList<IToolSkillProvider>)Array.Empty<IToolSkillProvider>();
+        _prunableLogs = prunableLogs?.ToList() ?? (IReadOnlyList<IPrunableLog>)Array.Empty<IPrunableLog>();
     }
 
     public Task StartAsync(CancellationToken cancellationToken)
@@ -493,6 +496,11 @@ private async Task DreamAsync()
         {
             var ct = slot.Token;
 
+            // Log retention runs first and unconditionally — append-only JSONL logs
+            // must be capped even on cycles where there's nothing to consolidate (the
+            // memory-count check below can early-return).
+            await RunLogRetentionPassAsync(ct);
+
             var all = await _memory.SearchAsync(new MemorySearchCriteria(MaxResults: 1000));
 
             if (all.Count < 2)
@@ -3804,6 +3812,47 @@ and identify pairs that contradict each other on the same subject (same tool, sa
     /// is rethrown so DreamAsync's outer handler can log a single
     /// "preempted by user request" line — see issue #333.
     /// </summary>
+    /// <summary>
+    /// Prunes every registered append-only JSONL log so they don't grow forever.
+    /// Each log knows its own on-disk shape and applies the policy via the shared
+    /// <see cref="JsonlLogRetention"/> helper; a failure in one log is logged and
+    /// does not abort the sweep. Gated by <see cref="DreamOptions.LogRetentionEnabled"/>.
+    /// </summary>
+    private async Task RunLogRetentionPassAsync(CancellationToken ct)
+    {
+        if (!_options.LogRetentionEnabled || _prunableLogs.Count == 0)
+            return;
+
+        await RunPassAsync("log retention", async () =>
+        {
+            var policy = new LogRetentionPolicy(
+                MaxFileAge: _options.LogRetentionMaxFileAge,
+                MaxFilesPerDirectory: _options.LogRetentionMaxFilesPerDirectory,
+                MaxLinesPerFile: _options.LogRetentionMaxLinesPerFile);
+
+            var total = 0;
+            foreach (var log in _prunableLogs)
+            {
+                ct.ThrowIfCancellationRequested();
+                try
+                {
+                    total += await log.PruneAsync(policy, ct);
+                }
+                catch (OperationCanceledException)
+                {
+                    throw;
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogWarning(ex, "DreamService: log retention failed for {Log}", log.GetType().Name);
+                }
+            }
+
+            if (total > 0)
+                _logger.LogInformation("DreamService: log retention removed {Total} stale log file(s)/line(s)", total);
+        });
+    }
+
     private async Task RunPassAsync(string passName, Func<Task> body)
     {
         try

src/RockBot.Host/FileFeedbackStore.cs

@@ -9,7 +9,7 @@ namespace RockBot.Host;
 /// File-based feedback store. Each session's entries are appended to a separate JSONL file:
 /// <c>{basePath}/{sessionId}.jsonl</c>. One JSON object per line.
 /// </summary>
-internal sealed class FileFeedbackStore : IFeedbackStore
+internal sealed class FileFeedbackStore : IFeedbackStore, IPrunableLog
 {
     private static readonly JsonSerializerOptions JsonOptions = new()
     {
@@ -54,6 +54,24 @@ public async Task AppendAsync(FeedbackEntry entry, CancellationToken cancellatio
         }
     }
 
+    /// <summary>
+    /// Per-session JSONL files accumulate one file per session forever. Retention
+    /// drops whole session files older than the configured window, caps the total
+    /// file count, then line-trims any surviving file (e.g. a persistent UI/CLI
+    /// session that never ages out) to the per-file line budget under that session's
+    /// write lock so the trim can't race an append.
+    /// </summary>
+    public async Task<int> PruneAsync(LogRetentionPolicy policy, CancellationToken ct = default)
+    {
+        var removed = await JsonlLogRetention.PruneAgedFilesAsync(
+            _basePath, policy.MaxFileAge, policy.MaxFilesPerDirectory, "*.jsonl", _logger, ct);
+        removed += await JsonlLogRetention.TrimSessionFilesAsync(
+            _basePath, policy.MaxLinesPerFile, "*.jsonl",
+            id => _writeLocks.GetOrAdd(id, _ => new SemaphoreSlim(1, 1)),
+            _logger, ct);
+        return removed;
+    }
+
     public async Task<IReadOnlyList<FeedbackEntry>> GetBySessionAsync(string sessionId, CancellationToken cancellationToken = default)
     {
         var path = Path.Combine(_basePath, $"{sessionId}.jsonl");

src/RockBot.Host/FileSkillResourceUsageStore.cs

@@ -10,7 +10,7 @@ namespace RockBot.Host;
 /// Mirrors <c>FileWispExecutionLog</c>'s shape — append-only, single-writer
 /// semaphore, lazy reads.
 /// </summary>
-internal sealed class FileSkillResourceUsageStore : ISkillResourceUsageStore
+internal sealed class FileSkillResourceUsageStore : ISkillResourceUsageStore, IPrunableLog
 {
     private static readonly JsonSerializerOptions JsonOptions = new()
     {
@@ -51,6 +51,24 @@ public async Task RecordCheckoutAsync(
         }
     }
 
+    /// <summary>
+    /// Single append-only file shared across all sessions. Retention trims it to the
+    /// last <see cref="LogRetentionPolicy.MaxLinesPerFile"/> lines, serialized against
+    /// the append writer so a trim never races a checkout record.
+    /// </summary>
+    public async Task<int> PruneAsync(LogRetentionPolicy policy, CancellationToken ct = default)
+    {
+        await _writeLock.WaitAsync(ct);
+        try
+        {
+            return await JsonlLogRetention.TrimToLastLinesAsync(_filePath, policy.MaxLinesPerFile, _logger, ct);
+        }
+        finally
+        {
+            _writeLock.Release();
+        }
+    }
+
     public async Task<IReadOnlyList<SkillResourceCheckoutEvent>> QueryCheckoutsAsync(
         string skillName, string filename, DateTimeOffset since, CancellationToken ct = default)
     {