From e07a30f0698ba0b2986619f7cb1a3c509a7379d0 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 3 Jun 2026 06:00:56 +0000 Subject: [PATCH 1/3] Cap append-only JSONL logs via a shared dream-driven retention pass The skill-usage, tool-call, feedback, skill-resource-usage, and wisp-executions JSONL logs were append-only with no rotation, so they grew without bound (per-session directories accumulated files forever; the two global files grew line-by-line indefinitely). Add a shared JsonlLogRetention helper plus an opt-in IPrunableLog contract. Each file-backed store implements IPrunableLog and delegates to the helper with its own on-disk layout: per-session directories drop aged session files then cap file count; single-file logs trim to a trailing line budget (serialized against their writer). DreamService resolves all registered IPrunableLog instances and prunes them once per cycle, before the memory-count early-return so retention runs every cycle. New DreamOptions knobs (LogRetentionEnabled, MaxFileAge, MaxFilesPerDirectory, MaxLinesPerFile) control the policy; no new background service is introduced. https://claude.ai/code/session_012sTRRT47bKJwQBSksmFuLm --- src/RockBot.Host.Abstractions/DreamOptions.cs | 32 +++ src/RockBot.Host/AgentMemoryExtensions.cs | 4 + src/RockBot.Host/DreamService.cs | 51 +++- src/RockBot.Host/FileFeedbackStore.cs | 11 +- .../FileSkillResourceUsageStore.cs | 20 +- src/RockBot.Host/FileSkillUsageStore.cs | 11 +- src/RockBot.Host/FileToolCallLog.cs | 11 +- src/RockBot.Host/IPrunableLog.cs | 30 +++ src/RockBot.Host/JsonlLogRetention.cs | 143 +++++++++++ src/RockBot.Wisp/FileWispExecutionLog.cs | 20 +- .../WispServiceCollectionExtensions.cs | 1 + .../JsonlLogRetentionTests.cs | 222 ++++++++++++++++++ 12 files changed, 550 insertions(+), 6 deletions(-) create mode 100644 src/RockBot.Host/IPrunableLog.cs create mode 100644 src/RockBot.Host/JsonlLogRetention.cs create mode 100644 tests/RockBot.Host.Tests/JsonlLogRetentionTests.cs diff --git a/src/RockBot.Host.Abstractions/DreamOptions.cs b/src/RockBot.Host.Abstractions/DreamOptions.cs index 17ac76ad..6c16b027 100644 --- a/src/RockBot.Host.Abstractions/DreamOptions.cs +++ b/src/RockBot.Host.Abstractions/DreamOptions.cs @@ -278,4 +278,36 @@ public sealed class DreamOptions /// discoverable via keyword search. /// public float ImportanceDecayFloor { get; set; } = 0.10f; + + /// + /// Whether the log-retention pass runs each dream cycle. When enabled, the dream + /// prunes the append-only JSONL logs (skill-usage, tool-call, feedback, + /// skill-resource-usage, wisp-executions) so they don't grow without bound. + /// Disable to retain every log line/file indefinitely. Default: true. + /// + public bool LogRetentionEnabled { get; set; } = true; + + /// + /// Per-session JSONL log files (one {sessionId}.jsonl per session for the + /// skill-usage, tool-call, and feedback logs) older than this — by last-write + /// time — are deleted by the retention pass. Set to + /// or negative to disable age-based pruning. Default: 30 days. + /// + public TimeSpan LogRetentionMaxFileAge { get; set; } = TimeSpan.FromDays(30); + + /// + /// Ceiling on the number of per-session JSONL files kept in each session-log + /// directory. After age pruning, if more remain, the oldest are deleted until the + /// count is within this cap. Set to zero or negative to disable count-based + /// pruning. Default: 1000. + /// + public int LogRetentionMaxFilesPerDirectory { get; set; } = 1000; + + /// + /// Ceiling on the number of lines retained in each single-file append-only log + /// (skill-resource-usage.jsonl, wisp-executions.jsonl). When a file exceeds this, + /// the retention pass rewrites it keeping only the most recent lines. Set to zero + /// or negative to disable trimming. Default: 50,000. + /// + public int LogRetentionMaxLinesPerFile { get; set; } = 50_000; } diff --git a/src/RockBot.Host/AgentMemoryExtensions.cs b/src/RockBot.Host/AgentMemoryExtensions.cs index d37b38fa..3f3ce445 100644 --- a/src/RockBot.Host/AgentMemoryExtensions.cs +++ b/src/RockBot.Host/AgentMemoryExtensions.cs @@ -106,9 +106,12 @@ public static AgentHostBuilder WithSkills( builder.Services.TryAddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); + builder.Services.AddSingleton(sp => (IPrunableLog)sp.GetRequiredService()); builder.Services.AddSingleton(); + builder.Services.AddSingleton(sp => (IPrunableLog)sp.GetRequiredService()); builder.Services.Configure(_ => { }); builder.Services.AddSingleton(); + builder.Services.AddSingleton(sp => (IPrunableLog)sp.GetRequiredService()); builder.Services.AddSingleton(); return builder; @@ -191,6 +194,7 @@ public static AgentHostBuilder WithFeedback( builder.Services.Configure(_ => { }); builder.Services.AddSingleton(); + builder.Services.AddSingleton(sp => (IPrunableLog)sp.GetRequiredService()); builder.Services.AddSingleton(); builder.Services.AddSingleton(sp => sp.GetRequiredService()); diff --git a/src/RockBot.Host/DreamService.cs b/src/RockBot.Host/DreamService.cs index 3222a658..f9335c3f 100644 --- a/src/RockBot.Host/DreamService.cs +++ b/src/RockBot.Host/DreamService.cs @@ -51,6 +51,7 @@ internal sealed class DreamService : IHostedService, IDisposable private readonly IRepairTicketVerifier? _repairTicketVerifier; private readonly RepairTicketOptions? _repairOptions; private readonly IReadOnlyList _toolSkillProviders; + private readonly IReadOnlyList _prunableLogs; private Timer? _timer; private CronExpression? _cron; private string? _dreamDirective; @@ -102,7 +103,8 @@ public DreamService( IRepairTicketVerifier? repairTicketVerifier = null, IOptions? repairOptions = null, IEnumerable? toolSkillProviders = null, - TieredChatClientRegistry? tieredRegistry = null) + TieredChatClientRegistry? tieredRegistry = null, + IEnumerable? prunableLogs = null) { _memory = memory; _skillStore = skillStores.FirstOrDefault(); @@ -139,6 +141,7 @@ public DreamService( _repairAppliers = map; } _toolSkillProviders = toolSkillProviders?.ToList() ?? (IReadOnlyList)Array.Empty(); + _prunableLogs = prunableLogs?.ToList() ?? (IReadOnlyList)Array.Empty(); } public Task StartAsync(CancellationToken cancellationToken) @@ -493,6 +496,11 @@ private async Task DreamAsync() { var ct = slot.Token; + // Log retention runs first and unconditionally — append-only JSONL logs + // must be capped even on cycles where there's nothing to consolidate (the + // memory-count check below can early-return). + await RunLogRetentionPassAsync(ct); + var all = await _memory.SearchAsync(new MemorySearchCriteria(MaxResults: 1000)); if (all.Count < 2) @@ -3804,6 +3812,47 @@ and identify pairs that contradict each other on the same subject (same tool, sa /// is rethrown so DreamAsync's outer handler can log a single /// "preempted by user request" line — see issue #333. /// + /// + /// Prunes every registered append-only JSONL log so they don't grow forever. + /// Each log knows its own on-disk shape and applies the policy via the shared + /// helper; a failure in one log is logged and + /// does not abort the sweep. Gated by . + /// + private async Task RunLogRetentionPassAsync(CancellationToken ct) + { + if (!_options.LogRetentionEnabled || _prunableLogs.Count == 0) + return; + + await RunPassAsync("log retention", async () => + { + var policy = new LogRetentionPolicy( + MaxFileAge: _options.LogRetentionMaxFileAge, + MaxFilesPerDirectory: _options.LogRetentionMaxFilesPerDirectory, + MaxLinesPerFile: _options.LogRetentionMaxLinesPerFile); + + var total = 0; + foreach (var log in _prunableLogs) + { + ct.ThrowIfCancellationRequested(); + try + { + total += await log.PruneAsync(policy, ct); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "DreamService: log retention failed for {Log}", log.GetType().Name); + } + } + + if (total > 0) + _logger.LogInformation("DreamService: log retention removed {Total} stale log file(s)/line(s)", total); + }); + } + private async Task RunPassAsync(string passName, Func body) { try diff --git a/src/RockBot.Host/FileFeedbackStore.cs b/src/RockBot.Host/FileFeedbackStore.cs index c9e598ae..92228b05 100644 --- a/src/RockBot.Host/FileFeedbackStore.cs +++ b/src/RockBot.Host/FileFeedbackStore.cs @@ -9,7 +9,7 @@ namespace RockBot.Host; /// File-based feedback store. Each session's entries are appended to a separate JSONL file: /// {basePath}/{sessionId}.jsonl. One JSON object per line. /// -internal sealed class FileFeedbackStore : IFeedbackStore +internal sealed class FileFeedbackStore : IFeedbackStore, IPrunableLog { private static readonly JsonSerializerOptions JsonOptions = new() { @@ -54,6 +54,15 @@ public async Task AppendAsync(FeedbackEntry entry, CancellationToken cancellatio } } + /// + /// Per-session JSONL files accumulate one file per session forever. Retention + /// drops whole session files older than the configured window, then caps the + /// total file count. + /// + public Task PruneAsync(LogRetentionPolicy policy, CancellationToken ct = default) + => JsonlLogRetention.PruneAgedFilesAsync( + _basePath, policy.MaxFileAge, policy.MaxFilesPerDirectory, "*.jsonl", _logger, ct); + public async Task> GetBySessionAsync(string sessionId, CancellationToken cancellationToken = default) { var path = Path.Combine(_basePath, $"{sessionId}.jsonl"); diff --git a/src/RockBot.Host/FileSkillResourceUsageStore.cs b/src/RockBot.Host/FileSkillResourceUsageStore.cs index e942e494..c6fcb997 100644 --- a/src/RockBot.Host/FileSkillResourceUsageStore.cs +++ b/src/RockBot.Host/FileSkillResourceUsageStore.cs @@ -10,7 +10,7 @@ namespace RockBot.Host; /// Mirrors FileWispExecutionLog's shape — append-only, single-writer /// semaphore, lazy reads. /// -internal sealed class FileSkillResourceUsageStore : ISkillResourceUsageStore +internal sealed class FileSkillResourceUsageStore : ISkillResourceUsageStore, IPrunableLog { private static readonly JsonSerializerOptions JsonOptions = new() { @@ -51,6 +51,24 @@ public async Task RecordCheckoutAsync( } } + /// + /// Single append-only file shared across all sessions. Retention trims it to the + /// last lines, serialized against + /// the append writer so a trim never races a checkout record. + /// + public async Task PruneAsync(LogRetentionPolicy policy, CancellationToken ct = default) + { + await _writeLock.WaitAsync(ct); + try + { + return await JsonlLogRetention.TrimToLastLinesAsync(_filePath, policy.MaxLinesPerFile, _logger, ct); + } + finally + { + _writeLock.Release(); + } + } + public async Task> QueryCheckoutsAsync( string skillName, string filename, DateTimeOffset since, CancellationToken ct = default) { diff --git a/src/RockBot.Host/FileSkillUsageStore.cs b/src/RockBot.Host/FileSkillUsageStore.cs index 1d801675..993bb567 100644 --- a/src/RockBot.Host/FileSkillUsageStore.cs +++ b/src/RockBot.Host/FileSkillUsageStore.cs @@ -9,7 +9,7 @@ namespace RockBot.Host; /// File-based skill usage store. Each session's invocation events are appended to a separate JSONL file: /// {basePath}/{sessionId}.jsonl. One JSON object per line. /// -internal sealed class FileSkillUsageStore : ISkillUsageStore +internal sealed class FileSkillUsageStore : ISkillUsageStore, IPrunableLog { private static readonly JsonSerializerOptions JsonOptions = new() { @@ -53,6 +53,15 @@ public async Task AppendAsync(SkillInvocationEvent evt, CancellationToken ct = d } } + /// + /// Per-session JSONL files accumulate one file per session forever. Retention + /// drops whole session files older than the configured window, then caps the + /// total file count. + /// + public Task PruneAsync(LogRetentionPolicy policy, CancellationToken ct = default) + => JsonlLogRetention.PruneAgedFilesAsync( + _basePath, policy.MaxFileAge, policy.MaxFilesPerDirectory, "*.jsonl", _logger, ct); + public async Task> GetBySessionAsync(string sessionId, CancellationToken ct = default) { var path = Path.Combine(_basePath, $"{sessionId}.jsonl"); diff --git a/src/RockBot.Host/FileToolCallLog.cs b/src/RockBot.Host/FileToolCallLog.cs index 466c24bd..8ed30807 100644 --- a/src/RockBot.Host/FileToolCallLog.cs +++ b/src/RockBot.Host/FileToolCallLog.cs @@ -9,7 +9,7 @@ namespace RockBot.Host; /// File-based tool-call log. Each session's tool invocations are appended to a separate JSONL file: /// {basePath}/{sessionId}.jsonl. One JSON object per line. /// -internal sealed class FileToolCallLog : IToolCallLog +internal sealed class FileToolCallLog : IToolCallLog, IPrunableLog { private static readonly JsonSerializerOptions JsonOptions = new() { @@ -53,6 +53,15 @@ public async Task AppendAsync(ToolCallEvent evt, CancellationToken ct = default) } } + /// + /// Per-session JSONL files accumulate one file per session forever. Retention + /// drops whole session files older than the configured window, then caps the + /// total file count. + /// + public Task PruneAsync(LogRetentionPolicy policy, CancellationToken ct = default) + => JsonlLogRetention.PruneAgedFilesAsync( + _basePath, policy.MaxFileAge, policy.MaxFilesPerDirectory, "*.jsonl", _logger, ct); + public async Task> GetBySessionAsync(string sessionId, CancellationToken ct = default) { var path = Path.Combine(_basePath, $"{sessionId}.jsonl"); diff --git a/src/RockBot.Host/IPrunableLog.cs b/src/RockBot.Host/IPrunableLog.cs new file mode 100644 index 00000000..f8002c97 --- /dev/null +++ b/src/RockBot.Host/IPrunableLog.cs @@ -0,0 +1,30 @@ +namespace RockBot.Host; + +/// +/// Retention knobs applied by the dream cycle's log-retention pass. +/// Single-file append-only logs honour ; +/// per-session directory logs (one {id}.jsonl per session) honour +/// and . +/// A non-positive value disables the corresponding dimension. +/// +public sealed record LogRetentionPolicy( + TimeSpan MaxFileAge, + int MaxFilesPerDirectory, + int MaxLinesPerFile); + +/// +/// Opt-in contract for append-only logs that can prune themselves. The file-backed +/// JSONL stores implement this; resolves every registered +/// instance and invokes once per dream cycle so the logs +/// don't grow without bound. Implementations know their own on-disk layout and +/// delegate the actual file work to . +/// +public interface IPrunableLog +{ + /// + /// Applies to this log and returns the number of files + /// or lines removed. Best-effort: implementations log and swallow I/O failures + /// rather than throwing, so one failing log never aborts the retention sweep. + /// + Task PruneAsync(LogRetentionPolicy policy, CancellationToken ct = default); +} diff --git a/src/RockBot.Host/JsonlLogRetention.cs b/src/RockBot.Host/JsonlLogRetention.cs new file mode 100644 index 00000000..a1d93c59 --- /dev/null +++ b/src/RockBot.Host/JsonlLogRetention.cs @@ -0,0 +1,143 @@ +using Microsoft.Extensions.Logging; + +namespace RockBot.Host; + +/// +/// Shared retention utilities for JSONL logs, invoked from the dream cycle. Two +/// shapes are supported: per-session directories of {id}.jsonl files (pruned +/// by age, then by count) and single append-only files (trimmed to a trailing line +/// budget). All operations are best-effort — I/O failures are logged and swallowed +/// so a retention sweep never aborts the caller. +/// +public static class JsonlLogRetention +{ + /// + /// Deletes files directly under + /// that are older than + /// (by last-write time), then, if more than remain, + /// deletes the oldest until the count is within budget. A non-positive + /// disables age pruning; a non-positive + /// disables count pruning. Returns the number of + /// files deleted. + /// + public static Task PruneAgedFilesAsync( + string directory, + TimeSpan maxAge, + int maxFiles, + string searchPattern, + ILogger logger, + CancellationToken ct = default) + { + if (!Directory.Exists(directory)) + return Task.FromResult(0); + + var deleted = 0; + try + { + var files = new DirectoryInfo(directory) + .EnumerateFiles(searchPattern, SearchOption.TopDirectoryOnly) + .ToList(); + + // Age-based pruning: drop files not written within the retention window. + if (maxAge > TimeSpan.Zero) + { + var cutoff = DateTime.UtcNow - maxAge; + foreach (var file in files.ToList()) + { + ct.ThrowIfCancellationRequested(); + if (file.LastWriteTimeUtc >= cutoff) + continue; + if (TryDelete(file, logger)) + { + files.Remove(file); + deleted++; + } + } + } + + // Count-based pruning: keep the most recently written maxFiles. + if (maxFiles > 0 && files.Count > maxFiles) + { + var oldestFirst = files.OrderBy(f => f.LastWriteTimeUtc).ToList(); + var toRemove = files.Count - maxFiles; + for (var i = 0; i < toRemove; i++) + { + ct.ThrowIfCancellationRequested(); + if (TryDelete(oldestFirst[i], logger)) + deleted++; + } + } + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + logger.LogWarning(ex, "JSONL retention: failed to prune directory {Directory}", directory); + } + + if (deleted > 0) + logger.LogInformation("JSONL retention: deleted {Count} file(s) from {Directory}", deleted, directory); + + return Task.FromResult(deleted); + } + + /// + /// Rewrites to retain only its last + /// lines when it exceeds that budget. A non-positive + /// is a no-op. The rewrite is atomic (temp file + + /// move). Returns the number of lines removed. Callers that also append to this + /// file MUST serialize this call against their writer. + /// + public static async Task TrimToLastLinesAsync( + string filePath, + int maxLines, + ILogger logger, + CancellationToken ct = default) + { + if (maxLines <= 0 || !File.Exists(filePath)) + return 0; + + try + { + var lines = await File.ReadAllLinesAsync(filePath, ct); + if (lines.Length <= maxLines) + return 0; + + var keep = lines[^maxLines..]; + var tempPath = filePath + ".tmp"; + await File.WriteAllLinesAsync(tempPath, keep, ct); + File.Move(tempPath, filePath, overwrite: true); + + var removed = lines.Length - keep.Length; + logger.LogInformation( + "JSONL retention: trimmed {Removed} line(s) from {Path} (kept last {Kept})", + removed, filePath, keep.Length); + return removed; + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + logger.LogWarning(ex, "JSONL retention: failed to trim {Path}", filePath); + return 0; + } + } + + private static bool TryDelete(FileInfo file, ILogger logger) + { + try + { + file.Delete(); + return true; + } + catch (Exception ex) + { + logger.LogWarning(ex, "JSONL retention: failed to delete {Path}", file.FullName); + return false; + } + } +} diff --git a/src/RockBot.Wisp/FileWispExecutionLog.cs b/src/RockBot.Wisp/FileWispExecutionLog.cs index 911219df..bcc58190 100644 --- a/src/RockBot.Wisp/FileWispExecutionLog.cs +++ b/src/RockBot.Wisp/FileWispExecutionLog.cs @@ -8,7 +8,7 @@ namespace RockBot.Wisp; /// File-based wisp execution log. All records are appended to a single JSONL file: /// {basePath}/wisp-executions.jsonl. One JSON object per line. /// -internal sealed class FileWispExecutionLog : IWispExecutionLog +internal sealed class FileWispExecutionLog : IWispExecutionLog, IPrunableLog { private static readonly JsonSerializerOptions JsonOptions = new() { @@ -43,6 +43,24 @@ public async Task AppendAsync(WispExecutionRecord record, CancellationToken ct = } } + /// + /// Single append-only file shared across all sessions. Retention trims it to the + /// last lines, serialized against + /// the append writer so a trim never races an execution record. + /// + public async Task PruneAsync(LogRetentionPolicy policy, CancellationToken ct = default) + { + await _writeLock.WaitAsync(ct); + try + { + return await JsonlLogRetention.TrimToLastLinesAsync(_filePath, policy.MaxLinesPerFile, _logger, ct); + } + finally + { + _writeLock.Release(); + } + } + public async Task> QueryRecentAsync( DateTimeOffset since, int maxResults, CancellationToken ct = default) { diff --git a/src/RockBot.Wisp/WispServiceCollectionExtensions.cs b/src/RockBot.Wisp/WispServiceCollectionExtensions.cs index ef2a0023..41e3b8eb 100644 --- a/src/RockBot.Wisp/WispServiceCollectionExtensions.cs +++ b/src/RockBot.Wisp/WispServiceCollectionExtensions.cs @@ -22,6 +22,7 @@ public static AgentHostBuilder AddWisps( builder.Services.AddSingleton(); builder.Services.AddSingleton(); + builder.Services.AddSingleton(sp => (IPrunableLog)sp.GetRequiredService()); builder.Services.AddHostedService(); builder.Services.AddSingleton(); diff --git a/tests/RockBot.Host.Tests/JsonlLogRetentionTests.cs b/tests/RockBot.Host.Tests/JsonlLogRetentionTests.cs new file mode 100644 index 00000000..69d52481 --- /dev/null +++ b/tests/RockBot.Host.Tests/JsonlLogRetentionTests.cs @@ -0,0 +1,222 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; + +namespace RockBot.Host.Tests; + +[TestClass] +public class JsonlLogRetentionTests +{ + private string _tempDir = null!; + + [TestInitialize] + public void Setup() + { + _tempDir = Path.Combine(Path.GetTempPath(), "rockbot-retention-test-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_tempDir); + } + + [TestCleanup] + public void Cleanup() + { + if (Directory.Exists(_tempDir)) + Directory.Delete(_tempDir, recursive: true); + } + + // ── TrimToLastLinesAsync ────────────────────────────────────────────────── + + [TestMethod] + public async Task Trim_KeepsLastLines_AndReturnsRemovedCount() + { + var path = Path.Combine(_tempDir, "global.jsonl"); + var lines = Enumerable.Range(0, 100).Select(i => $"{{\"n\":{i}}}").ToArray(); + await File.WriteAllLinesAsync(path, lines); + + var removed = await JsonlLogRetention.TrimToLastLinesAsync(path, maxLines: 10, NullLogger.Instance); + + Assert.AreEqual(90, removed); + var kept = await File.ReadAllLinesAsync(path); + Assert.AreEqual(10, kept.Length); + Assert.AreEqual("{\"n\":90}", kept[0]); + Assert.AreEqual("{\"n\":99}", kept[^1]); + } + + [TestMethod] + public async Task Trim_UnderBudget_IsNoOp() + { + var path = Path.Combine(_tempDir, "global.jsonl"); + var lines = Enumerable.Range(0, 5).Select(i => $"line-{i}").ToArray(); + await File.WriteAllLinesAsync(path, lines); + + var removed = await JsonlLogRetention.TrimToLastLinesAsync(path, maxLines: 10, NullLogger.Instance); + + Assert.AreEqual(0, removed); + Assert.AreEqual(5, (await File.ReadAllLinesAsync(path)).Length); + } + + [TestMethod] + public async Task Trim_NonPositiveMaxLines_IsNoOp() + { + var path = Path.Combine(_tempDir, "global.jsonl"); + await File.WriteAllLinesAsync(path, new[] { "a", "b", "c" }); + + var removed = await JsonlLogRetention.TrimToLastLinesAsync(path, maxLines: 0, NullLogger.Instance); + + Assert.AreEqual(0, removed); + Assert.AreEqual(3, (await File.ReadAllLinesAsync(path)).Length); + } + + [TestMethod] + public async Task Trim_MissingFile_ReturnsZero() + { + var path = Path.Combine(_tempDir, "does-not-exist.jsonl"); + var removed = await JsonlLogRetention.TrimToLastLinesAsync(path, maxLines: 10, NullLogger.Instance); + Assert.AreEqual(0, removed); + } + + [TestMethod] + public async Task Trim_LeavesNoTempFileBehind() + { + var path = Path.Combine(_tempDir, "global.jsonl"); + await File.WriteAllLinesAsync(path, Enumerable.Range(0, 50).Select(i => i.ToString()).ToArray()); + + await JsonlLogRetention.TrimToLastLinesAsync(path, maxLines: 5, NullLogger.Instance); + + Assert.IsFalse(File.Exists(path + ".tmp")); + } + + // ── PruneAgedFilesAsync ─────────────────────────────────────────────────── + + [TestMethod] + public async Task Prune_DeletesFilesOlderThanMaxAge() + { + var dir = Path.Combine(_tempDir, "sessions"); + Directory.CreateDirectory(dir); + + var oldFile = WriteSessionFile(dir, "old", ageDays: 40); + var freshFile = WriteSessionFile(dir, "fresh", ageDays: 1); + + var deleted = await JsonlLogRetention.PruneAgedFilesAsync( + dir, maxAge: TimeSpan.FromDays(30), maxFiles: 0, "*.jsonl", NullLogger.Instance); + + Assert.AreEqual(1, deleted); + Assert.IsFalse(File.Exists(oldFile)); + Assert.IsTrue(File.Exists(freshFile)); + } + + [TestMethod] + public async Task Prune_CapsFileCount_KeepingNewest() + { + var dir = Path.Combine(_tempDir, "sessions"); + Directory.CreateDirectory(dir); + + // Five files, all within the age window, distinct ages so ordering is deterministic. + var paths = new List<(string path, int age)>(); + for (var i = 0; i < 5; i++) + paths.Add((WriteSessionFile(dir, $"s{i}", ageDays: i), i)); + + var deleted = await JsonlLogRetention.PruneAgedFilesAsync( + dir, maxAge: TimeSpan.Zero, maxFiles: 2, "*.jsonl", NullLogger.Instance); + + Assert.AreEqual(3, deleted); + // The two most recently written (smallest age) survive. + Assert.IsTrue(File.Exists(paths[0].path)); + Assert.IsTrue(File.Exists(paths[1].path)); + Assert.IsFalse(File.Exists(paths[4].path)); + } + + [TestMethod] + public async Task Prune_AgeThenCount_Combined() + { + var dir = Path.Combine(_tempDir, "sessions"); + Directory.CreateDirectory(dir); + + WriteSessionFile(dir, "ancient", ageDays: 100); + var keep1 = WriteSessionFile(dir, "k1", ageDays: 1); + var keep2 = WriteSessionFile(dir, "k2", ageDays: 2); + var dropByCount = WriteSessionFile(dir, "k3", ageDays: 3); + + // Age prunes "ancient"; then count cap of 2 drops the oldest survivor. + var deleted = await JsonlLogRetention.PruneAgedFilesAsync( + dir, maxAge: TimeSpan.FromDays(30), maxFiles: 2, "*.jsonl", NullLogger.Instance); + + Assert.AreEqual(2, deleted); + Assert.IsTrue(File.Exists(keep1)); + Assert.IsTrue(File.Exists(keep2)); + Assert.IsFalse(File.Exists(dropByCount)); + } + + [TestMethod] + public async Task Prune_DisabledDimensions_NoOp() + { + var dir = Path.Combine(_tempDir, "sessions"); + Directory.CreateDirectory(dir); + WriteSessionFile(dir, "old", ageDays: 100); + WriteSessionFile(dir, "new", ageDays: 1); + + var deleted = await JsonlLogRetention.PruneAgedFilesAsync( + dir, maxAge: TimeSpan.Zero, maxFiles: 0, "*.jsonl", NullLogger.Instance); + + Assert.AreEqual(0, deleted); + Assert.AreEqual(2, Directory.GetFiles(dir, "*.jsonl").Length); + } + + [TestMethod] + public async Task Prune_MissingDirectory_ReturnsZero() + { + var deleted = await JsonlLogRetention.PruneAgedFilesAsync( + Path.Combine(_tempDir, "nope"), TimeSpan.FromDays(1), 1, "*.jsonl", NullLogger.Instance); + Assert.AreEqual(0, deleted); + } + + // ── Store integration ───────────────────────────────────────────────────── + + [TestMethod] + public async Task PerSessionStore_PruneAsync_DropsAgedSessionFiles() + { + var skillOptions = Options.Create(new SkillOptions { UsageBasePath = Path.Combine(_tempDir, "skill-usage") }); + var profileOptions = Options.Create(new AgentProfileOptions { BasePath = _tempDir }); + var store = new FileSkillUsageStore(skillOptions, profileOptions, NullLogger.Instance); + + await store.AppendAsync(new SkillInvocationEvent( + Id: "a", SkillName: "x", SessionId: "stale", Timestamp: DateTimeOffset.UtcNow)); + await store.AppendAsync(new SkillInvocationEvent( + Id: "b", SkillName: "y", SessionId: "fresh", Timestamp: DateTimeOffset.UtcNow)); + + var dir = Path.Combine(_tempDir, "skill-usage"); + File.SetLastWriteTimeUtc(Path.Combine(dir, "stale.jsonl"), DateTime.UtcNow.AddDays(-40)); + + var removed = await store.PruneAsync(new LogRetentionPolicy( + MaxFileAge: TimeSpan.FromDays(30), MaxFilesPerDirectory: 0, MaxLinesPerFile: 0)); + + Assert.AreEqual(1, removed); + Assert.AreEqual(0, (await store.GetBySessionAsync("stale")).Count); + Assert.AreEqual(1, (await store.GetBySessionAsync("fresh")).Count); + } + + [TestMethod] + public async Task SingleFileStore_PruneAsync_TrimsToLineBudget() + { + var profileOptions = Options.Create(new AgentProfileOptions { BasePath = _tempDir }); + var store = new FileSkillResourceUsageStore(profileOptions, NullLogger.Instance); + + for (var i = 0; i < 25; i++) + await store.RecordCheckoutAsync("skill", $"file-{i}.txt", "session", DateTimeOffset.UtcNow); + + var removed = await store.PruneAsync(new LogRetentionPolicy( + MaxFileAge: TimeSpan.Zero, MaxFilesPerDirectory: 0, MaxLinesPerFile: 10)); + + Assert.AreEqual(15, removed); + var path = Path.Combine(_tempDir, "skill-resource-usage.jsonl"); + Assert.AreEqual(10, (await File.ReadAllLinesAsync(path)).Length); + } + + // ── Helpers ─────────────────────────────────────────────────────────────── + + private static string WriteSessionFile(string dir, string name, int ageDays) + { + var path = Path.Combine(dir, $"{name}.jsonl"); + File.WriteAllText(path, "{\"x\":1}" + Environment.NewLine); + File.SetLastWriteTimeUtc(path, DateTime.UtcNow.AddDays(-ageDays)); + return path; + } +} From 5389c1634b22e14ac951c44fd5b334345dae4c61 Mon Sep 17 00:00:00 2001 From: Rockford Lhotka Date: Wed, 3 Jun 2026 10:13:53 +0200 Subject: [PATCH 2/3] Wire log retention to k8s config + line-trim persistent session files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The retention knobs added by this PR were never bound to configuration — WithDreaming() took no callback, so DreamOptions (including the new LogRetention* values, and the pre-existing Dream:CronSchedule) were always defaults. Bind the Dream section in Program.cs and surface the four knobs as agent.logRetention.* Helm values + Dream__* ConfigMap keys. The ConfigMap uses `dig` rather than `default` so an explicit false/0 (which disable a dimension) are honoured instead of being swallowed as "empty". Values are sized from observed live traffic: maxFileAge 30d (floored at the widest dream query window so pruning never starves a pass), maxFilesPerDirectory 1000 (backstop; age pruning is the real control), maxLinesPerFile 10000 (~11 MB for the wisp log at ~1.1 KB/line, vs the never-trimming 50k code default). Also close the persistent-session gap: age/count pruning never reaps a continuously-written {sessionId}.jsonl (blazor-session, cli-session) because it is never aged out and never the oldest file — on the live cluster the UI session's tool-call log alone was 2 MB and growing. Add JsonlLogRetention.TrimSessionFilesAsync, which line-trims each surviving session file to MaxLinesPerFile while holding the writer's own per-session semaphore so a trim can never race an append; wire it into the three per-session stores after their age/count prune. A byte-size gate skips the tiny ephemeral files unread. Tests: per-session line-trim (over-budget trimmed, under-budget skipped, correct lock key, store integration) and config-binding regression (TimeSpan/bool/0 shapes from the ConfigMap). Docs updated (dream-service, agent-host, values). Co-Authored-By: Claude Opus 4.8 (1M context) --- deploy/helm/rockbot/templates/configmap.yaml | 10 +++ deploy/helm/rockbot/values.yaml | 22 ++++++ docs/agent-host.md | 5 ++ docs/dream-service.md | 51 +++++++++++++- src/RockBot.Agent/Program.cs | 2 +- src/RockBot.Host.Abstractions/DreamOptions.cs | 11 +-- src/RockBot.Host/FileFeedbackStore.cs | 17 +++-- src/RockBot.Host/FileSkillUsageStore.cs | 17 +++-- src/RockBot.Host/FileToolCallLog.cs | 17 +++-- src/RockBot.Host/JsonlLogRetention.cs | 62 +++++++++++++++++ .../DreamOptionsRetentionBindingTests.cs | 67 ++++++++++++++++++ .../JsonlLogRetentionTests.cs | 69 +++++++++++++++++++ 12 files changed, 330 insertions(+), 20 deletions(-) create mode 100644 tests/RockBot.Host.Tests/DreamOptionsRetentionBindingTests.cs diff --git a/deploy/helm/rockbot/templates/configmap.yaml b/deploy/helm/rockbot/templates/configmap.yaml index 36892154..2a600572 100644 --- a/deploy/helm/rockbot/templates/configmap.yaml +++ b/deploy/helm/rockbot/templates/configmap.yaml @@ -28,6 +28,16 @@ data: AgentProfile__BasePath: "/data/agent" AgentProfile__TierRoutingLogMaxEntries: {{ .Values.agent.tierRoutingLogMaxEntries | default 1500 | quote }} Memory__BasePath: "/data/agent/memory" + + # ── Append-only JSONL log retention (applied once per dream cycle) ───────── + # `dig` (not `default`) so an explicit false / 0 — which disable a dimension — + # are honoured rather than swallowed, while a wholly-absent block (e.g. a + # --reuse-values upgrade) still falls back to the chart defaults below. + {{- $logRetention := .Values.agent.logRetention | default dict }} + Dream__LogRetentionEnabled: {{ dig "enabled" true $logRetention | quote }} + Dream__LogRetentionMaxFileAge: {{ dig "maxFileAge" "30.00:00:00" $logRetention | quote }} + Dream__LogRetentionMaxFilesPerDirectory: {{ dig "maxFilesPerDirectory" 1000 $logRetention | quote }} + Dream__LogRetentionMaxLinesPerFile: {{ dig "maxLinesPerFile" 10000 $logRetention | quote }} Skill__BasePath: "/data/agent/skills" McpBridge__ConfigPath: "/data/agent/mcp.json" LlmPricing__ConfigPath: "/data/agent/llm-pricing.json" diff --git a/deploy/helm/rockbot/values.yaml b/deploy/helm/rockbot/values.yaml index b4bcc25a..5435bbcf 100644 --- a/deploy/helm/rockbot/values.yaml +++ b/deploy/helm/rockbot/values.yaml @@ -20,6 +20,28 @@ agent: # get_routing_summary tool). Once the cap is reached the logger trims the oldest # entries on every append. ~500 bytes per entry, so 1500 ≈ 750 KB on disk. tierRoutingLogMaxEntries: 1500 + # Append-only JSONL log retention, applied once per dream cycle (see + # docs/dream-service.md → "Log retention"). Covers skill-usage, tool-call, + # feedback (per-session directories) and skill-resource-usage, + # wisp-executions (single files). Values below are sized from observed live + # traffic; raise them if you query these logs further back in time. + logRetention: + enabled: true + # Per-session directory logs: delete {sessionId}.jsonl files whose last-write + # time is older than this. Floor it at the widest dream query window (skill + # usage looks back 30 days) so pruning never starves a pass. TimeSpan format + # "d.hh:mm:ss". Set to "0" to disable age pruning. + maxFileAge: "30.00:00:00" + # Per-session directory logs: backstop cap on file count after age pruning + # (oldest dropped first). Age pruning is the primary control; this only bites + # during a session storm. Set to 0 to disable count pruning. + maxFilesPerDirectory: 1000 + # Per-file line cap. Applies to the single-file logs (wisp-executions.jsonl, + # skill-resource-usage.jsonl) AND to each per-session file — this is what bounds + # a persistent UI/CLI session's {id}.jsonl, which age/count pruning never reaps. + # At ~1.1 KB/line for wisp records, 10000 ≈ 11 MB, well above the 14-day window + # the wisp dream pass reads. Set to 0 to disable line trimming. + maxLinesPerFile: 10000 resources: requests: cpu: 500m diff --git a/docs/agent-host.md b/docs/agent-host.md index e85d7059..89bec0eb 100644 --- a/docs/agent-host.md +++ b/docs/agent-host.md @@ -155,6 +155,11 @@ One JSON object per line. Per-session semaphores prevent concurrent write races. `QueryRecentAsync` scans all JSONL files to find entries since a given timestamp — used by the dream cycle to gather quality signals for memory consolidation and skill optimization. +These per-session files (along with the skill-usage and tool-call logs) are append-only and are +capped by the dream cycle's [log-retention pass](dream-service.md#pass-0--log-retention) — aged +files are deleted, the directory is held under a file-count cap, and each surviving file is +line-trimmed (so a persistent UI/CLI session that never ages out is still bounded). + ### `SessionSummaryService` Background hosted service that evaluates completed sessions: diff --git a/docs/dream-service.md b/docs/dream-service.md index a72bb27d..91cd2818 100644 --- a/docs/dream-service.md +++ b/docs/dream-service.md @@ -33,9 +33,40 @@ Startup ## Passes -Each dream cycle runs five passes in sequence. Passes that depend on optional services -(`IConversationLog`, `IFeedbackStore`, `ISkillUsageStore`) are skipped when those services are -not registered. +Each dream cycle runs a log-retention pass followed by five knowledge passes in sequence. +Passes that depend on optional services (`IConversationLog`, `IFeedbackStore`, +`ISkillUsageStore`) are skipped when those services are not registered. + +### Pass 0 — Log retention + +Runs **first and unconditionally**, before the knowledge passes — and crucially before the +"fewer than two memories → early return" guard, so the append-only logs are capped on every +cycle even when there is nothing to consolidate. + +The agent's append-only JSONL telemetry logs (skill-usage, tool-call, feedback, +skill-resource-usage, wisp-executions) have no rotation of their own, so without this pass +they grow forever. The pass resolves every registered `IPrunableLog` and applies the +configured `LogRetentionPolicy`. Each log knows its own on-disk shape and delegates the file +work to the shared `JsonlLogRetention` helper: + +| Log | Shape | Retention applied | +|---|---|---| +| skill-usage, tool-call, feedback | per-session directory of `{sessionId}.jsonl` | delete files older than `LogRetentionMaxFileAge` (by last-write time); cap the directory at `LogRetentionMaxFilesPerDirectory` (oldest dropped first); then line-trim each surviving file to `LogRetentionMaxLinesPerFile` under that session's write lock | +| skill-resource-usage, wisp-executions | single append-only file | trim to the last `LogRetentionMaxLinesPerFile` lines (atomic temp-file rewrite, serialized against the writer) | + +Retention is best-effort: a failure pruning one log is logged and does not abort the sweep or +the rest of the dream cycle. A non-positive value disables the corresponding dimension. + +The per-session line-trim is what bounds a *persistent* session file — `blazor-session.jsonl`, +`cli-session.jsonl` — that age/count pruning alone never reaps, because such a file is written +continuously (never aged out by last-write time) and is never the oldest file (never +count-pruned). On a long-running deployment the UI session's tool-call log is the largest single +file; line-trimming holds it to `LogRetentionMaxLinesPerFile`. Trimming reuses the store's own +per-session semaphore, so it can never race a concurrent append. (Scope matches age/count +pruning — top-level `{sessionId}.jsonl` files only; namespaced session files in subdirectories +are not swept.) + +**Enabled/disabled by:** `DreamOptions.LogRetentionEnabled` (default `true`). ### Pass 1 — Memory consolidation @@ -269,9 +300,23 @@ public sealed class DreamOptions // Feature flags public bool PreferenceInferenceEnabled { get; set; } = true; public bool SkillGapEnabled { get; set; } = true; + + // Append-only JSONL log retention (Pass 0) + public bool LogRetentionEnabled { get; set; } = true; + public TimeSpan LogRetentionMaxFileAge { get; set; } = TimeSpan.FromDays(30); // per-session dirs + public int LogRetentionMaxFilesPerDirectory { get; set; } = 1000; // per-session dirs + public int LogRetentionMaxLinesPerFile { get; set; } = 50_000; // single-file logs } ``` +In Kubernetes these are bound from the `Dream` configuration section via the agent ConfigMap +(`Dream__LogRetentionEnabled`, `Dream__LogRetentionMaxFileAge`, +`Dream__LogRetentionMaxFilesPerDirectory`, `Dream__LogRetentionMaxLinesPerFile`), driven by the +`agent.logRetention.*` Helm values. The Helm chart ships tighter, traffic-sized values than the +code defaults (`maxLinesPerFile: 10000` ≈ 11 MB for the wisp log at ~1.1 KB/line). Floor +`maxFileAge` at the widest dream query window (skill usage looks back 30 days) so age pruning +never starves a downstream pass. + --- ## DI registration diff --git a/src/RockBot.Agent/Program.cs b/src/RockBot.Agent/Program.cs index 7c9dac64..daa15b4b 100644 --- a/src/RockBot.Agent/Program.cs +++ b/src/RockBot.Agent/Program.cs @@ -276,7 +276,7 @@ async Task BuildClientForTierAsync(LlmTierConfig config, string tie agent.WithKnowledgeGraph(); agent.WithFailureClusterStore(); agent.WithRepairTickets(); - agent.WithDreaming(); + agent.WithDreaming(opts => builder.Configuration.GetSection("Dream").Bind(opts)); agent.AddToolHandler(); agent.AddMcpToolProxy(); agent.AddFileSystemTools(opts => builder.Configuration.GetSection("FileSystem").Bind(opts)); diff --git a/src/RockBot.Host.Abstractions/DreamOptions.cs b/src/RockBot.Host.Abstractions/DreamOptions.cs index 6c16b027..ff2c0962 100644 --- a/src/RockBot.Host.Abstractions/DreamOptions.cs +++ b/src/RockBot.Host.Abstractions/DreamOptions.cs @@ -304,10 +304,13 @@ public sealed class DreamOptions public int LogRetentionMaxFilesPerDirectory { get; set; } = 1000; /// - /// Ceiling on the number of lines retained in each single-file append-only log - /// (skill-resource-usage.jsonl, wisp-executions.jsonl). When a file exceeds this, - /// the retention pass rewrites it keeping only the most recent lines. Set to zero - /// or negative to disable trimming. Default: 50,000. + /// Ceiling on the number of lines retained in any single JSONL log file. Applies to + /// the single-file append-only logs (skill-resource-usage.jsonl, + /// wisp-executions.jsonl) and to each individual per-session file (e.g. a persistent + /// UI/CLI session's {sessionId}.jsonl that age/count pruning never reaps + /// because it is continuously written). When a file exceeds this, the retention pass + /// rewrites it keeping only the most recent lines. Set to zero or negative to disable + /// trimming. Default: 50,000. /// public int LogRetentionMaxLinesPerFile { get; set; } = 50_000; } diff --git a/src/RockBot.Host/FileFeedbackStore.cs b/src/RockBot.Host/FileFeedbackStore.cs index 92228b05..397499e0 100644 --- a/src/RockBot.Host/FileFeedbackStore.cs +++ b/src/RockBot.Host/FileFeedbackStore.cs @@ -56,12 +56,21 @@ public async Task AppendAsync(FeedbackEntry entry, CancellationToken cancellatio /// /// Per-session JSONL files accumulate one file per session forever. Retention - /// drops whole session files older than the configured window, then caps the - /// total file count. + /// drops whole session files older than the configured window, caps the total + /// file count, then line-trims any surviving file (e.g. a persistent UI/CLI + /// session that never ages out) to the per-file line budget under that session's + /// write lock so the trim can't race an append. /// - public Task PruneAsync(LogRetentionPolicy policy, CancellationToken ct = default) - => JsonlLogRetention.PruneAgedFilesAsync( + public async Task PruneAsync(LogRetentionPolicy policy, CancellationToken ct = default) + { + var removed = await JsonlLogRetention.PruneAgedFilesAsync( _basePath, policy.MaxFileAge, policy.MaxFilesPerDirectory, "*.jsonl", _logger, ct); + removed += await JsonlLogRetention.TrimSessionFilesAsync( + _basePath, policy.MaxLinesPerFile, "*.jsonl", + id => _writeLocks.GetOrAdd(id, _ => new SemaphoreSlim(1, 1)), + _logger, ct); + return removed; + } public async Task> GetBySessionAsync(string sessionId, CancellationToken cancellationToken = default) { diff --git a/src/RockBot.Host/FileSkillUsageStore.cs b/src/RockBot.Host/FileSkillUsageStore.cs index 993bb567..d94d5435 100644 --- a/src/RockBot.Host/FileSkillUsageStore.cs +++ b/src/RockBot.Host/FileSkillUsageStore.cs @@ -55,12 +55,21 @@ public async Task AppendAsync(SkillInvocationEvent evt, CancellationToken ct = d /// /// Per-session JSONL files accumulate one file per session forever. Retention - /// drops whole session files older than the configured window, then caps the - /// total file count. + /// drops whole session files older than the configured window, caps the total + /// file count, then line-trims any surviving file (e.g. a persistent UI/CLI + /// session that never ages out) to the per-file line budget under that session's + /// write lock so the trim can't race an append. /// - public Task PruneAsync(LogRetentionPolicy policy, CancellationToken ct = default) - => JsonlLogRetention.PruneAgedFilesAsync( + public async Task PruneAsync(LogRetentionPolicy policy, CancellationToken ct = default) + { + var removed = await JsonlLogRetention.PruneAgedFilesAsync( _basePath, policy.MaxFileAge, policy.MaxFilesPerDirectory, "*.jsonl", _logger, ct); + removed += await JsonlLogRetention.TrimSessionFilesAsync( + _basePath, policy.MaxLinesPerFile, "*.jsonl", + id => _writeLocks.GetOrAdd(id, _ => new SemaphoreSlim(1, 1)), + _logger, ct); + return removed; + } public async Task> GetBySessionAsync(string sessionId, CancellationToken ct = default) { diff --git a/src/RockBot.Host/FileToolCallLog.cs b/src/RockBot.Host/FileToolCallLog.cs index 8ed30807..75038767 100644 --- a/src/RockBot.Host/FileToolCallLog.cs +++ b/src/RockBot.Host/FileToolCallLog.cs @@ -55,12 +55,21 @@ public async Task AppendAsync(ToolCallEvent evt, CancellationToken ct = default) /// /// Per-session JSONL files accumulate one file per session forever. Retention - /// drops whole session files older than the configured window, then caps the - /// total file count. + /// drops whole session files older than the configured window, caps the total + /// file count, then line-trims any surviving file (e.g. a persistent UI/CLI + /// session that never ages out) to the per-file line budget under that session's + /// write lock so the trim can't race an append. /// - public Task PruneAsync(LogRetentionPolicy policy, CancellationToken ct = default) - => JsonlLogRetention.PruneAgedFilesAsync( + public async Task PruneAsync(LogRetentionPolicy policy, CancellationToken ct = default) + { + var removed = await JsonlLogRetention.PruneAgedFilesAsync( _basePath, policy.MaxFileAge, policy.MaxFilesPerDirectory, "*.jsonl", _logger, ct); + removed += await JsonlLogRetention.TrimSessionFilesAsync( + _basePath, policy.MaxLinesPerFile, "*.jsonl", + id => _writeLocks.GetOrAdd(id, _ => new SemaphoreSlim(1, 1)), + _logger, ct); + return removed; + } public async Task> GetBySessionAsync(string sessionId, CancellationToken ct = default) { diff --git a/src/RockBot.Host/JsonlLogRetention.cs b/src/RockBot.Host/JsonlLogRetention.cs index a1d93c59..22dbe55e 100644 --- a/src/RockBot.Host/JsonlLogRetention.cs +++ b/src/RockBot.Host/JsonlLogRetention.cs @@ -127,6 +127,68 @@ public static async Task TrimToLastLinesAsync( } } + /// + /// Line-trims every file directly under + /// to its last lines. + /// Where drops whole stale files, this bounds a + /// persistent session file (e.g. a long-lived UI or CLI session's + /// {id}.jsonl) that age/count pruning never touches because it is written + /// continuously — never aged out, never the oldest file. Each file is trimmed while + /// holding the writer's own per-session lock, obtained via + /// keyed by the session id (the file name without extension), so a trim never races + /// an append. A non-positive is a no-op. Returns the + /// total number of lines removed across all files. + /// + public static async Task TrimSessionFilesAsync( + string directory, + int maxLines, + string searchPattern, + Func lockFor, + ILogger logger, + CancellationToken ct = default) + { + if (maxLines <= 0 || !Directory.Exists(directory)) + return 0; + + var removed = 0; + try + { + foreach (var file in new DirectoryInfo(directory) + .EnumerateFiles(searchPattern, SearchOption.TopDirectoryOnly)) + { + ct.ThrowIfCancellationRequested(); + + // A file of N bytes can hold at most N+1 lines, so anything smaller than + // the line budget cannot exceed it — skip without opening (most session + // files are tiny; only the rare persistent file needs a read). + if (file.Length < maxLines) + continue; + + var sessionId = Path.GetFileNameWithoutExtension(file.Name); + var sem = lockFor(sessionId); + await sem.WaitAsync(ct); + try + { + removed += await TrimToLastLinesAsync(file.FullName, maxLines, logger, ct); + } + finally + { + sem.Release(); + } + } + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + logger.LogWarning(ex, "JSONL retention: failed to line-trim session files in {Directory}", directory); + } + + return removed; + } + private static bool TryDelete(FileInfo file, ILogger logger) { try diff --git a/tests/RockBot.Host.Tests/DreamOptionsRetentionBindingTests.cs b/tests/RockBot.Host.Tests/DreamOptionsRetentionBindingTests.cs new file mode 100644 index 00000000..ff7c86e9 --- /dev/null +++ b/tests/RockBot.Host.Tests/DreamOptionsRetentionBindingTests.cs @@ -0,0 +1,67 @@ +using Microsoft.Extensions.Configuration; + +namespace RockBot.Host.Tests; + +/// +/// Guards the config wiring added so the Helm ConfigMap's Dream__LogRetention* +/// keys actually reach . The agent binds the Dream +/// section via GetSection("Dream").Bind(opts); these tests reproduce that bind +/// against the exact string shapes the ConfigMap emits — in particular the +/// TimeSpan "d.hh:mm:ss" form, which silently falls back to the default if the +/// binder can't parse it. +/// +[TestClass] +public class DreamOptionsRetentionBindingTests +{ + private static DreamOptions Bind(Dictionary values) + { + var config = new ConfigurationBuilder().AddInMemoryCollection(values).Build(); + var opts = new DreamOptions(); + config.GetSection("Dream").Bind(opts); + return opts; + } + + [TestMethod] + public void Binds_RetentionKnobs_FromConfigMapStringShapes() + { + // Exactly what configmap.yaml renders (env vars use "__" as the section separator). + var opts = Bind(new Dictionary + { + ["Dream:LogRetentionEnabled"] = "false", + ["Dream:LogRetentionMaxFileAge"] = "30.00:00:00", + ["Dream:LogRetentionMaxFilesPerDirectory"] = "1000", + ["Dream:LogRetentionMaxLinesPerFile"] = "10000", + }); + + Assert.IsFalse(opts.LogRetentionEnabled); + Assert.AreEqual(TimeSpan.FromDays(30), opts.LogRetentionMaxFileAge); + Assert.AreEqual(1000, opts.LogRetentionMaxFilesPerDirectory); + Assert.AreEqual(10_000, opts.LogRetentionMaxLinesPerFile); + } + + [TestMethod] + public void MissingSection_LeavesCodeDefaults() + { + var opts = Bind(new Dictionary { ["Other:Key"] = "x" }); + + Assert.IsTrue(opts.LogRetentionEnabled); + Assert.AreEqual(TimeSpan.FromDays(30), opts.LogRetentionMaxFileAge); + Assert.AreEqual(1000, opts.LogRetentionMaxFilesPerDirectory); + Assert.AreEqual(50_000, opts.LogRetentionMaxLinesPerFile); + } + + [TestMethod] + public void ZeroValues_DisableDimensions_AndBind() + { + var opts = Bind(new Dictionary + { + ["Dream:LogRetentionMaxFileAge"] = "0", + ["Dream:LogRetentionMaxFilesPerDirectory"] = "0", + ["Dream:LogRetentionMaxLinesPerFile"] = "0", + }); + + Assert.AreEqual(TimeSpan.Zero, opts.LogRetentionMaxFileAge); + Assert.AreEqual(0, opts.LogRetentionMaxFilesPerDirectory); + Assert.AreEqual(0, opts.LogRetentionMaxLinesPerFile); + } +} diff --git a/tests/RockBot.Host.Tests/JsonlLogRetentionTests.cs b/tests/RockBot.Host.Tests/JsonlLogRetentionTests.cs index 69d52481..d510ad8d 100644 --- a/tests/RockBot.Host.Tests/JsonlLogRetentionTests.cs +++ b/tests/RockBot.Host.Tests/JsonlLogRetentionTests.cs @@ -1,3 +1,4 @@ +using System.Collections.Concurrent; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; @@ -210,6 +211,74 @@ public async Task SingleFileStore_PruneAsync_TrimsToLineBudget() Assert.AreEqual(10, (await File.ReadAllLinesAsync(path)).Length); } + // ── TrimSessionFilesAsync ───────────────────────────────────────────────── + + [TestMethod] + public async Task TrimSessionFiles_TrimsOverBudget_SkipsUnderBudget_AndLocksBySession() + { + var dir = Path.Combine(_tempDir, "sessions"); + Directory.CreateDirectory(dir); + + // Persistent session: over the line budget — must be trimmed. + var big = Path.Combine(dir, "blazor-session.jsonl"); + await File.WriteAllLinesAsync(big, Enumerable.Range(0, 100).Select(i => $"{{\"n\":{i}}}").ToArray()); + + // Ephemeral session: under the byte gate — must be left untouched. + var small = Path.Combine(dir, "subagent-x.jsonl"); + await File.WriteAllLinesAsync(small, new[] { "a", "b", "c" }); + + var locks = new ConcurrentDictionary(); + var removed = await JsonlLogRetention.TrimSessionFilesAsync( + dir, maxLines: 10, "*.jsonl", + id => locks.GetOrAdd(id, _ => new SemaphoreSlim(1, 1)), + NullLogger.Instance); + + Assert.AreEqual(90, removed); + var keptBig = await File.ReadAllLinesAsync(big); + Assert.AreEqual(10, keptBig.Length); + Assert.AreEqual("{\"n\":90}", keptBig[0]); + Assert.AreEqual("{\"n\":99}", keptBig[^1]); + Assert.AreEqual(3, (await File.ReadAllLinesAsync(small)).Length); + // The trim acquired the writer's lock for the over-budget session, keyed by + // file name without extension — i.e. the exact SessionId the store writes under. + Assert.IsTrue(locks.ContainsKey("blazor-session")); + } + + [TestMethod] + public async Task TrimSessionFiles_NonPositiveMaxLines_IsNoOp() + { + var dir = Path.Combine(_tempDir, "sessions"); + Directory.CreateDirectory(dir); + var f = Path.Combine(dir, "s.jsonl"); + await File.WriteAllLinesAsync(f, Enumerable.Range(0, 50).Select(i => i.ToString()).ToArray()); + + var removed = await JsonlLogRetention.TrimSessionFilesAsync( + dir, maxLines: 0, "*.jsonl", _ => new SemaphoreSlim(1, 1), NullLogger.Instance); + + Assert.AreEqual(0, removed); + Assert.AreEqual(50, (await File.ReadAllLinesAsync(f)).Length); + } + + [TestMethod] + public async Task PerSessionStore_PruneAsync_LineTrimsPersistentSessionFile() + { + var skillOptions = Options.Create(new SkillOptions { UsageBasePath = Path.Combine(_tempDir, "skill-usage") }); + var profileOptions = Options.Create(new AgentProfileOptions { BasePath = _tempDir }); + var store = new FileSkillUsageStore(skillOptions, profileOptions, NullLogger.Instance); + + // A single, continuously-written session — age (fresh mtime) and count (only + // file) never reap it; only the per-file line trim bounds it. + for (var i = 0; i < 30; i++) + await store.AppendAsync(new SkillInvocationEvent( + Id: i.ToString(), SkillName: "x", SessionId: "blazor-session", Timestamp: DateTimeOffset.UtcNow)); + + var removed = await store.PruneAsync(new LogRetentionPolicy( + MaxFileAge: TimeSpan.FromDays(30), MaxFilesPerDirectory: 1000, MaxLinesPerFile: 10)); + + Assert.AreEqual(20, removed); + Assert.AreEqual(10, (await store.GetBySessionAsync("blazor-session")).Count); + } + // ── Helpers ─────────────────────────────────────────────────────────────── private static string WriteSessionFile(string dir, string name, int ageDays) From f17dbe2ef39f7c85badf5ea44c065adc5729499b Mon Sep 17 00:00:00 2001 From: Rockford Lhotka Date: Fri, 5 Jun 2026 08:39:50 +0200 Subject: [PATCH 3/3] Bump version to 0.12.27 Tag for the log-retention image deployed to the live cluster for testing. Co-Authored-By: Claude Opus 4.8 (1M context) --- Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Build.props b/Directory.Build.props index 90b7a8bf..12208b2e 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -9,7 +9,7 @@ -0.12.26 +0.12.27