From 40da3e9d97c276c496293fd374a9104090817977 Mon Sep 17 00:00:00 2001 From: keyolk Date: Fri, 28 Aug 2026 22:01:13 +0900 Subject: [PATCH 1/2] feat: back cross-session search with a SQLite FTS5 index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ctrl+S re-read every transcript on each query. On a 2.5 GB / 995-session corpus that is 0.6-1.9s per search, growing with history. Index transcript content into ~/.claude/.ccx-index.db and query that instead, falling back to the full scan whenever the index cannot answer a query. Measured on the real corpus: worktree (broad) 1765ms -> 44ms goreleaser (rare) 674ms -> 5ms 워크트리 (Korean) 627ms -> 24ms tool:Bash worktree 1647ms -> 47ms Index is 401 MB, first build ~70s, incremental refresh 25-162ms. Choices worth knowing, all forced by measurement rather than taste: - modernc.org/sqlite (pure Go) keeps the goreleaser cross-compile matrix working; a CGO driver would break darwin/linux x amd64/arm64. - tokenize='trigram' preserves the existing strings.Contains semantics and Korean matching. unicode61 found 1930 hits where the scan found 10471, so it is not a drop-in. - detail=full is required, not chosen: trigram matching is internally a phrase query and FTS5 rejects those unless detail=full, which rules out the smaller layouts. - content='' with contentless_delete=1 avoids storing a second copy of text that is already on disk (410 MB -> 284 MB on a sample) while still allowing a changed file's rows to be deleted and reindexed. tool_result is deliberately not indexed: it is half the corpus and mostly file dumps, and including it nearly doubles the index. The modal says "tool output not indexed" so the gap is visible rather than silent, and queries with a sub-trigram term fall back to the scan automatically. Also fixes a pre-existing bug in executeSearch: it ranged the same result channel from two goroutines, so results were split non-deterministically and one goroutine's share was thrown away. Ctrl+S has been dropping results. --- README.md | 20 +- go.mod | 12 +- go.sum | 20 + internal/session/index.go | 626 +++++++++++++++++++++++++++++++ internal/session/index_test.go | 370 ++++++++++++++++++ internal/session/search_index.go | 220 +++++++++++ internal/tui/app.go | 8 +- internal/tui/search.go | 55 ++- 8 files changed, 1310 insertions(+), 21 deletions(-) create mode 100644 internal/session/index.go create mode 100644 internal/session/index_test.go create mode 100644 internal/session/search_index.go diff --git a/README.md b/README.md index a3f7b89..10a53a0 100644 --- a/README.md +++ b/README.md @@ -189,12 +189,28 @@ Search inside conversation content across all sessions (`Ctrl+S` or `:search`). - `tool:ToolName` — Only search specific tool calls **Features:** -- Searches text, tool inputs/outputs, thinking blocks -- Results stream in real-time as they're found +- Searches text, tool inputs, thinking blocks, and system tags +- Results are ordered newest session first - Matched terms are highlighted in snippets - Press `Enter` to jump directly to the matching message - Press `/` to edit the query +**Index:** + +Search is backed by a SQLite FTS5 index at `~/.claude/.ccx-index.db` (~400 MB for +a 2.5 GB transcript corpus), which makes a typical query take tens of +milliseconds instead of seconds. + +- The index refreshes when you search: only transcripts whose size or mtime + changed are re-read, so results are always current. The first build takes + about a minute; after that a refresh is milliseconds. +- **Tool output (`tool_result`) is not indexed.** It is half the corpus and + mostly file dumps, so including it would roughly double the index for little + search value. The modal says `tool output not indexed` when this applies. +- Queries with a term shorter than 3 characters fall back to a full scan + automatically — a trigram index cannot match them. +- The index is a cache: deleting it is safe, and a corrupt one is rebuilt. + **Example queries:** ``` database migration # Find both terms diff --git a/go.mod b/go.mod index f10d999..389e264 100644 --- a/go.mod +++ b/go.mod @@ -22,16 +22,24 @@ require ( github.com/clipperhouse/displaywidth v0.9.0 // indirect github.com/clipperhouse/stringish v0.1.1 // indirect github.com/clipperhouse/uax29/v2 v2.5.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect + github.com/google/uuid v1.6.0 // indirect github.com/lucasb-eyer/go-colorful v1.3.0 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.16.0 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/sahilm/fuzzy v0.1.1 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect - golang.org/x/sys v0.42.0 // indirect + golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.3.8 // indirect + modernc.org/libc v1.74.4 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect + modernc.org/sqlite v1.57.0 // indirect ) diff --git a/go.sum b/go.sum index 62b62ee..bad9f72 100644 --- a/go.sum +++ b/go.sum @@ -26,14 +26,20 @@ github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfa github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= github.com/clipperhouse/uax29/v2 v2.5.0 h1:x7T0T4eTHDONxFJsL94uKNKPHrclyFI0lm7+w94cO8U= github.com/clipperhouse/uax29/v2 v2.5.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag= github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= @@ -44,6 +50,10 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/sahilm/fuzzy v0.1.1 h1:ceu5RHF8DGgoi+/dR5PsECjCDH1BE3Fnmpo7aVXOdRA= @@ -56,6 +66,8 @@ golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= @@ -64,3 +76,11 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k= +modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/sqlite v1.57.0 h1:qNQP6xnx5M0ISNtlnxoOX0+cD5bJ0/gr9aMmndFczzg= +modernc.org/sqlite v1.57.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ= diff --git a/internal/session/index.go b/internal/session/index.go new file mode 100644 index 0000000..d066a96 --- /dev/null +++ b/internal/session/index.go @@ -0,0 +1,626 @@ +package session + +import ( + "bufio" + "context" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" + + _ "modernc.org/sqlite" +) + +// Cross-session content search is backed by an FTS5 index so that a query does +// not have to re-read every transcript (~2.5 GB) on each keystroke. +// +// Design decisions worth knowing before changing anything here: +// +// - tokenize='trigram' — the TUI's search has always been substring-based +// (strings.Contains), and transcripts are heavily Korean. The default +// unicode61 tokenizer splits on whitespace/punctuation, so it misses both +// mid-word matches ("orktre") and Korean ("워크트리" inside "워크트리를"). +// Trigram reproduces the existing semantics; unicode61 measurably does not +// (1930 vs 10471 hits for "worktree" on a 10% corpus sample). +// +// - content='' (contentless) — the block text also lives in the transcript on +// disk, so storing a second copy inside FTS is pure overhead. Contentless +// cut the index from 410 MB to 284 MB on the sample. Snippets are rebuilt by +// re-reading the one matching line via its byte offset. +// +// - detail=full — required, not chosen. Trigram matching is internally a +// phrase query, and FTS5 rejects phrase queries unless detail=full. That +// rules out the smaller detail=none layout. +// +// - contentless_delete=1 — lets us delete a file's rows without knowing their +// original text, which is what makes incremental reindexing possible. +// +// - tool_result blocks are not indexed. They are half of all transcript text +// but are mostly file dumps and command output; excluding them takes the +// full-corpus index from ~850 MB to ~450 MB. + +const ( + // indexSchemaVersion is bumped whenever the schema or the set of indexed + // blocks changes in a way that makes an existing index wrong rather than + // merely stale. On mismatch the index is dropped and rebuilt. + indexSchemaVersion = 1 + + // minTrigramTerm is the shortest term a trigram index can match. Queries + // containing anything shorter cannot be answered from the index at all. + minTrigramTerm = 3 +) + +// Index is an FTS5 index over transcript content blocks. +type Index struct { + db *sql.DB + path string +} + +func indexFilePath(claudeDir string) string { + return filepath.Join(claudeDir, ".ccx-index.db") +} + +// OpenIndex opens (creating if needed) the content index for claudeDir. +// A schema-version mismatch or an unreadable file rebuilds from scratch rather +// than failing: the index is a cache, and a corrupt cache must not break search. +func OpenIndex(claudeDir string) (*Index, error) { + path := indexFilePath(claudeDir) + + idx, err := openIndexAt(path) + if err == nil { + return idx, nil + } + + // Unusable index — discard and retry once from empty. + os.Remove(path) + os.Remove(path + "-wal") + os.Remove(path + "-shm") + return openIndexAt(path) +} + +func openIndexAt(path string) (*Index, error) { + // synchronous=normal: losing the tail of the index after a crash costs a + // reindex of a few sessions, which is cheaper than fsync per commit. + dsn := path + "?_pragma=journal_mode(wal)&_pragma=synchronous(normal)&_pragma=busy_timeout(5000)" + db, err := sql.Open("sqlite", dsn) + if err != nil { + return nil, err + } + + idx := &Index{db: db, path: path} + if err := idx.migrate(); err != nil { + db.Close() + return nil, err + } + return idx, nil +} + +func (ix *Index) migrate() error { + var version int + row := ix.db.QueryRow(`select value from meta where key='schema_version'`) + if err := row.Scan(&version); err != nil { + if !isMissingTable(err) && !errors.Is(err, sql.ErrNoRows) { + return err + } + version = 0 + } + + if version == indexSchemaVersion { + return nil + } + if version != 0 { + // Stale layout: drop everything and fall through to a fresh create. + for _, stmt := range []string{ + `drop table if exists blocks`, + `drop table if exists locs`, + `drop table if exists files`, + `drop table if exists meta`, + } { + if _, err := ix.db.Exec(stmt); err != nil { + return err + } + } + } + + schema := []string{ + `create table if not exists meta(key text primary key, value text)`, + + // One row per indexed transcript file. mod_unix/size detect changes. + `create table if not exists files( + id integer primary key, + path text unique not null, + mod_unix integer not null, + size integer not null + )`, + + // Locations of indexed blocks. Shares rowids with the FTS table, which + // is what lets us delete a file's blocks and filter by role/tool. + `create table if not exists locs( + rowid integer primary key, + file_id integer not null, + line_off integer not null, + block_idx integer not null, + role text not null, + tool text not null + )`, + `create index if not exists locs_file on locs(file_id)`, + + `create virtual table if not exists blocks using fts5( + body, + tokenize='trigram', + content='', + contentless_delete=1, + detail=full + )`, + } + for _, stmt := range schema { + if _, err := ix.db.Exec(stmt); err != nil { + return err + } + } + + _, err := ix.db.Exec( + `insert into meta(key,value) values('schema_version',?) + on conflict(key) do update set value=excluded.value`, + fmt.Sprint(indexSchemaVersion)) + return err +} + +func isMissingTable(err error) bool { + return err != nil && strings.Contains(err.Error(), "no such table") +} + +func (ix *Index) Close() error { + if ix == nil || ix.db == nil { + return nil + } + return ix.db.Close() +} + +// Path returns the on-disk location of the index. +func (ix *Index) Path() string { return ix.path } + +// indexedFile is the stored fingerprint of an already-indexed transcript. +type indexedFile struct { + id int64 + modUnix int64 + size int64 +} + +func (ix *Index) knownFiles() (map[string]indexedFile, error) { + rows, err := ix.db.Query(`select id, path, mod_unix, size from files`) + if err != nil { + return nil, err + } + defer rows.Close() + + known := make(map[string]indexedFile) + for rows.Next() { + var f indexedFile + var p string + if err := rows.Scan(&f.id, &p, &f.modUnix, &f.size); err != nil { + return nil, err + } + known[p] = f + } + return known, rows.Err() +} + +// SyncStats reports what a Sync did, for progress reporting and tests. +type SyncStats struct { + Scanned int // transcripts considered + Indexed int // transcripts (re)indexed + Removed int // transcripts dropped from the index + Blocks int // content blocks written + Elapsed time.Duration // +} + +// Sync brings the index in line with the given sessions: transcripts whose +// mtime or size changed are reindexed, transcripts that disappeared are +// dropped, and unchanged transcripts are skipped. It is safe to call on every +// search — the common case is a stat of each file and no writes. +func (ix *Index) Sync(ctx context.Context, sessions []*Session, progress func(done, total int)) (SyncStats, error) { + start := time.Now() + var stats SyncStats + + known, err := ix.knownFiles() + if err != nil { + return stats, err + } + + type work struct { + path string + modUnix int64 + size int64 + prevID int64 + hasPrev bool + } + + var todo []work + seen := make(map[string]bool, len(sessions)) + + for _, s := range sessions { + if s == nil || s.FilePath == "" || seen[s.FilePath] { + continue + } + seen[s.FilePath] = true + stats.Scanned++ + + fi, err := os.Stat(s.FilePath) + if err != nil { + continue + } + mod, size := fi.ModTime().Unix(), fi.Size() + + prev, ok := known[s.FilePath] + if ok && prev.modUnix == mod && prev.size == size { + continue // unchanged + } + todo = append(todo, work{ + path: s.FilePath, modUnix: mod, size: size, + prevID: prev.id, hasPrev: ok, + }) + } + + // Transcripts that vanished (project deleted, session pruned). + for path, f := range known { + if !seen[path] { + if err := ix.dropFile(f.id); err != nil { + return stats, err + } + stats.Removed++ + } + } + + for i, w := range todo { + select { + case <-ctx.Done(): + stats.Elapsed = time.Since(start) + return stats, ctx.Err() + default: + } + + if progress != nil { + progress(i, len(todo)) + } + + n, err := ix.indexFile(w.path, w.modUnix, w.size, w.prevID, w.hasPrev) + if err != nil { + // A single unreadable transcript must not abort the whole sync. + continue + } + stats.Indexed++ + stats.Blocks += n + } + if progress != nil && len(todo) > 0 { + progress(len(todo), len(todo)) + } + + stats.Elapsed = time.Since(start) + return stats, nil +} + +func (ix *Index) dropFile(fileID int64) error { + tx, err := ix.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + if _, err := tx.Exec( + `delete from blocks where rowid in (select rowid from locs where file_id=?)`, fileID); err != nil { + return err + } + if _, err := tx.Exec(`delete from locs where file_id=?`, fileID); err != nil { + return err + } + if _, err := tx.Exec(`delete from files where id=?`, fileID); err != nil { + return err + } + return tx.Commit() +} + +// indexFile reindexes one transcript, replacing any previously indexed rows. +func (ix *Index) indexFile(path string, modUnix, size, prevID int64, hasPrev bool) (int, error) { + f, err := os.Open(path) + if err != nil { + return 0, err + } + defer f.Close() + + tx, err := ix.db.Begin() + if err != nil { + return 0, err + } + defer tx.Rollback() + + fileID := prevID + if hasPrev { + if _, err := tx.Exec( + `delete from blocks where rowid in (select rowid from locs where file_id=?)`, fileID); err != nil { + return 0, err + } + if _, err := tx.Exec(`delete from locs where file_id=?`, fileID); err != nil { + return 0, err + } + if _, err := tx.Exec(`update files set mod_unix=?, size=? where id=?`, modUnix, size, fileID); err != nil { + return 0, err + } + } else { + res, err := tx.Exec(`insert into files(path, mod_unix, size) values (?,?,?)`, path, modUnix, size) + if err != nil { + return 0, err + } + if fileID, err = res.LastInsertId(); err != nil { + return 0, err + } + } + + insBlock, err := tx.Prepare(`insert into blocks(rowid, body) values (?,?)`) + if err != nil { + return 0, err + } + insLoc, err := tx.Prepare( + `insert into locs(rowid, file_id, line_off, block_idx, role, tool) values (?,?,?,?,?,?)`) + if err != nil { + return 0, err + } + + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 256*1024), 10*1024*1024) + + var lineOff int64 + var count int + + for sc.Scan() { + line := sc.Bytes() + off := lineOff + lineOff += int64(len(line)) + 1 // +1 for the newline the scanner strips + + if len(line) == 0 { + continue + } + entry, err := ParseEntry(string(line)) + if err != nil || entry.IsMeta { + continue + } + + for i := range entry.Content { + block := &entry.Content[i] + if !indexableBlock(block) { + continue + } + body := blockSearchText(block) + if body == "" { + continue + } + + // rowid is assigned by SQLite; both tables must agree on it, so we + // take the one FTS picked and reuse it for the location row. + res, err := insBlock.Exec(nil, body) + if err != nil { + return 0, err + } + rowid, err := res.LastInsertId() + if err != nil { + return 0, err + } + if _, err := insLoc.Exec(rowid, fileID, off, i, entry.Role, block.ToolName); err != nil { + return 0, err + } + count++ + } + } + if err := sc.Err(); err != nil { + return 0, err + } + + if err := tx.Commit(); err != nil { + return 0, err + } + return count, nil +} + +// indexableBlock reports whether a block's text goes into the index. +// +// tool_result is the one deliberate omission: it is roughly half of all +// transcript text but is dominated by file dumps and command output, and +// indexing it nearly doubles the index for little search value. Queries that +// need it fall back to the full scan (see IndexCoverage). +// +// Everything else the scan can match must be listed here, or the index silently +// returns fewer results than the scan for the same query. system_tag in +// particular carries command names and system reminders that users do search +// for, at ~0.6% of the corpus. +func indexableBlock(b *ContentBlock) bool { + // Empty bodies are skipped by the caller, so image blocks and other + // text-free types drop out on their own. + return b.Type != "tool_result" +} + +// Stats describes the current index contents. +type Stats struct { + Files int + Blocks int + Bytes int64 +} + +func (ix *Index) Stats() (Stats, error) { + var s Stats + if err := ix.db.QueryRow(`select count(*) from files`).Scan(&s.Files); err != nil { + return s, err + } + if err := ix.db.QueryRow(`select count(*) from locs`).Scan(&s.Blocks); err != nil { + return s, err + } + if fi, err := os.Stat(ix.path); err == nil { + s.Bytes = fi.Size() + } + return s, nil +} + +// Optimize compacts the FTS index. Worth running after a large rebuild; not +// needed on the incremental path. +func (ix *Index) Optimize() error { + _, err := ix.db.Exec(`insert into blocks(blocks) values('optimize')`) + return err +} + +// --- querying --------------------------------------------------------------- + +// IndexCoverage says whether the index can answer a query on its own. +type IndexCoverage int + +const ( + // CoverageFull means index results are equivalent to a full scan. + CoverageFull IndexCoverage = iota + // CoverageNone means the query must use the full scan; the index cannot + // answer it (a term shorter than a trigram, or an empty query). + CoverageNone + // CoveragePartial means the index answers the query but over a subset of + // blocks: tool_result content is not indexed, so a full scan would find + // strictly more. + CoveragePartial +) + +// Coverage reports how well the index can serve q. +func (ix *Index) Coverage(q SearchQuery) IndexCoverage { + if q.IsEmpty() { + return CoverageNone + } + // Trigram cannot match anything shorter than three characters. + for _, t := range q.Terms { + if len([]rune(t)) < minTrigramTerm { + return CoverageNone + } + } + for _, p := range q.Phrases { + if len([]rune(p)) < minTrigramTerm { + return CoverageNone + } + } + // Exclusions are applied after retrieval, so a short one is harmless. + + // A tool: filter restricts to tool_use blocks, which are fully indexed. + if q.ToolName != "" { + return CoverageFull + } + return CoveragePartial +} + +// fts5Quote renders s as an FTS5 string literal. Trigram matching treats the +// content literally, so every term becomes a quoted phrase; the only character +// needing care is the double quote, which doubles. +func fts5Quote(s string) string { + return `"` + strings.ReplaceAll(s, `"`, `""`) + `"` +} + +// buildMatchExpr renders the positive part of a query as an FTS5 MATCH +// expression. Exclusions are intentionally left out: they are substring +// exclusions over the block text, which is cheaper and more faithful to apply +// after the rows come back. +func buildMatchExpr(q SearchQuery) string { + var parts []string + for _, t := range q.Terms { + parts = append(parts, fts5Quote(t)) + } + for _, p := range q.Phrases { + parts = append(parts, fts5Quote(p)) + } + if q.ToolName != "" && !strings.HasSuffix(q.ToolName, "*") { + // A concrete tool name is also indexed in the block body (tool_use + // bodies are "name input"), so it usefully narrows the FTS scan. A + // prefix filter is left to the post-filter. + parts = append(parts, fts5Quote(q.ToolName)) + } + return strings.Join(parts, " AND ") +} + +// indexHit is one row from the index: where the block lives, nothing more. +type indexHit struct { + path string + lineOff int64 + blockIdx int +} + +// queryIndex returns matching block locations, newest session first. +func (ix *Index) queryIndex(ctx context.Context, q SearchQuery, allowed map[string]*Session, limit int) ([]indexHit, error) { + expr := buildMatchExpr(q) + if expr == "" { + return nil, nil + } + + var ( + where []string + args []any + ) + args = append(args, expr) + + if q.Role != "" { + where = append(where, `locs.role = ?`) + args = append(args, q.Role) + } + if q.ToolName != "" { + if strings.HasSuffix(q.ToolName, "*") { + where = append(where, `lower(locs.tool) like ?`) + args = append(args, strings.ToLower(strings.TrimSuffix(q.ToolName, "*"))+"%") + } else { + where = append(where, `lower(locs.tool) = ?`) + args = append(args, strings.ToLower(q.ToolName)) + } + } + + sqlText := `select files.path, locs.line_off, locs.block_idx + from blocks + join locs on locs.rowid = blocks.rowid + join files on files.id = locs.file_id + where blocks match ?` + if len(where) > 0 { + sqlText += " and " + strings.Join(where, " and ") + } + // Ordering is applied in Go against session mtime; SQL only bounds the work. + if limit > 0 { + sqlText += fmt.Sprintf(" limit %d", limit) + } + + rows, err := ix.db.QueryContext(ctx, sqlText, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + var hits []indexHit + for rows.Next() { + var h indexHit + if err := rows.Scan(&h.path, &h.lineOff, &h.blockIdx); err != nil { + return nil, err + } + if allowed != nil { + if _, ok := allowed[h.path]; !ok { + continue + } + } + hits = append(hits, h) + } + if err := rows.Err(); err != nil { + return nil, err + } + + // Newest session first, matching the session browser's ordering; within a + // session, transcript order. + sort.SliceStable(hits, func(i, j int) bool { + si, sj := allowed[hits[i].path], allowed[hits[j].path] + if si != nil && sj != nil && !si.ModTime.Equal(sj.ModTime) { + return si.ModTime.After(sj.ModTime) + } + if hits[i].path != hits[j].path { + return hits[i].path < hits[j].path + } + return hits[i].lineOff < hits[j].lineOff + }) + return hits, nil +} diff --git a/internal/session/index_test.go b/internal/session/index_test.go new file mode 100644 index 0000000..49b0a50 --- /dev/null +++ b/internal/session/index_test.go @@ -0,0 +1,370 @@ +package session + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "testing" + "time" +) + +// writeIndexTranscript writes a .jsonl transcript and returns a Session for it. +func writeIndexTranscript(t *testing.T, dir, name string, lines []string) *Session { + t.Helper() + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")+"\n"), 0o644); err != nil { + t.Fatal(err) + } + fi, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + return &Session{FilePath: path, ModTime: fi.ModTime(), ProjectName: name} +} + +func userLine(text string) string { + return fmt.Sprintf(`{"type":"user","message":{"role":"user","content":[{"type":"text","text":%q}]}}`, text) +} + +func assistantLine(text string) string { + return fmt.Sprintf(`{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":%q}]}}`, text) +} + +func toolLine(name, input string) string { + return fmt.Sprintf(`{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","name":%q,"input":{"command":%q}}]}}`, name, input) +} + +func toolResultLine(text string) string { + return fmt.Sprintf(`{"type":"user","message":{"role":"user","content":[{"type":"tool_result","content":%q}]}}`, text) +} + +func openTestIndex(t *testing.T) (*Index, string) { + t.Helper() + dir := t.TempDir() + ix, err := OpenIndex(dir) + if err != nil { + t.Fatalf("OpenIndex: %v", err) + } + t.Cleanup(func() { ix.Close() }) + return ix, dir +} + +func syncAll(t *testing.T, ix *Index, sessions []*Session) SyncStats { + t.Helper() + stats, err := ix.Sync(context.Background(), sessions, nil) + if err != nil { + t.Fatalf("Sync: %v", err) + } + return stats +} + +func searchIdx(t *testing.T, ix *Index, sessions []*Session, query string) ([]SearchResult, SearchMode) { + t.Helper() + res, mode, err := SearchWithIndex(context.Background(), ix, sessions, ParseSearchQuery(query), 0) + if err != nil { + t.Fatalf("SearchWithIndex(%q): %v", query, err) + } + return res, mode +} + +func TestIndexFindsBasicMatches(t *testing.T) { + ix, dir := openTestIndex(t) + s := writeIndexTranscript(t, dir, "a.jsonl", []string{ + userLine("please prune the worktree"), + assistantLine("removed the stale worktree entry"), + userLine("unrelated content"), + }) + syncAll(t, ix, []*Session{s}) + + res, mode := searchIdx(t, ix, []*Session{s}, "worktree") + if len(res) != 2 { + t.Fatalf("hits = %d, want 2", len(res)) + } + if mode != SearchModeIndexPartial { + t.Errorf("mode = %v, want index", mode) + } +} + +// The index must not change what a search means. Any divergence from the +// full scan is a bug, so compare the two on the same corpus. +func TestIndexResultsMatchFullScan(t *testing.T) { + ix, dir := openTestIndex(t) + sessions := []*Session{ + writeIndexTranscript(t, dir, "a.jsonl", []string{ + userLine("deploy the worktree now"), + assistantLine("WORKTREE uppercase mention"), + toolLine("Bash", "git worktree prune"), + userLine("worktree and prune together"), + }), + writeIndexTranscript(t, dir, "b.jsonl", []string{ + userLine("워크트리를 정리했다"), + assistantLine("only prune here"), + toolLine("Read", "worktree config"), + }), + } + syncAll(t, ix, sessions) + + queries := []string{ + "worktree", + "WORKTREE", + "worktree prune", + "worktree -prune", + `"worktree and prune"`, + "워크트리", + "orktre", + "user:worktree", + "assistant:worktree", + "tool:Bash", + "tool:Bash worktree", + } + + key := func(r SearchResult) string { + return fmt.Sprintf("%s|%s|%s", r.Session.FilePath, r.Block.Type, blockSearchText(r.Block)) + } + keys := func(rs []SearchResult) []string { + out := make([]string, 0, len(rs)) + for _, r := range rs { + out = append(out, key(r)) + } + sort.Strings(out) + return out + } + + for _, q := range queries { + t.Run(q, func(t *testing.T) { + parsed := ParseSearchQuery(q) + scan := collectScan(context.Background(), sessions, parsed, 0) + + // tool_result is not indexed, so only compare on corpora where the + // scan's extra reach cannot matter — this corpus has none. + idx, _, err := SearchWithIndex(context.Background(), ix, sessions, parsed, 0) + if err != nil { + t.Fatal(err) + } + + got, want := keys(idx), keys(scan) + if len(got) != len(want) { + t.Fatalf("index=%d scan=%d\n index=%v\n scan=%v", len(got), len(want), got, want) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("result %d:\n index=%s\n scan=%s", i, got[i], want[i]) + } + } + }) + } +} + +func TestIndexSkipsToolResultAndFallsBack(t *testing.T) { + ix, dir := openTestIndex(t) + s := writeIndexTranscript(t, dir, "a.jsonl", []string{ + toolResultLine("secretmarker lives only in tool output"), + }) + syncAll(t, ix, []*Session{s}) + + // Indexed search cannot see it... + res, mode := searchIdx(t, ix, []*Session{s}, "secretmarker") + if len(res) != 0 { + t.Errorf("index returned %d hits for tool_result-only content, want 0", len(res)) + } + if mode != SearchModeIndexPartial { + t.Errorf("mode = %v, want partial (so the UI can say so)", mode) + } + + // ...but the scan does, which is why partial coverage must be surfaced. + scan := collectScan(context.Background(), []*Session{s}, ParseSearchQuery("secretmarker"), 0) + if len(scan) != 1 { + t.Errorf("scan hits = %d, want 1", len(scan)) + } +} + +// Terms shorter than a trigram cannot be answered by the index at all; the +// search must silently fall back rather than return nothing. +func TestShortTermFallsBackToScan(t *testing.T) { + ix, dir := openTestIndex(t) + s := writeIndexTranscript(t, dir, "a.jsonl", []string{ + userLine("go is short"), + }) + syncAll(t, ix, []*Session{s}) + + if cov := ix.Coverage(ParseSearchQuery("go")); cov != CoverageNone { + t.Errorf("Coverage(2-char) = %v, want CoverageNone", cov) + } + res, mode := searchIdx(t, ix, []*Session{s}, "go") + if mode != SearchModeScan { + t.Errorf("mode = %v, want scan", mode) + } + if len(res) != 1 { + t.Errorf("hits = %d, want 1 (fallback must still find it)", len(res)) + } +} + +func TestIncrementalSyncReindexesOnlyChangedFiles(t *testing.T) { + ix, dir := openTestIndex(t) + a := writeIndexTranscript(t, dir, "a.jsonl", []string{userLine("alpha content")}) + b := writeIndexTranscript(t, dir, "b.jsonl", []string{userLine("bravo content")}) + sessions := []*Session{a, b} + + if st := syncAll(t, ix, sessions); st.Indexed != 2 { + t.Fatalf("initial Indexed = %d, want 2", st.Indexed) + } + // Nothing changed — a re-sync must not rewrite anything. + if st := syncAll(t, ix, sessions); st.Indexed != 0 { + t.Errorf("no-op Indexed = %d, want 0", st.Indexed) + } + + // Append to one file; only it should be reindexed. + f, err := os.OpenFile(a.FilePath, os.O_APPEND|os.O_WRONLY, 0o644) + if err != nil { + t.Fatal(err) + } + f.WriteString(userLine("charlie appended") + "\n") + f.Close() + // Ensure mtime differs even on coarse-grained filesystems. + future := time.Now().Add(2 * time.Second) + os.Chtimes(a.FilePath, future, future) + fi, _ := os.Stat(a.FilePath) + a.ModTime = fi.ModTime() + + if st := syncAll(t, ix, sessions); st.Indexed != 1 { + t.Errorf("after edit Indexed = %d, want 1", st.Indexed) + } + if res, _ := searchIdx(t, ix, sessions, "charlie"); len(res) != 1 { + t.Errorf("appended content hits = %d, want 1", len(res)) + } + // The old content must still be there exactly once — not duplicated by the + // reindex, which is the classic contentless-FTS failure. + if res, _ := searchIdx(t, ix, sessions, "alpha"); len(res) != 1 { + t.Errorf("pre-existing content hits = %d, want 1 (duplicate rows?)", len(res)) + } +} + +func TestSyncDropsDeletedTranscripts(t *testing.T) { + ix, dir := openTestIndex(t) + a := writeIndexTranscript(t, dir, "a.jsonl", []string{userLine("alpha content")}) + b := writeIndexTranscript(t, dir, "b.jsonl", []string{userLine("bravo content")}) + syncAll(t, ix, []*Session{a, b}) + + os.Remove(b.FilePath) + st := syncAll(t, ix, []*Session{a}) + if st.Removed != 1 { + t.Errorf("Removed = %d, want 1", st.Removed) + } + if res, _ := searchIdx(t, ix, []*Session{a}, "bravo"); len(res) != 0 { + t.Errorf("deleted session still returns %d hits", len(res)) + } +} + +// A quote in a query must not produce an FTS5 syntax error. +func TestQuotesInQueryAreEscaped(t *testing.T) { + ix, dir := openTestIndex(t) + s := writeIndexTranscript(t, dir, "a.jsonl", []string{ + userLine(`he said "worktree" loudly`), + }) + syncAll(t, ix, []*Session{s}) + + for _, q := range []string{`said "worktree`, `"he said \"worktree\" loudly"`} { + if _, _, err := SearchWithIndex(context.Background(), ix, []*Session{s}, ParseSearchQuery(q), 0); err != nil { + t.Errorf("query %q: %v", q, err) + } + } +} + +func TestResultsOrderedByRecency(t *testing.T) { + ix, dir := openTestIndex(t) + old := writeIndexTranscript(t, dir, "old.jsonl", []string{userLine("shared marker here")}) + recent := writeIndexTranscript(t, dir, "recent.jsonl", []string{userLine("shared marker here")}) + + past := time.Now().Add(-48 * time.Hour) + os.Chtimes(old.FilePath, past, past) + fi, _ := os.Stat(old.FilePath) + old.ModTime = fi.ModTime() + + sessions := []*Session{old, recent} + syncAll(t, ix, sessions) + + res, _ := searchIdx(t, ix, sessions, "marker") + if len(res) != 2 { + t.Fatalf("hits = %d, want 2", len(res)) + } + if res[0].Session.FilePath != recent.FilePath { + t.Errorf("first result = %s, want the newer session", filepath.Base(res[0].Session.FilePath)) + } +} + +func TestCorruptIndexIsRebuilt(t *testing.T) { + dir := t.TempDir() + path := indexFilePath(dir) + if err := os.WriteFile(path, []byte("this is not a database"), 0o644); err != nil { + t.Fatal(err) + } + + ix, err := OpenIndex(dir) + if err != nil { + t.Fatalf("OpenIndex on corrupt file: %v", err) + } + defer ix.Close() + + s := writeIndexTranscript(t, dir, "a.jsonl", []string{userLine("worktree content")}) + syncAll(t, ix, []*Session{s}) + if res, _ := searchIdx(t, ix, []*Session{s}, "worktree"); len(res) != 1 { + t.Errorf("hits after rebuild = %d, want 1", len(res)) + } +} + +// tool_result is the only block type the index is allowed to omit. Any other +// omission is a silent divergence from the scan, so enumerate the types the +// parser can produce and assert the rule directly. Found the hard way: system_tag +// blocks were dropped, costing real hits on the live corpus. +func TestOnlyToolResultIsExcludedFromIndex(t *testing.T) { + types := []string{ + "text", "tool_use", "thinking", "system_tag", + "redacted_thinking", "server_tool_use", "advisor_tool_result", + } + for _, ty := range types { + b := &ContentBlock{Type: ty, Text: "body"} + if !indexableBlock(b) { + t.Errorf("block type %q is excluded from the index but the scan matches it", ty) + } + } + if indexableBlock(&ContentBlock{Type: "tool_result", Text: "body"}) { + t.Error("tool_result must stay out of the index (it is half the corpus)") + } +} + +// A system_tag block must be findable through the index, end to end. +func TestSystemTagBlocksAreIndexed(t *testing.T) { + ix, dir := openTestIndex(t) + s := writeIndexTranscript(t, dir, "a.jsonl", []string{ + userLine("uniquemarker inside a tag"), + }) + syncAll(t, ix, []*Session{s}) + + res, _ := searchIdx(t, ix, []*Session{s}, "uniquemarker") + scan := collectScan(context.Background(), []*Session{s}, ParseSearchQuery("uniquemarker"), 0) + if len(res) != len(scan) { + t.Errorf("index=%d scan=%d for system_tag content", len(res), len(scan)) + } + if len(res) == 0 { + t.Error("system_tag content not searchable via index") + } +} + +func TestSnippetHighlightsMatch(t *testing.T) { + ix, dir := openTestIndex(t) + s := writeIndexTranscript(t, dir, "a.jsonl", []string{ + userLine("a long preamble that goes on before the worktree token appears here"), + }) + syncAll(t, ix, []*Session{s}) + + res, _ := searchIdx(t, ix, []*Session{s}, "worktree") + if len(res) != 1 { + t.Fatalf("hits = %d, want 1", len(res)) + } + if !strings.Contains(res[0].Snippet, "worktree") { + t.Errorf("snippet lacks the match: %q", res[0].Snippet) + } +} diff --git a/internal/session/search_index.go b/internal/session/search_index.go new file mode 100644 index 0000000..786fc8a --- /dev/null +++ b/internal/session/search_index.go @@ -0,0 +1,220 @@ +package session + +import ( + "bufio" + "context" + "os" + "sort" + "strings" +) + +// SearchMode records how a set of results was produced, so the UI can be honest +// about what was searched. +type SearchMode int + +const ( + // SearchModeScan means every transcript was read end to end. + SearchModeScan SearchMode = iota + // SearchModeIndex means results came from the FTS index and are equivalent + // to a scan for this query. + SearchModeIndex + // SearchModeIndexPartial means results came from the index, which does not + // cover tool_result content. + SearchModeIndexPartial +) + +func (m SearchMode) String() string { + switch m { + case SearchModeIndex: + return "index" + case SearchModeIndexPartial: + return "index (no tool output)" + default: + return "full scan" + } +} + +// SearchWithIndex answers q from the FTS index when it can, and falls back to +// the full scan when it cannot. It returns the mode actually used. +// +// The caller is responsible for having Sync'd the index; this function never +// writes to it, so a stale index yields stale results rather than a stall. +func SearchWithIndex(ctx context.Context, ix *Index, sessions []*Session, q SearchQuery, limit int) ([]SearchResult, SearchMode, error) { + if ix == nil || ix.Coverage(q) == CoverageNone { + return collectScan(ctx, sessions, q, limit), SearchModeScan, nil + } + + bySession := make(map[string]*Session, len(sessions)) + for _, s := range sessions { + if s != nil && s.FilePath != "" { + bySession[s.FilePath] = s + } + } + + // Over-fetch: post-filtering (exclusions, tool prefix) drops some hits, and + // a limit applied in SQL would otherwise silently truncate good results. + sqlLimit := 0 + if limit > 0 { + sqlLimit = limit * 4 + } + + hits, err := ix.queryIndex(ctx, q, bySession, sqlLimit) + if err != nil { + // An index failure is not a search failure — degrade to the scan. + return collectScan(ctx, sessions, q, limit), SearchModeScan, nil + } + + results, err := hydrateHits(ctx, hits, bySession, q, limit) + if err != nil { + return collectScan(ctx, sessions, q, limit), SearchModeScan, nil + } + + mode := SearchModeIndex + if ix.Coverage(q) == CoveragePartial { + mode = SearchModeIndexPartial + } + return results, mode, nil +} + +// hydrateHits turns index locations back into SearchResults by re-reading the +// one transcript line each hit points at. Hits arrive grouped by path, so the +// same file is opened once. +func hydrateHits(ctx context.Context, hits []indexHit, bySession map[string]*Session, q SearchQuery, limit int) ([]SearchResult, error) { + var results []SearchResult + + for i := 0; i < len(hits); { + select { + case <-ctx.Done(): + return results, ctx.Err() + default: + } + + path := hits[i].path + j := i + for j < len(hits) && hits[j].path == path { + j++ + } + group := hits[i:j] + i = j + + sess := bySession[path] + if sess == nil { + continue + } + f, err := os.Open(path) + if err != nil { + continue + } + + for _, h := range group { + line, err := readLineAt(f, h.lineOff) + if err != nil || line == "" { + continue + } + entry, err := ParseEntry(line) + if err != nil || h.blockIdx >= len(entry.Content) { + continue + } + block := &entry.Content[h.blockIdx] + + res, ok := matchBlock(sess, &entry, block, q) + if !ok { + continue + } + results = append(results, res) + if limit > 0 && len(results) >= limit { + f.Close() + return results, nil + } + } + f.Close() + } + return results, nil +} + +// readLineAt reads the single newline-terminated line starting at off. +func readLineAt(f *os.File, off int64) (string, error) { + if _, err := f.Seek(off, 0); err != nil { + return "", err + } + r := bufio.NewReaderSize(f, 256*1024) + sc := bufio.NewScanner(r) + sc.Buffer(make([]byte, 256*1024), 10*1024*1024) + if !sc.Scan() { + return "", sc.Err() + } + return sc.Text(), nil +} + +// matchBlock re-applies the query to a single block. The index narrows +// candidates; this is what makes the result set exactly match the scan's +// semantics, including exclusions and tool-name prefixes. +func matchBlock(sess *Session, entry *Entry, block *ContentBlock, q SearchQuery) (SearchResult, bool) { + if entry.IsMeta { + return SearchResult{}, false + } + if q.Role != "" && entry.Role != q.Role { + return SearchResult{}, false + } + if q.ToolName != "" { + if block.Type != "tool_use" || !toolNameMatches(block.ToolName, q.ToolName) { + return SearchResult{}, false + } + } + + text := blockSearchText(block) + lower := strings.ToLower(text) + + for _, term := range q.Terms { + if !strings.Contains(lower, term) { + return SearchResult{}, false + } + } + for _, phrase := range q.Phrases { + if !strings.Contains(lower, phrase) { + return SearchResult{}, false + } + } + for _, excl := range q.Exclude { + if strings.Contains(lower, excl) { + return SearchResult{}, false + } + } + + // The entry is a loop-local value in the caller; copy it so the returned + // pointer stays valid and distinct per result. + e := *entry + return SearchResult{ + Session: sess, + Entry: &e, + Block: block, + Snippet: buildSnippet(text, q.Terms, q.Phrases), + }, true +} + +// collectScan drains the existing full-scan search into an ordered slice. +// Results are sorted newest session first to match the indexed path. +func collectScan(ctx context.Context, sessions []*Session, q SearchQuery, limit int) []SearchResult { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + var out []SearchResult + for res := range SearchSessions(sessions, q, ctx) { + out = append(out, res) + if limit > 0 && len(out) >= limit*4 { + break // over-fetch, then sort and trim below + } + } + + sort.SliceStable(out, func(i, j int) bool { + si, sj := out[i].Session, out[j].Session + if si == nil || sj == nil { + return false + } + return si.ModTime.After(sj.ModTime) + }) + if limit > 0 && len(out) > limit { + out = out[:limit] + } + return out +} diff --git a/internal/tui/app.go b/internal/tui/app.go index 1cbd6f3..3ed8fa5 100644 --- a/internal/tui/app.go +++ b/internal/tui/app.go @@ -574,6 +574,12 @@ type App struct { searchResultList list.Model searchLoading bool searchCancel context.CancelFunc + searchMode session.SearchMode + + // contentIndex is the FTS index backing cross-session search. It is opened + // lazily on the first search and owned by the main loop; search commands + // only read it. + contentIndex *session.Index } // selectedSession returns the currently selected session from the session list. @@ -1342,7 +1348,7 @@ func (a *App) Update(msg tea.Msg) (tea.Model, tea.Cmd) { return a, nil case searchBatchMsg: - a.updateSearchResults(msg.results) + a.updateSearchResults(msg.results, msg.mode) return a, nil case refsExtractedMsg: diff --git a/internal/tui/search.go b/internal/tui/search.go index 0f252e7..294853a 100644 --- a/internal/tui/search.go +++ b/internal/tui/search.go @@ -88,29 +88,42 @@ func (a *App) executeSearch() tea.Cmd { parsed := session.ParseSearchQuery(query) - return func() tea.Msg { - results := session.SearchSessions(sessions, parsed, ctx) + // Open the index on the main loop, not inside the command: the command runs + // on its own goroutine and must not mutate App state. + if a.contentIndex == nil { + if ix, err := session.OpenIndex(a.config.ClaudeDir); err == nil { + a.contentIndex = ix + } + } + ix := a.contentIndex - go func() { - for result := range results { - // Send each result as a message (will be batched by tea runtime) - // This is a simplified approach - in production you'd batch these - _ = result + return func() tea.Msg { + // Bring the index up to date first: only transcripts whose mtime or + // size moved are re-read, so the steady-state cost is a stat per + // session. A failure here is not fatal — SearchWithIndex falls back to + // the full scan when the index is nil or unusable. + if ix != nil { + if _, err := ix.Sync(ctx, sessions, nil); err != nil && ctx.Err() != nil { + return searchBatchMsg{} } - }() - - // Collect all results synchronously for simplicity - var allResults []session.SearchResult - for result := range results { - allResults = append(allResults, result) } - return searchBatchMsg{results: allResults} + results, mode, err := session.SearchWithIndex(ctx, ix, sessions, parsed, searchResultLimit) + if err != nil { + return searchBatchMsg{} + } + return searchBatchMsg{results: results, mode: mode} } } +// searchResultLimit caps how many hits are hydrated and shown. A broad query +// can match tens of thousands of blocks; past the first few hundred the list is +// no longer something a person scrolls, and building them all costs real time. +const searchResultLimit = 500 + type searchBatchMsg struct { results []session.SearchResult + mode session.SearchMode } func (a *App) handleSearchKey(msg tea.KeyMsg) (tea.Model, tea.Cmd) { @@ -230,7 +243,16 @@ func (a *App) renderSearchModal(bg string) string { case a.searchQuery != "" && len(a.searchResults) == 0: sb.WriteString(dimStyle.Render("No results found")) case len(a.searchResults) > 0: - sb.WriteString(dimStyle.Render(fmt.Sprintf("%d results", len(a.searchResults))) + "\n") + // Say what was actually searched. The index does not cover tool_result + // content, so a silent "N results" would overstate the coverage. + count := fmt.Sprintf("%d results", len(a.searchResults)) + if len(a.searchResults) >= searchResultLimit { + count = fmt.Sprintf("first %d results", searchResultLimit) + } + if a.searchMode == session.SearchModeIndexPartial { + count += dimStyle.Render(" · tool output not indexed") + } + sb.WriteString(dimStyle.Render(count) + "\n") // Reserve rows already used (title + input box(3) + count + help) so the // list fits inside the modal without overflowing. listH := max(min(len(a.searchResults), bodyMaxH-6), 3) @@ -277,8 +299,9 @@ func (a *App) renderSearchModal(bg string) string { return overlayCenter(bg, modalStyle.Render(body), screenW, screenH) } -func (a *App) updateSearchResults(results []session.SearchResult) { +func (a *App) updateSearchResults(results []session.SearchResult, mode session.SearchMode) { a.searchResults = results + a.searchMode = mode a.searchLoading = false items := make([]list.Item, len(results)) From 0b918b95e7f926d1e956f15962a2da50ff020647 Mon Sep 17 00:00:00 2001 From: keyolk Date: Sun, 30 Aug 2026 06:32:48 +0900 Subject: [PATCH 2/2] feat: highlight search matches in conversation preview and details MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Filtering told you which blocks matched but not where, so the term still had to be found by eye in the pane on the right. Three paths were missing the paint: - Block filter (`/`) in the conversation view: decided visibility only. Now the surviving blocks have their matching text highlighted. - Cross-session search (Ctrl+S): the query was dropped at the jump, so the message you landed on looked like any other. The query's text terms now follow the jump into the preview and details. - Session-list preview: the one-line summary was already highlighted via highlightSnippet, but expanding a row showed the full body unpainted — which is precisely when you are hunting for the term. Structural filter tokens (is:tool, tool:Bash) select whole blocks rather than text within them, so highlightableTerms drops them; painting them would mark the literal string "is:tool" wherever it appeared. Negated terms (!foo) mark what must be absent and are dropped for the same reason. openConversation clears the carried highlight by default and the search path re-applies it immediately after. Seven of the eight callers are not search jumps, so defaulting to off is what keeps a stale query from leaking into an unrelated session. --- internal/tui/app.go | 6 + internal/tui/blockfilter.go | 27 ++++ internal/tui/conversation.go | 13 ++ internal/tui/conversation_text.go | 26 +++- internal/tui/highlight_search_test.go | 201 ++++++++++++++++++++++++++ internal/tui/messages.go | 6 + internal/tui/search.go | 15 ++ internal/tui/splitpane.go | 10 ++ 8 files changed, 299 insertions(+), 5 deletions(-) create mode 100644 internal/tui/highlight_search_test.go diff --git a/internal/tui/app.go b/internal/tui/app.go index 3ed8fa5..f3f032a 100644 --- a/internal/tui/app.go +++ b/internal/tui/app.go @@ -576,6 +576,12 @@ type App struct { searchCancel context.CancelFunc searchMode session.SearchMode + // convHighlightTerms are the plain-text terms of the query that led here via + // a cross-session search result. They keep the match visible in the + // conversation the jump lands in, and are cleared when the conversation is + // left or another search runs. + convHighlightTerms []string + // contentIndex is the FTS index backing cross-session search. It is opened // lazily on the first search and owned by the main loop; search commands // only read it. diff --git a/internal/tui/blockfilter.go b/internal/tui/blockfilter.go index 8b1fae7..44fc63d 100644 --- a/internal/tui/blockfilter.go +++ b/internal/tui/blockfilter.go @@ -133,3 +133,30 @@ func countVisibleBlocks(vis []bool) int { } return n } + +// highlightableTerms extracts the plain-text terms of a filter expression — +// the ones that match against block *content* and are therefore worth +// highlighting in the rendered output. +// +// Structured tokens (is:tool, tool:Name) select whole blocks rather than text +// inside them, so highlighting them would paint the literal string "is:tool" +// wherever it happened to appear. Negated terms mark what must be *absent*, so +// by definition there is nothing to highlight. +func highlightableTerms(filter string) []string { + filter = strings.TrimSpace(filter) + if filter == "" { + return nil + } + var out []string + for _, term := range strings.Fields(filter) { + if strings.HasPrefix(term, "!") { + continue + } + lower := strings.ToLower(term) + if strings.HasPrefix(lower, "is:") || strings.HasPrefix(lower, "tool:") { + continue + } + out = append(out, term) + } + return out +} diff --git a/internal/tui/conversation.go b/internal/tui/conversation.go index a159a3a..459b38f 100644 --- a/internal/tui/conversation.go +++ b/internal/tui/conversation.go @@ -53,6 +53,14 @@ func (a *App) openConversation(sess session.Session) tea.Cmd { a.conv.toolUseToAgent = buildToolUseToAgentMap(entries) a.conv.inspector = conversationInspector{Scope: session.ScopeNode} a.conv.split.PreviewOnly = false + // Opening a conversation clears any carried-over search highlight by + // default; the cross-session search path re-applies its terms right after + // this call. Defaulting to off here means the seven non-search entry points + // cannot leak a stale query into an unrelated session. + a.convHighlightTerms = nil + if a.conv.split.Folds != nil { + a.conv.split.Folds.ExtraHighlight = nil + } // File-backed tasks provide durable metadata; transcript events provide the // latest state and IDs for current TaskCreate/TaskUpdate calls. @@ -1425,6 +1433,11 @@ func (a *App) setConvPreviewText(content string) { func (a *App) setConvPreviewTextKey(content, cacheKey string) { sp := &a.conv.split + // Every preview and details render funnels through here, so this is the one + // place that keeps a cross-session search match visible after the jump. + if len(a.convHighlightTerms) > 0 { + content = highlightSearchTerms(content, a.convHighlightTerms, -1) + } oldOffset := sp.Preview.YOffset sameKey := sp.CacheKey == cacheKey sp.CacheKey = cacheKey diff --git a/internal/tui/conversation_text.go b/internal/tui/conversation_text.go index 05fa0f9..3e344f6 100644 --- a/internal/tui/conversation_text.go +++ b/internal/tui/conversation_text.go @@ -33,14 +33,30 @@ func highlightSearchMatches(content, term string, currentLine int) string { if term == "" { return content } - lowerTerm := strings.ToLower(term) + return highlightSearchTerms(content, []string{term}, currentLine) +} + +// highlightSearchTerms is highlightSearchMatches for several terms at once. +// Filter expressions are AND-ed sets of words, so every one of them is a reason +// the block is on screen and all of them get painted. +// +// Terms are applied one after another over the already-highlighted line. That +// is safe because highlightLine walks visible characters and copies ANSI +// sequences through untouched, so an earlier term's escapes neither shift the +// match positions of a later term nor get matched themselves. +func highlightSearchTerms(content string, terms []string, currentLine int) string { + if len(terms) == 0 { + return content + } lines := strings.Split(content, "\n") for i, line := range lines { - plain := stripANSI(line) - if !strings.Contains(strings.ToLower(plain), lowerTerm) { - continue + plain := strings.ToLower(stripANSI(line)) + for _, term := range terms { + if term == "" || !strings.Contains(plain, strings.ToLower(term)) { + continue + } + lines[i] = highlightLine(lines[i], term, i == currentLine) } - lines[i] = highlightLine(line, term, i == currentLine) } return strings.Join(lines, "\n") } diff --git a/internal/tui/highlight_search_test.go b/internal/tui/highlight_search_test.go new file mode 100644 index 0000000..fbb308c --- /dev/null +++ b/internal/tui/highlight_search_test.go @@ -0,0 +1,201 @@ +package tui + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/sendbird/ccx/internal/session" +) + +// hlCount counts highlighted spans by looking for the ANSI start codes +// highlightLine emits. +func hlCount(s string) int { + return strings.Count(s, "\x1b[43;30m") + strings.Count(s, "\x1b[46;30m") +} + +func TestHighlightSearchTermsPaintsEveryTerm(t *testing.T) { + content := "the worktree was pruned\nunrelated line\nprune again" + got := highlightSearchTerms(content, []string{"worktree", "prune"}, -1) + + // worktree(1) + pruned(1) on line 1, prune(1) on line 3. + if n := hlCount(got); n != 3 { + t.Errorf("highlighted spans = %d, want 3\n%q", n, got) + } + if strings.Contains(strings.Split(got, "\n")[1], "\x1b[43;30m") { + t.Error("non-matching line was highlighted") + } +} + +// Applying a second term must not corrupt the first term's escapes or match +// them as if they were text. +func TestHighlightSearchTermsOverlappingApplication(t *testing.T) { + got := highlightSearchTerms("alpha beta gamma", []string{"alpha", "beta", "gamma"}, -1) + if n := hlCount(got); n != 3 { + t.Errorf("spans = %d, want 3\n%q", n, got) + } + if plain := stripANSI(got); plain != "alpha beta gamma" { + t.Errorf("visible text changed: %q", plain) + } +} + +func TestHighlightSearchTermsEmptyInputs(t *testing.T) { + if got := highlightSearchTerms("text", nil, -1); got != "text" { + t.Errorf("nil terms changed content: %q", got) + } + if got := highlightSearchTerms("text", []string{""}, -1); got != "text" { + t.Errorf("empty term changed content: %q", got) + } +} + +// Structured filter tokens select whole blocks; highlighting them would paint +// the literal string "is:tool" wherever it appears in the text. +func TestHighlightableTermsSkipsStructuralTokens(t *testing.T) { + cases := []struct { + filter string + want []string + }{ + {"worktree", []string{"worktree"}}, + {"is:tool worktree", []string{"worktree"}}, + {"tool:Bash prune", []string{"prune"}}, + {"is:error", nil}, + {"tool:mcp*", nil}, + {"!skipme worktree", []string{"worktree"}}, + {"alpha beta", []string{"alpha", "beta"}}, + {"", nil}, + {" ", nil}, + } + for _, c := range cases { + got := highlightableTerms(c.filter) + if len(got) != len(c.want) { + t.Errorf("highlightableTerms(%q) = %v, want %v", c.filter, got, c.want) + continue + } + for i := range got { + if got[i] != c.want[i] { + t.Errorf("highlightableTerms(%q) = %v, want %v", c.filter, got, c.want) + break + } + } + } +} + +// A negated term marks what must be absent — there is nothing to paint, and +// painting it would be actively misleading. +func TestNegatedTermIsNotHighlighted(t *testing.T) { + terms := highlightableTerms("!worktree prune") + for _, term := range terms { + if strings.Contains(term, "worktree") { + t.Errorf("negated term %q would be highlighted", term) + } + } +} + +// Opening a conversation from anywhere other than a search result must not +// inherit a previous jump's highlight. Seven of the eight openConversation +// callers are not search jumps, so the clear has to live in the callee. +func TestOpenConversationClearsSearchHighlight(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "s.jsonl") + line := `{"type":"user","uuid":"u1","message":{"role":"user","content":[{"type":"text","text":"hello there"}]}}` + if err := os.WriteFile(path, []byte(line+"\n"), 0o644); err != nil { + t.Fatal(err) + } + + a := &App{} + a.conv.split.Folds = &FoldState{} + a.convHighlightTerms = []string{"stale"} + a.conv.split.Folds.ExtraHighlight = []string{"stale"} + + a.openConversation(session.Session{FilePath: path}) + + if len(a.convHighlightTerms) != 0 { + t.Errorf("convHighlightTerms = %v, want cleared", a.convHighlightTerms) + } + if len(a.conv.split.Folds.ExtraHighlight) != 0 { + t.Errorf("Folds.ExtraHighlight = %v, want cleared", a.conv.split.Folds.ExtraHighlight) + } +} + +func TestSearchModeStringsAreDistinct(t *testing.T) { + seen := map[string]bool{} + for _, m := range []session.SearchMode{ + session.SearchModeScan, + session.SearchModeIndex, + session.SearchModeIndexPartial, + } { + s := m.String() + if s == "" { + t.Error("empty mode string") + } + if seen[s] { + t.Errorf("duplicate mode string %q", s) + } + seen[s] = true + } +} + +// End-to-end through the real App: picking a cross-session search result must +// leave the term highlighted in the preview the jump lands on. Unit tests cover +// the paint function; this covers the wiring between search, openConversation, +// and the preview renderer. +func TestSearchJumpHighlightsPreviewEndToEnd(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "s.jsonl") + lines := []string{ + `{"type":"user","uuid":"u1","message":{"role":"user","content":[{"type":"text","text":"please prune the worktree now"}]}}`, + `{"type":"assistant","uuid":"a1","message":{"role":"assistant","content":[{"type":"text","text":"done, worktree removed"}]}}`, + } + if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")+"\n"), 0o644); err != nil { + t.Fatal(err) + } + fi, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + sess := session.Session{ + ID: "sid1", ShortID: "sid1", + ProjectPath: dir, ProjectName: "proj", + FilePath: path, ModTime: fi.ModTime(), MsgCount: 2, + } + + a := newTestApp([]session.Session{sess}) + a.searchQuery = "worktree" + + entries, err := session.LoadMessages(path) + if err != nil { + t.Fatal(err) + } + if len(entries) == 0 { + t.Fatal("no entries loaded") + } + result := session.SearchResult{ + Session: &sess, + Entry: &entries[0], + Block: &entries[0].Content[0], + } + + a.openSearchResult(result) + + if len(a.convHighlightTerms) == 0 { + t.Fatal("jump did not carry the query terms into the conversation") + } + if a.convHighlightTerms[0] != "worktree" { + t.Errorf("carried terms = %v, want [worktree]", a.convHighlightTerms) + } + + // The preview content itself must carry the highlight escapes. + a.conv.split.Show = true + a.setConvPreviewText("the worktree line in preview") + got := a.conv.split.Preview.View() + if hlCount(got) == 0 { + t.Errorf("preview has no highlight for the searched term:\n%q", stripANSI(got)) + } + + // Opening an unrelated session afterwards must not keep painting. + a.openConversation(sess) + if len(a.convHighlightTerms) != 0 { + t.Errorf("highlight leaked into a later conversation: %v", a.convHighlightTerms) + } +} diff --git a/internal/tui/messages.go b/internal/tui/messages.go index ce7d135..253d16e 100644 --- a/internal/tui/messages.go +++ b/internal/tui/messages.go @@ -253,6 +253,12 @@ func renderConversationPreview(msgs []mergedMsg, width, cursor int, expanded map text := entryFullText(e) if text != "" { wrapped := wrapText(text, textW) + // The one-line preview above is highlighted via highlightSnippet, + // but that only covers the summary. Expanding is exactly when the + // user is looking for the term in the body, so paint it here too. + if filterTerm != "" { + wrapped = highlightSearchTerms(wrapped, highlightableTerms(filterTerm), -1) + } for _, line := range strings.Split(wrapped, "\n") { row.WriteString(" " + line + "\n") } diff --git a/internal/tui/search.go b/internal/tui/search.go index 294853a..88aa8f3 100644 --- a/internal/tui/search.go +++ b/internal/tui/search.go @@ -44,6 +44,9 @@ func (a *App) enterSearchMode() { a.searchQuery = "" a.searchResults = nil a.searchLoading = false + // Drop any highlight carried over from a previous jump; the next result + // picked from this session sets its own. + a.convHighlightTerms = nil ti := textinput.New() ti.Placeholder = "Search all sessions..." @@ -174,12 +177,24 @@ func (a *App) handleSearchKey(msg tea.KeyMsg) (tea.Model, tea.Cmd) { } func (a *App) openSearchResult(result session.SearchResult) { + // The query's text terms, kept so the match the user picked stays visible in + // the conversation they land on. + parsed := session.ParseSearchQuery(a.searchQuery) + terms := append(append([]string(nil), parsed.Terms...), parsed.Phrases...) + for i, sess := range a.sessions { if sess.ID == result.Session.ID { a.sessionList.Select(i) a.currentSess = sess a.openConversation(sess) + // Set after openConversation, which clears the highlight so that the + // other entry points cannot inherit a stale query. + a.convHighlightTerms = terms + if a.conv.split.Folds != nil { + a.conv.split.Folds.ExtraHighlight = terms + } + // Jump to the message containing the search result. Selection indices // are always in the list's visible coordinate space. targetUUID := result.Entry.UUID diff --git a/internal/tui/splitpane.go b/internal/tui/splitpane.go index 790d29b..594b6c4 100644 --- a/internal/tui/splitpane.go +++ b/internal/tui/splitpane.go @@ -58,6 +58,10 @@ type FoldState struct { BlockStarts []int BlockVisible []bool // nil = all visible; non-nil = per-block visibility BlockFilter string // current filter expression (empty = no filter) + // ExtraHighlight are terms to paint that did not come from BlockFilter — + // currently the cross-session search query the jump into this conversation + // came from. Kept separate so clearing the block filter does not drop them. + ExtraHighlight []string HideHooks bool // true = suppress hook badges/details in render Selected foldSet // block indices selected for copy BlockSourceIdx []int // parallel to Entry.Content; -1 = unknown source @@ -562,6 +566,12 @@ func (sp *SplitPane) RefreshFoldPreview(totalW, splitRatio int) { } content := rp.content + // Paint the filter's text terms. The filter already decided which blocks are + // on screen; without this the user still has to hunt for the word inside + // them. Block-selecting tokens (is:/tool:) are skipped by highlightableTerms. + if terms := append(highlightableTerms(sp.Folds.BlockFilter), sp.Folds.ExtraHighlight...); len(terms) > 0 { + content = highlightSearchTerms(content, terms, -1) + } padLines := 0 if sp.BottomAlign && rp.lineCount < sp.Preview.Height { padLines = sp.Preview.Height - rp.lineCount