diff --git a/README.md b/README.md index a3f7b89..10a53a0 100644 --- a/README.md +++ b/README.md @@ -189,12 +189,28 @@ Search inside conversation content across all sessions (`Ctrl+S` or `:search`). - `tool:ToolName` — Only search specific tool calls **Features:** -- Searches text, tool inputs/outputs, thinking blocks -- Results stream in real-time as they're found +- Searches text, tool inputs, thinking blocks, and system tags +- Results are ordered newest session first - Matched terms are highlighted in snippets - Press `Enter` to jump directly to the matching message - Press `/` to edit the query +**Index:** + +Search is backed by a SQLite FTS5 index at `~/.claude/.ccx-index.db` (~400 MB for +a 2.5 GB transcript corpus), which makes a typical query take tens of +milliseconds instead of seconds. + +- The index refreshes when you search: only transcripts whose size or mtime + changed are re-read, so results are always current. The first build takes + about a minute; after that a refresh is milliseconds. +- **Tool output (`tool_result`) is not indexed.** It is half the corpus and + mostly file dumps, so including it would roughly double the index for little + search value. The modal says `tool output not indexed` when this applies. +- Queries with a term shorter than 3 characters fall back to a full scan + automatically — a trigram index cannot match them. +- The index is a cache: deleting it is safe, and a corrupt one is rebuilt. + **Example queries:** ``` database migration # Find both terms diff --git a/go.mod b/go.mod index f10d999..389e264 100644 --- a/go.mod +++ b/go.mod @@ -22,16 +22,24 @@ require ( github.com/clipperhouse/displaywidth v0.9.0 // indirect github.com/clipperhouse/stringish v0.1.1 // indirect github.com/clipperhouse/uax29/v2 v2.5.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect + github.com/google/uuid v1.6.0 // indirect github.com/lucasb-eyer/go-colorful v1.3.0 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.16.0 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/sahilm/fuzzy v0.1.1 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect - golang.org/x/sys v0.42.0 // indirect + golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.3.8 // indirect + modernc.org/libc v1.74.4 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect + modernc.org/sqlite v1.57.0 // indirect ) diff --git a/go.sum b/go.sum index 62b62ee..bad9f72 100644 --- a/go.sum +++ b/go.sum @@ -26,14 +26,20 @@ github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfa github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= github.com/clipperhouse/uax29/v2 v2.5.0 h1:x7T0T4eTHDONxFJsL94uKNKPHrclyFI0lm7+w94cO8U= github.com/clipperhouse/uax29/v2 v2.5.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag= github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= @@ -44,6 +50,10 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/sahilm/fuzzy v0.1.1 h1:ceu5RHF8DGgoi+/dR5PsECjCDH1BE3Fnmpo7aVXOdRA= @@ -56,6 +66,8 @@ golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= @@ -64,3 +76,11 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k= +modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/sqlite v1.57.0 h1:qNQP6xnx5M0ISNtlnxoOX0+cD5bJ0/gr9aMmndFczzg= +modernc.org/sqlite v1.57.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ= diff --git a/internal/session/index.go b/internal/session/index.go new file mode 100644 index 0000000..d066a96 --- /dev/null +++ b/internal/session/index.go @@ -0,0 +1,626 @@ +package session + +import ( + "bufio" + "context" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" + + _ "modernc.org/sqlite" +) + +// Cross-session content search is backed by an FTS5 index so that a query does +// not have to re-read every transcript (~2.5 GB) on each keystroke. +// +// Design decisions worth knowing before changing anything here: +// +// - tokenize='trigram' — the TUI's search has always been substring-based +// (strings.Contains), and transcripts are heavily Korean. The default +// unicode61 tokenizer splits on whitespace/punctuation, so it misses both +// mid-word matches ("orktre") and Korean ("워크트리" inside "워크트리를"). +// Trigram reproduces the existing semantics; unicode61 measurably does not +// (1930 vs 10471 hits for "worktree" on a 10% corpus sample). +// +// - content='' (contentless) — the block text also lives in the transcript on +// disk, so storing a second copy inside FTS is pure overhead. Contentless +// cut the index from 410 MB to 284 MB on the sample. Snippets are rebuilt by +// re-reading the one matching line via its byte offset. +// +// - detail=full — required, not chosen. Trigram matching is internally a +// phrase query, and FTS5 rejects phrase queries unless detail=full. That +// rules out the smaller detail=none layout. +// +// - contentless_delete=1 — lets us delete a file's rows without knowing their +// original text, which is what makes incremental reindexing possible. +// +// - tool_result blocks are not indexed. They are half of all transcript text +// but are mostly file dumps and command output; excluding them takes the +// full-corpus index from ~850 MB to ~450 MB. + +const ( + // indexSchemaVersion is bumped whenever the schema or the set of indexed + // blocks changes in a way that makes an existing index wrong rather than + // merely stale. On mismatch the index is dropped and rebuilt. + indexSchemaVersion = 1 + + // minTrigramTerm is the shortest term a trigram index can match. Queries + // containing anything shorter cannot be answered from the index at all. + minTrigramTerm = 3 +) + +// Index is an FTS5 index over transcript content blocks. +type Index struct { + db *sql.DB + path string +} + +func indexFilePath(claudeDir string) string { + return filepath.Join(claudeDir, ".ccx-index.db") +} + +// OpenIndex opens (creating if needed) the content index for claudeDir. +// A schema-version mismatch or an unreadable file rebuilds from scratch rather +// than failing: the index is a cache, and a corrupt cache must not break search. +func OpenIndex(claudeDir string) (*Index, error) { + path := indexFilePath(claudeDir) + + idx, err := openIndexAt(path) + if err == nil { + return idx, nil + } + + // Unusable index — discard and retry once from empty. + os.Remove(path) + os.Remove(path + "-wal") + os.Remove(path + "-shm") + return openIndexAt(path) +} + +func openIndexAt(path string) (*Index, error) { + // synchronous=normal: losing the tail of the index after a crash costs a + // reindex of a few sessions, which is cheaper than fsync per commit. + dsn := path + "?_pragma=journal_mode(wal)&_pragma=synchronous(normal)&_pragma=busy_timeout(5000)" + db, err := sql.Open("sqlite", dsn) + if err != nil { + return nil, err + } + + idx := &Index{db: db, path: path} + if err := idx.migrate(); err != nil { + db.Close() + return nil, err + } + return idx, nil +} + +func (ix *Index) migrate() error { + var version int + row := ix.db.QueryRow(`select value from meta where key='schema_version'`) + if err := row.Scan(&version); err != nil { + if !isMissingTable(err) && !errors.Is(err, sql.ErrNoRows) { + return err + } + version = 0 + } + + if version == indexSchemaVersion { + return nil + } + if version != 0 { + // Stale layout: drop everything and fall through to a fresh create. + for _, stmt := range []string{ + `drop table if exists blocks`, + `drop table if exists locs`, + `drop table if exists files`, + `drop table if exists meta`, + } { + if _, err := ix.db.Exec(stmt); err != nil { + return err + } + } + } + + schema := []string{ + `create table if not exists meta(key text primary key, value text)`, + + // One row per indexed transcript file. mod_unix/size detect changes. + `create table if not exists files( + id integer primary key, + path text unique not null, + mod_unix integer not null, + size integer not null + )`, + + // Locations of indexed blocks. Shares rowids with the FTS table, which + // is what lets us delete a file's blocks and filter by role/tool. + `create table if not exists locs( + rowid integer primary key, + file_id integer not null, + line_off integer not null, + block_idx integer not null, + role text not null, + tool text not null + )`, + `create index if not exists locs_file on locs(file_id)`, + + `create virtual table if not exists blocks using fts5( + body, + tokenize='trigram', + content='', + contentless_delete=1, + detail=full + )`, + } + for _, stmt := range schema { + if _, err := ix.db.Exec(stmt); err != nil { + return err + } + } + + _, err := ix.db.Exec( + `insert into meta(key,value) values('schema_version',?) + on conflict(key) do update set value=excluded.value`, + fmt.Sprint(indexSchemaVersion)) + return err +} + +func isMissingTable(err error) bool { + return err != nil && strings.Contains(err.Error(), "no such table") +} + +func (ix *Index) Close() error { + if ix == nil || ix.db == nil { + return nil + } + return ix.db.Close() +} + +// Path returns the on-disk location of the index. +func (ix *Index) Path() string { return ix.path } + +// indexedFile is the stored fingerprint of an already-indexed transcript. +type indexedFile struct { + id int64 + modUnix int64 + size int64 +} + +func (ix *Index) knownFiles() (map[string]indexedFile, error) { + rows, err := ix.db.Query(`select id, path, mod_unix, size from files`) + if err != nil { + return nil, err + } + defer rows.Close() + + known := make(map[string]indexedFile) + for rows.Next() { + var f indexedFile + var p string + if err := rows.Scan(&f.id, &p, &f.modUnix, &f.size); err != nil { + return nil, err + } + known[p] = f + } + return known, rows.Err() +} + +// SyncStats reports what a Sync did, for progress reporting and tests. +type SyncStats struct { + Scanned int // transcripts considered + Indexed int // transcripts (re)indexed + Removed int // transcripts dropped from the index + Blocks int // content blocks written + Elapsed time.Duration // +} + +// Sync brings the index in line with the given sessions: transcripts whose +// mtime or size changed are reindexed, transcripts that disappeared are +// dropped, and unchanged transcripts are skipped. It is safe to call on every +// search — the common case is a stat of each file and no writes. +func (ix *Index) Sync(ctx context.Context, sessions []*Session, progress func(done, total int)) (SyncStats, error) { + start := time.Now() + var stats SyncStats + + known, err := ix.knownFiles() + if err != nil { + return stats, err + } + + type work struct { + path string + modUnix int64 + size int64 + prevID int64 + hasPrev bool + } + + var todo []work + seen := make(map[string]bool, len(sessions)) + + for _, s := range sessions { + if s == nil || s.FilePath == "" || seen[s.FilePath] { + continue + } + seen[s.FilePath] = true + stats.Scanned++ + + fi, err := os.Stat(s.FilePath) + if err != nil { + continue + } + mod, size := fi.ModTime().Unix(), fi.Size() + + prev, ok := known[s.FilePath] + if ok && prev.modUnix == mod && prev.size == size { + continue // unchanged + } + todo = append(todo, work{ + path: s.FilePath, modUnix: mod, size: size, + prevID: prev.id, hasPrev: ok, + }) + } + + // Transcripts that vanished (project deleted, session pruned). + for path, f := range known { + if !seen[path] { + if err := ix.dropFile(f.id); err != nil { + return stats, err + } + stats.Removed++ + } + } + + for i, w := range todo { + select { + case <-ctx.Done(): + stats.Elapsed = time.Since(start) + return stats, ctx.Err() + default: + } + + if progress != nil { + progress(i, len(todo)) + } + + n, err := ix.indexFile(w.path, w.modUnix, w.size, w.prevID, w.hasPrev) + if err != nil { + // A single unreadable transcript must not abort the whole sync. + continue + } + stats.Indexed++ + stats.Blocks += n + } + if progress != nil && len(todo) > 0 { + progress(len(todo), len(todo)) + } + + stats.Elapsed = time.Since(start) + return stats, nil +} + +func (ix *Index) dropFile(fileID int64) error { + tx, err := ix.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + if _, err := tx.Exec( + `delete from blocks where rowid in (select rowid from locs where file_id=?)`, fileID); err != nil { + return err + } + if _, err := tx.Exec(`delete from locs where file_id=?`, fileID); err != nil { + return err + } + if _, err := tx.Exec(`delete from files where id=?`, fileID); err != nil { + return err + } + return tx.Commit() +} + +// indexFile reindexes one transcript, replacing any previously indexed rows. +func (ix *Index) indexFile(path string, modUnix, size, prevID int64, hasPrev bool) (int, error) { + f, err := os.Open(path) + if err != nil { + return 0, err + } + defer f.Close() + + tx, err := ix.db.Begin() + if err != nil { + return 0, err + } + defer tx.Rollback() + + fileID := prevID + if hasPrev { + if _, err := tx.Exec( + `delete from blocks where rowid in (select rowid from locs where file_id=?)`, fileID); err != nil { + return 0, err + } + if _, err := tx.Exec(`delete from locs where file_id=?`, fileID); err != nil { + return 0, err + } + if _, err := tx.Exec(`update files set mod_unix=?, size=? where id=?`, modUnix, size, fileID); err != nil { + return 0, err + } + } else { + res, err := tx.Exec(`insert into files(path, mod_unix, size) values (?,?,?)`, path, modUnix, size) + if err != nil { + return 0, err + } + if fileID, err = res.LastInsertId(); err != nil { + return 0, err + } + } + + insBlock, err := tx.Prepare(`insert into blocks(rowid, body) values (?,?)`) + if err != nil { + return 0, err + } + insLoc, err := tx.Prepare( + `insert into locs(rowid, file_id, line_off, block_idx, role, tool) values (?,?,?,?,?,?)`) + if err != nil { + return 0, err + } + + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 256*1024), 10*1024*1024) + + var lineOff int64 + var count int + + for sc.Scan() { + line := sc.Bytes() + off := lineOff + lineOff += int64(len(line)) + 1 // +1 for the newline the scanner strips + + if len(line) == 0 { + continue + } + entry, err := ParseEntry(string(line)) + if err != nil || entry.IsMeta { + continue + } + + for i := range entry.Content { + block := &entry.Content[i] + if !indexableBlock(block) { + continue + } + body := blockSearchText(block) + if body == "" { + continue + } + + // rowid is assigned by SQLite; both tables must agree on it, so we + // take the one FTS picked and reuse it for the location row. + res, err := insBlock.Exec(nil, body) + if err != nil { + return 0, err + } + rowid, err := res.LastInsertId() + if err != nil { + return 0, err + } + if _, err := insLoc.Exec(rowid, fileID, off, i, entry.Role, block.ToolName); err != nil { + return 0, err + } + count++ + } + } + if err := sc.Err(); err != nil { + return 0, err + } + + if err := tx.Commit(); err != nil { + return 0, err + } + return count, nil +} + +// indexableBlock reports whether a block's text goes into the index. +// +// tool_result is the one deliberate omission: it is roughly half of all +// transcript text but is dominated by file dumps and command output, and +// indexing it nearly doubles the index for little search value. Queries that +// need it fall back to the full scan (see IndexCoverage). +// +// Everything else the scan can match must be listed here, or the index silently +// returns fewer results than the scan for the same query. system_tag in +// particular carries command names and system reminders that users do search +// for, at ~0.6% of the corpus. +func indexableBlock(b *ContentBlock) bool { + // Empty bodies are skipped by the caller, so image blocks and other + // text-free types drop out on their own. + return b.Type != "tool_result" +} + +// Stats describes the current index contents. +type Stats struct { + Files int + Blocks int + Bytes int64 +} + +func (ix *Index) Stats() (Stats, error) { + var s Stats + if err := ix.db.QueryRow(`select count(*) from files`).Scan(&s.Files); err != nil { + return s, err + } + if err := ix.db.QueryRow(`select count(*) from locs`).Scan(&s.Blocks); err != nil { + return s, err + } + if fi, err := os.Stat(ix.path); err == nil { + s.Bytes = fi.Size() + } + return s, nil +} + +// Optimize compacts the FTS index. Worth running after a large rebuild; not +// needed on the incremental path. +func (ix *Index) Optimize() error { + _, err := ix.db.Exec(`insert into blocks(blocks) values('optimize')`) + return err +} + +// --- querying --------------------------------------------------------------- + +// IndexCoverage says whether the index can answer a query on its own. +type IndexCoverage int + +const ( + // CoverageFull means index results are equivalent to a full scan. + CoverageFull IndexCoverage = iota + // CoverageNone means the query must use the full scan; the index cannot + // answer it (a term shorter than a trigram, or an empty query). + CoverageNone + // CoveragePartial means the index answers the query but over a subset of + // blocks: tool_result content is not indexed, so a full scan would find + // strictly more. + CoveragePartial +) + +// Coverage reports how well the index can serve q. +func (ix *Index) Coverage(q SearchQuery) IndexCoverage { + if q.IsEmpty() { + return CoverageNone + } + // Trigram cannot match anything shorter than three characters. + for _, t := range q.Terms { + if len([]rune(t)) < minTrigramTerm { + return CoverageNone + } + } + for _, p := range q.Phrases { + if len([]rune(p)) < minTrigramTerm { + return CoverageNone + } + } + // Exclusions are applied after retrieval, so a short one is harmless. + + // A tool: filter restricts to tool_use blocks, which are fully indexed. + if q.ToolName != "" { + return CoverageFull + } + return CoveragePartial +} + +// fts5Quote renders s as an FTS5 string literal. Trigram matching treats the +// content literally, so every term becomes a quoted phrase; the only character +// needing care is the double quote, which doubles. +func fts5Quote(s string) string { + return `"` + strings.ReplaceAll(s, `"`, `""`) + `"` +} + +// buildMatchExpr renders the positive part of a query as an FTS5 MATCH +// expression. Exclusions are intentionally left out: they are substring +// exclusions over the block text, which is cheaper and more faithful to apply +// after the rows come back. +func buildMatchExpr(q SearchQuery) string { + var parts []string + for _, t := range q.Terms { + parts = append(parts, fts5Quote(t)) + } + for _, p := range q.Phrases { + parts = append(parts, fts5Quote(p)) + } + if q.ToolName != "" && !strings.HasSuffix(q.ToolName, "*") { + // A concrete tool name is also indexed in the block body (tool_use + // bodies are "name input"), so it usefully narrows the FTS scan. A + // prefix filter is left to the post-filter. + parts = append(parts, fts5Quote(q.ToolName)) + } + return strings.Join(parts, " AND ") +} + +// indexHit is one row from the index: where the block lives, nothing more. +type indexHit struct { + path string + lineOff int64 + blockIdx int +} + +// queryIndex returns matching block locations, newest session first. +func (ix *Index) queryIndex(ctx context.Context, q SearchQuery, allowed map[string]*Session, limit int) ([]indexHit, error) { + expr := buildMatchExpr(q) + if expr == "" { + return nil, nil + } + + var ( + where []string + args []any + ) + args = append(args, expr) + + if q.Role != "" { + where = append(where, `locs.role = ?`) + args = append(args, q.Role) + } + if q.ToolName != "" { + if strings.HasSuffix(q.ToolName, "*") { + where = append(where, `lower(locs.tool) like ?`) + args = append(args, strings.ToLower(strings.TrimSuffix(q.ToolName, "*"))+"%") + } else { + where = append(where, `lower(locs.tool) = ?`) + args = append(args, strings.ToLower(q.ToolName)) + } + } + + sqlText := `select files.path, locs.line_off, locs.block_idx + from blocks + join locs on locs.rowid = blocks.rowid + join files on files.id = locs.file_id + where blocks match ?` + if len(where) > 0 { + sqlText += " and " + strings.Join(where, " and ") + } + // Ordering is applied in Go against session mtime; SQL only bounds the work. + if limit > 0 { + sqlText += fmt.Sprintf(" limit %d", limit) + } + + rows, err := ix.db.QueryContext(ctx, sqlText, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + var hits []indexHit + for rows.Next() { + var h indexHit + if err := rows.Scan(&h.path, &h.lineOff, &h.blockIdx); err != nil { + return nil, err + } + if allowed != nil { + if _, ok := allowed[h.path]; !ok { + continue + } + } + hits = append(hits, h) + } + if err := rows.Err(); err != nil { + return nil, err + } + + // Newest session first, matching the session browser's ordering; within a + // session, transcript order. + sort.SliceStable(hits, func(i, j int) bool { + si, sj := allowed[hits[i].path], allowed[hits[j].path] + if si != nil && sj != nil && !si.ModTime.Equal(sj.ModTime) { + return si.ModTime.After(sj.ModTime) + } + if hits[i].path != hits[j].path { + return hits[i].path < hits[j].path + } + return hits[i].lineOff < hits[j].lineOff + }) + return hits, nil +} diff --git a/internal/session/index_test.go b/internal/session/index_test.go new file mode 100644 index 0000000..49b0a50 --- /dev/null +++ b/internal/session/index_test.go @@ -0,0 +1,370 @@ +package session + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "testing" + "time" +) + +// writeIndexTranscript writes a .jsonl transcript and returns a Session for it. +func writeIndexTranscript(t *testing.T, dir, name string, lines []string) *Session { + t.Helper() + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")+"\n"), 0o644); err != nil { + t.Fatal(err) + } + fi, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + return &Session{FilePath: path, ModTime: fi.ModTime(), ProjectName: name} +} + +func userLine(text string) string { + return fmt.Sprintf(`{"type":"user","message":{"role":"user","content":[{"type":"text","text":%q}]}}`, text) +} + +func assistantLine(text string) string { + return fmt.Sprintf(`{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":%q}]}}`, text) +} + +func toolLine(name, input string) string { + return fmt.Sprintf(`{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","name":%q,"input":{"command":%q}}]}}`, name, input) +} + +func toolResultLine(text string) string { + return fmt.Sprintf(`{"type":"user","message":{"role":"user","content":[{"type":"tool_result","content":%q}]}}`, text) +} + +func openTestIndex(t *testing.T) (*Index, string) { + t.Helper() + dir := t.TempDir() + ix, err := OpenIndex(dir) + if err != nil { + t.Fatalf("OpenIndex: %v", err) + } + t.Cleanup(func() { ix.Close() }) + return ix, dir +} + +func syncAll(t *testing.T, ix *Index, sessions []*Session) SyncStats { + t.Helper() + stats, err := ix.Sync(context.Background(), sessions, nil) + if err != nil { + t.Fatalf("Sync: %v", err) + } + return stats +} + +func searchIdx(t *testing.T, ix *Index, sessions []*Session, query string) ([]SearchResult, SearchMode) { + t.Helper() + res, mode, err := SearchWithIndex(context.Background(), ix, sessions, ParseSearchQuery(query), 0) + if err != nil { + t.Fatalf("SearchWithIndex(%q): %v", query, err) + } + return res, mode +} + +func TestIndexFindsBasicMatches(t *testing.T) { + ix, dir := openTestIndex(t) + s := writeIndexTranscript(t, dir, "a.jsonl", []string{ + userLine("please prune the worktree"), + assistantLine("removed the stale worktree entry"), + userLine("unrelated content"), + }) + syncAll(t, ix, []*Session{s}) + + res, mode := searchIdx(t, ix, []*Session{s}, "worktree") + if len(res) != 2 { + t.Fatalf("hits = %d, want 2", len(res)) + } + if mode != SearchModeIndexPartial { + t.Errorf("mode = %v, want index", mode) + } +} + +// The index must not change what a search means. Any divergence from the +// full scan is a bug, so compare the two on the same corpus. +func TestIndexResultsMatchFullScan(t *testing.T) { + ix, dir := openTestIndex(t) + sessions := []*Session{ + writeIndexTranscript(t, dir, "a.jsonl", []string{ + userLine("deploy the worktree now"), + assistantLine("WORKTREE uppercase mention"), + toolLine("Bash", "git worktree prune"), + userLine("worktree and prune together"), + }), + writeIndexTranscript(t, dir, "b.jsonl", []string{ + userLine("워크트리를 정리했다"), + assistantLine("only prune here"), + toolLine("Read", "worktree config"), + }), + } + syncAll(t, ix, sessions) + + queries := []string{ + "worktree", + "WORKTREE", + "worktree prune", + "worktree -prune", + `"worktree and prune"`, + "워크트리", + "orktre", + "user:worktree", + "assistant:worktree", + "tool:Bash", + "tool:Bash worktree", + } + + key := func(r SearchResult) string { + return fmt.Sprintf("%s|%s|%s", r.Session.FilePath, r.Block.Type, blockSearchText(r.Block)) + } + keys := func(rs []SearchResult) []string { + out := make([]string, 0, len(rs)) + for _, r := range rs { + out = append(out, key(r)) + } + sort.Strings(out) + return out + } + + for _, q := range queries { + t.Run(q, func(t *testing.T) { + parsed := ParseSearchQuery(q) + scan := collectScan(context.Background(), sessions, parsed, 0) + + // tool_result is not indexed, so only compare on corpora where the + // scan's extra reach cannot matter — this corpus has none. + idx, _, err := SearchWithIndex(context.Background(), ix, sessions, parsed, 0) + if err != nil { + t.Fatal(err) + } + + got, want := keys(idx), keys(scan) + if len(got) != len(want) { + t.Fatalf("index=%d scan=%d\n index=%v\n scan=%v", len(got), len(want), got, want) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("result %d:\n index=%s\n scan=%s", i, got[i], want[i]) + } + } + }) + } +} + +func TestIndexSkipsToolResultAndFallsBack(t *testing.T) { + ix, dir := openTestIndex(t) + s := writeIndexTranscript(t, dir, "a.jsonl", []string{ + toolResultLine("secretmarker lives only in tool output"), + }) + syncAll(t, ix, []*Session{s}) + + // Indexed search cannot see it... + res, mode := searchIdx(t, ix, []*Session{s}, "secretmarker") + if len(res) != 0 { + t.Errorf("index returned %d hits for tool_result-only content, want 0", len(res)) + } + if mode != SearchModeIndexPartial { + t.Errorf("mode = %v, want partial (so the UI can say so)", mode) + } + + // ...but the scan does, which is why partial coverage must be surfaced. + scan := collectScan(context.Background(), []*Session{s}, ParseSearchQuery("secretmarker"), 0) + if len(scan) != 1 { + t.Errorf("scan hits = %d, want 1", len(scan)) + } +} + +// Terms shorter than a trigram cannot be answered by the index at all; the +// search must silently fall back rather than return nothing. +func TestShortTermFallsBackToScan(t *testing.T) { + ix, dir := openTestIndex(t) + s := writeIndexTranscript(t, dir, "a.jsonl", []string{ + userLine("go is short"), + }) + syncAll(t, ix, []*Session{s}) + + if cov := ix.Coverage(ParseSearchQuery("go")); cov != CoverageNone { + t.Errorf("Coverage(2-char) = %v, want CoverageNone", cov) + } + res, mode := searchIdx(t, ix, []*Session{s}, "go") + if mode != SearchModeScan { + t.Errorf("mode = %v, want scan", mode) + } + if len(res) != 1 { + t.Errorf("hits = %d, want 1 (fallback must still find it)", len(res)) + } +} + +func TestIncrementalSyncReindexesOnlyChangedFiles(t *testing.T) { + ix, dir := openTestIndex(t) + a := writeIndexTranscript(t, dir, "a.jsonl", []string{userLine("alpha content")}) + b := writeIndexTranscript(t, dir, "b.jsonl", []string{userLine("bravo content")}) + sessions := []*Session{a, b} + + if st := syncAll(t, ix, sessions); st.Indexed != 2 { + t.Fatalf("initial Indexed = %d, want 2", st.Indexed) + } + // Nothing changed — a re-sync must not rewrite anything. + if st := syncAll(t, ix, sessions); st.Indexed != 0 { + t.Errorf("no-op Indexed = %d, want 0", st.Indexed) + } + + // Append to one file; only it should be reindexed. + f, err := os.OpenFile(a.FilePath, os.O_APPEND|os.O_WRONLY, 0o644) + if err != nil { + t.Fatal(err) + } + f.WriteString(userLine("charlie appended") + "\n") + f.Close() + // Ensure mtime differs even on coarse-grained filesystems. + future := time.Now().Add(2 * time.Second) + os.Chtimes(a.FilePath, future, future) + fi, _ := os.Stat(a.FilePath) + a.ModTime = fi.ModTime() + + if st := syncAll(t, ix, sessions); st.Indexed != 1 { + t.Errorf("after edit Indexed = %d, want 1", st.Indexed) + } + if res, _ := searchIdx(t, ix, sessions, "charlie"); len(res) != 1 { + t.Errorf("appended content hits = %d, want 1", len(res)) + } + // The old content must still be there exactly once — not duplicated by the + // reindex, which is the classic contentless-FTS failure. + if res, _ := searchIdx(t, ix, sessions, "alpha"); len(res) != 1 { + t.Errorf("pre-existing content hits = %d, want 1 (duplicate rows?)", len(res)) + } +} + +func TestSyncDropsDeletedTranscripts(t *testing.T) { + ix, dir := openTestIndex(t) + a := writeIndexTranscript(t, dir, "a.jsonl", []string{userLine("alpha content")}) + b := writeIndexTranscript(t, dir, "b.jsonl", []string{userLine("bravo content")}) + syncAll(t, ix, []*Session{a, b}) + + os.Remove(b.FilePath) + st := syncAll(t, ix, []*Session{a}) + if st.Removed != 1 { + t.Errorf("Removed = %d, want 1", st.Removed) + } + if res, _ := searchIdx(t, ix, []*Session{a}, "bravo"); len(res) != 0 { + t.Errorf("deleted session still returns %d hits", len(res)) + } +} + +// A quote in a query must not produce an FTS5 syntax error. +func TestQuotesInQueryAreEscaped(t *testing.T) { + ix, dir := openTestIndex(t) + s := writeIndexTranscript(t, dir, "a.jsonl", []string{ + userLine(`he said "worktree" loudly`), + }) + syncAll(t, ix, []*Session{s}) + + for _, q := range []string{`said "worktree`, `"he said \"worktree\" loudly"`} { + if _, _, err := SearchWithIndex(context.Background(), ix, []*Session{s}, ParseSearchQuery(q), 0); err != nil { + t.Errorf("query %q: %v", q, err) + } + } +} + +func TestResultsOrderedByRecency(t *testing.T) { + ix, dir := openTestIndex(t) + old := writeIndexTranscript(t, dir, "old.jsonl", []string{userLine("shared marker here")}) + recent := writeIndexTranscript(t, dir, "recent.jsonl", []string{userLine("shared marker here")}) + + past := time.Now().Add(-48 * time.Hour) + os.Chtimes(old.FilePath, past, past) + fi, _ := os.Stat(old.FilePath) + old.ModTime = fi.ModTime() + + sessions := []*Session{old, recent} + syncAll(t, ix, sessions) + + res, _ := searchIdx(t, ix, sessions, "marker") + if len(res) != 2 { + t.Fatalf("hits = %d, want 2", len(res)) + } + if res[0].Session.FilePath != recent.FilePath { + t.Errorf("first result = %s, want the newer session", filepath.Base(res[0].Session.FilePath)) + } +} + +func TestCorruptIndexIsRebuilt(t *testing.T) { + dir := t.TempDir() + path := indexFilePath(dir) + if err := os.WriteFile(path, []byte("this is not a database"), 0o644); err != nil { + t.Fatal(err) + } + + ix, err := OpenIndex(dir) + if err != nil { + t.Fatalf("OpenIndex on corrupt file: %v", err) + } + defer ix.Close() + + s := writeIndexTranscript(t, dir, "a.jsonl", []string{userLine("worktree content")}) + syncAll(t, ix, []*Session{s}) + if res, _ := searchIdx(t, ix, []*Session{s}, "worktree"); len(res) != 1 { + t.Errorf("hits after rebuild = %d, want 1", len(res)) + } +} + +// tool_result is the only block type the index is allowed to omit. Any other +// omission is a silent divergence from the scan, so enumerate the types the +// parser can produce and assert the rule directly. Found the hard way: system_tag +// blocks were dropped, costing real hits on the live corpus. +func TestOnlyToolResultIsExcludedFromIndex(t *testing.T) { + types := []string{ + "text", "tool_use", "thinking", "system_tag", + "redacted_thinking", "server_tool_use", "advisor_tool_result", + } + for _, ty := range types { + b := &ContentBlock{Type: ty, Text: "body"} + if !indexableBlock(b) { + t.Errorf("block type %q is excluded from the index but the scan matches it", ty) + } + } + if indexableBlock(&ContentBlock{Type: "tool_result", Text: "body"}) { + t.Error("tool_result must stay out of the index (it is half the corpus)") + } +} + +// A system_tag block must be findable through the index, end to end. +func TestSystemTagBlocksAreIndexed(t *testing.T) { + ix, dir := openTestIndex(t) + s := writeIndexTranscript(t, dir, "a.jsonl", []string{ + userLine("uniquemarker inside a tag"), + }) + syncAll(t, ix, []*Session{s}) + + res, _ := searchIdx(t, ix, []*Session{s}, "uniquemarker") + scan := collectScan(context.Background(), []*Session{s}, ParseSearchQuery("uniquemarker"), 0) + if len(res) != len(scan) { + t.Errorf("index=%d scan=%d for system_tag content", len(res), len(scan)) + } + if len(res) == 0 { + t.Error("system_tag content not searchable via index") + } +} + +func TestSnippetHighlightsMatch(t *testing.T) { + ix, dir := openTestIndex(t) + s := writeIndexTranscript(t, dir, "a.jsonl", []string{ + userLine("a long preamble that goes on before the worktree token appears here"), + }) + syncAll(t, ix, []*Session{s}) + + res, _ := searchIdx(t, ix, []*Session{s}, "worktree") + if len(res) != 1 { + t.Fatalf("hits = %d, want 1", len(res)) + } + if !strings.Contains(res[0].Snippet, "worktree") { + t.Errorf("snippet lacks the match: %q", res[0].Snippet) + } +} diff --git a/internal/session/search_index.go b/internal/session/search_index.go new file mode 100644 index 0000000..786fc8a --- /dev/null +++ b/internal/session/search_index.go @@ -0,0 +1,220 @@ +package session + +import ( + "bufio" + "context" + "os" + "sort" + "strings" +) + +// SearchMode records how a set of results was produced, so the UI can be honest +// about what was searched. +type SearchMode int + +const ( + // SearchModeScan means every transcript was read end to end. + SearchModeScan SearchMode = iota + // SearchModeIndex means results came from the FTS index and are equivalent + // to a scan for this query. + SearchModeIndex + // SearchModeIndexPartial means results came from the index, which does not + // cover tool_result content. + SearchModeIndexPartial +) + +func (m SearchMode) String() string { + switch m { + case SearchModeIndex: + return "index" + case SearchModeIndexPartial: + return "index (no tool output)" + default: + return "full scan" + } +} + +// SearchWithIndex answers q from the FTS index when it can, and falls back to +// the full scan when it cannot. It returns the mode actually used. +// +// The caller is responsible for having Sync'd the index; this function never +// writes to it, so a stale index yields stale results rather than a stall. +func SearchWithIndex(ctx context.Context, ix *Index, sessions []*Session, q SearchQuery, limit int) ([]SearchResult, SearchMode, error) { + if ix == nil || ix.Coverage(q) == CoverageNone { + return collectScan(ctx, sessions, q, limit), SearchModeScan, nil + } + + bySession := make(map[string]*Session, len(sessions)) + for _, s := range sessions { + if s != nil && s.FilePath != "" { + bySession[s.FilePath] = s + } + } + + // Over-fetch: post-filtering (exclusions, tool prefix) drops some hits, and + // a limit applied in SQL would otherwise silently truncate good results. + sqlLimit := 0 + if limit > 0 { + sqlLimit = limit * 4 + } + + hits, err := ix.queryIndex(ctx, q, bySession, sqlLimit) + if err != nil { + // An index failure is not a search failure — degrade to the scan. + return collectScan(ctx, sessions, q, limit), SearchModeScan, nil + } + + results, err := hydrateHits(ctx, hits, bySession, q, limit) + if err != nil { + return collectScan(ctx, sessions, q, limit), SearchModeScan, nil + } + + mode := SearchModeIndex + if ix.Coverage(q) == CoveragePartial { + mode = SearchModeIndexPartial + } + return results, mode, nil +} + +// hydrateHits turns index locations back into SearchResults by re-reading the +// one transcript line each hit points at. Hits arrive grouped by path, so the +// same file is opened once. +func hydrateHits(ctx context.Context, hits []indexHit, bySession map[string]*Session, q SearchQuery, limit int) ([]SearchResult, error) { + var results []SearchResult + + for i := 0; i < len(hits); { + select { + case <-ctx.Done(): + return results, ctx.Err() + default: + } + + path := hits[i].path + j := i + for j < len(hits) && hits[j].path == path { + j++ + } + group := hits[i:j] + i = j + + sess := bySession[path] + if sess == nil { + continue + } + f, err := os.Open(path) + if err != nil { + continue + } + + for _, h := range group { + line, err := readLineAt(f, h.lineOff) + if err != nil || line == "" { + continue + } + entry, err := ParseEntry(line) + if err != nil || h.blockIdx >= len(entry.Content) { + continue + } + block := &entry.Content[h.blockIdx] + + res, ok := matchBlock(sess, &entry, block, q) + if !ok { + continue + } + results = append(results, res) + if limit > 0 && len(results) >= limit { + f.Close() + return results, nil + } + } + f.Close() + } + return results, nil +} + +// readLineAt reads the single newline-terminated line starting at off. +func readLineAt(f *os.File, off int64) (string, error) { + if _, err := f.Seek(off, 0); err != nil { + return "", err + } + r := bufio.NewReaderSize(f, 256*1024) + sc := bufio.NewScanner(r) + sc.Buffer(make([]byte, 256*1024), 10*1024*1024) + if !sc.Scan() { + return "", sc.Err() + } + return sc.Text(), nil +} + +// matchBlock re-applies the query to a single block. The index narrows +// candidates; this is what makes the result set exactly match the scan's +// semantics, including exclusions and tool-name prefixes. +func matchBlock(sess *Session, entry *Entry, block *ContentBlock, q SearchQuery) (SearchResult, bool) { + if entry.IsMeta { + return SearchResult{}, false + } + if q.Role != "" && entry.Role != q.Role { + return SearchResult{}, false + } + if q.ToolName != "" { + if block.Type != "tool_use" || !toolNameMatches(block.ToolName, q.ToolName) { + return SearchResult{}, false + } + } + + text := blockSearchText(block) + lower := strings.ToLower(text) + + for _, term := range q.Terms { + if !strings.Contains(lower, term) { + return SearchResult{}, false + } + } + for _, phrase := range q.Phrases { + if !strings.Contains(lower, phrase) { + return SearchResult{}, false + } + } + for _, excl := range q.Exclude { + if strings.Contains(lower, excl) { + return SearchResult{}, false + } + } + + // The entry is a loop-local value in the caller; copy it so the returned + // pointer stays valid and distinct per result. + e := *entry + return SearchResult{ + Session: sess, + Entry: &e, + Block: block, + Snippet: buildSnippet(text, q.Terms, q.Phrases), + }, true +} + +// collectScan drains the existing full-scan search into an ordered slice. +// Results are sorted newest session first to match the indexed path. +func collectScan(ctx context.Context, sessions []*Session, q SearchQuery, limit int) []SearchResult { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + var out []SearchResult + for res := range SearchSessions(sessions, q, ctx) { + out = append(out, res) + if limit > 0 && len(out) >= limit*4 { + break // over-fetch, then sort and trim below + } + } + + sort.SliceStable(out, func(i, j int) bool { + si, sj := out[i].Session, out[j].Session + if si == nil || sj == nil { + return false + } + return si.ModTime.After(sj.ModTime) + }) + if limit > 0 && len(out) > limit { + out = out[:limit] + } + return out +} diff --git a/internal/tui/app.go b/internal/tui/app.go index 1cbd6f3..f3f032a 100644 --- a/internal/tui/app.go +++ b/internal/tui/app.go @@ -574,6 +574,18 @@ type App struct { searchResultList list.Model searchLoading bool searchCancel context.CancelFunc + searchMode session.SearchMode + + // convHighlightTerms are the plain-text terms of the query that led here via + // a cross-session search result. They keep the match visible in the + // conversation the jump lands in, and are cleared when the conversation is + // left or another search runs. + convHighlightTerms []string + + // contentIndex is the FTS index backing cross-session search. It is opened + // lazily on the first search and owned by the main loop; search commands + // only read it. + contentIndex *session.Index } // selectedSession returns the currently selected session from the session list. @@ -1342,7 +1354,7 @@ func (a *App) Update(msg tea.Msg) (tea.Model, tea.Cmd) { return a, nil case searchBatchMsg: - a.updateSearchResults(msg.results) + a.updateSearchResults(msg.results, msg.mode) return a, nil case refsExtractedMsg: diff --git a/internal/tui/blockfilter.go b/internal/tui/blockfilter.go index 8b1fae7..44fc63d 100644 --- a/internal/tui/blockfilter.go +++ b/internal/tui/blockfilter.go @@ -133,3 +133,30 @@ func countVisibleBlocks(vis []bool) int { } return n } + +// highlightableTerms extracts the plain-text terms of a filter expression — +// the ones that match against block *content* and are therefore worth +// highlighting in the rendered output. +// +// Structured tokens (is:tool, tool:Name) select whole blocks rather than text +// inside them, so highlighting them would paint the literal string "is:tool" +// wherever it happened to appear. Negated terms mark what must be *absent*, so +// by definition there is nothing to highlight. +func highlightableTerms(filter string) []string { + filter = strings.TrimSpace(filter) + if filter == "" { + return nil + } + var out []string + for _, term := range strings.Fields(filter) { + if strings.HasPrefix(term, "!") { + continue + } + lower := strings.ToLower(term) + if strings.HasPrefix(lower, "is:") || strings.HasPrefix(lower, "tool:") { + continue + } + out = append(out, term) + } + return out +} diff --git a/internal/tui/conversation.go b/internal/tui/conversation.go index a159a3a..459b38f 100644 --- a/internal/tui/conversation.go +++ b/internal/tui/conversation.go @@ -53,6 +53,14 @@ func (a *App) openConversation(sess session.Session) tea.Cmd { a.conv.toolUseToAgent = buildToolUseToAgentMap(entries) a.conv.inspector = conversationInspector{Scope: session.ScopeNode} a.conv.split.PreviewOnly = false + // Opening a conversation clears any carried-over search highlight by + // default; the cross-session search path re-applies its terms right after + // this call. Defaulting to off here means the seven non-search entry points + // cannot leak a stale query into an unrelated session. + a.convHighlightTerms = nil + if a.conv.split.Folds != nil { + a.conv.split.Folds.ExtraHighlight = nil + } // File-backed tasks provide durable metadata; transcript events provide the // latest state and IDs for current TaskCreate/TaskUpdate calls. @@ -1425,6 +1433,11 @@ func (a *App) setConvPreviewText(content string) { func (a *App) setConvPreviewTextKey(content, cacheKey string) { sp := &a.conv.split + // Every preview and details render funnels through here, so this is the one + // place that keeps a cross-session search match visible after the jump. + if len(a.convHighlightTerms) > 0 { + content = highlightSearchTerms(content, a.convHighlightTerms, -1) + } oldOffset := sp.Preview.YOffset sameKey := sp.CacheKey == cacheKey sp.CacheKey = cacheKey diff --git a/internal/tui/conversation_text.go b/internal/tui/conversation_text.go index 05fa0f9..3e344f6 100644 --- a/internal/tui/conversation_text.go +++ b/internal/tui/conversation_text.go @@ -33,14 +33,30 @@ func highlightSearchMatches(content, term string, currentLine int) string { if term == "" { return content } - lowerTerm := strings.ToLower(term) + return highlightSearchTerms(content, []string{term}, currentLine) +} + +// highlightSearchTerms is highlightSearchMatches for several terms at once. +// Filter expressions are AND-ed sets of words, so every one of them is a reason +// the block is on screen and all of them get painted. +// +// Terms are applied one after another over the already-highlighted line. That +// is safe because highlightLine walks visible characters and copies ANSI +// sequences through untouched, so an earlier term's escapes neither shift the +// match positions of a later term nor get matched themselves. +func highlightSearchTerms(content string, terms []string, currentLine int) string { + if len(terms) == 0 { + return content + } lines := strings.Split(content, "\n") for i, line := range lines { - plain := stripANSI(line) - if !strings.Contains(strings.ToLower(plain), lowerTerm) { - continue + plain := strings.ToLower(stripANSI(line)) + for _, term := range terms { + if term == "" || !strings.Contains(plain, strings.ToLower(term)) { + continue + } + lines[i] = highlightLine(lines[i], term, i == currentLine) } - lines[i] = highlightLine(line, term, i == currentLine) } return strings.Join(lines, "\n") } diff --git a/internal/tui/highlight_search_test.go b/internal/tui/highlight_search_test.go new file mode 100644 index 0000000..fbb308c --- /dev/null +++ b/internal/tui/highlight_search_test.go @@ -0,0 +1,201 @@ +package tui + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/sendbird/ccx/internal/session" +) + +// hlCount counts highlighted spans by looking for the ANSI start codes +// highlightLine emits. +func hlCount(s string) int { + return strings.Count(s, "\x1b[43;30m") + strings.Count(s, "\x1b[46;30m") +} + +func TestHighlightSearchTermsPaintsEveryTerm(t *testing.T) { + content := "the worktree was pruned\nunrelated line\nprune again" + got := highlightSearchTerms(content, []string{"worktree", "prune"}, -1) + + // worktree(1) + pruned(1) on line 1, prune(1) on line 3. + if n := hlCount(got); n != 3 { + t.Errorf("highlighted spans = %d, want 3\n%q", n, got) + } + if strings.Contains(strings.Split(got, "\n")[1], "\x1b[43;30m") { + t.Error("non-matching line was highlighted") + } +} + +// Applying a second term must not corrupt the first term's escapes or match +// them as if they were text. +func TestHighlightSearchTermsOverlappingApplication(t *testing.T) { + got := highlightSearchTerms("alpha beta gamma", []string{"alpha", "beta", "gamma"}, -1) + if n := hlCount(got); n != 3 { + t.Errorf("spans = %d, want 3\n%q", n, got) + } + if plain := stripANSI(got); plain != "alpha beta gamma" { + t.Errorf("visible text changed: %q", plain) + } +} + +func TestHighlightSearchTermsEmptyInputs(t *testing.T) { + if got := highlightSearchTerms("text", nil, -1); got != "text" { + t.Errorf("nil terms changed content: %q", got) + } + if got := highlightSearchTerms("text", []string{""}, -1); got != "text" { + t.Errorf("empty term changed content: %q", got) + } +} + +// Structured filter tokens select whole blocks; highlighting them would paint +// the literal string "is:tool" wherever it appears in the text. +func TestHighlightableTermsSkipsStructuralTokens(t *testing.T) { + cases := []struct { + filter string + want []string + }{ + {"worktree", []string{"worktree"}}, + {"is:tool worktree", []string{"worktree"}}, + {"tool:Bash prune", []string{"prune"}}, + {"is:error", nil}, + {"tool:mcp*", nil}, + {"!skipme worktree", []string{"worktree"}}, + {"alpha beta", []string{"alpha", "beta"}}, + {"", nil}, + {" ", nil}, + } + for _, c := range cases { + got := highlightableTerms(c.filter) + if len(got) != len(c.want) { + t.Errorf("highlightableTerms(%q) = %v, want %v", c.filter, got, c.want) + continue + } + for i := range got { + if got[i] != c.want[i] { + t.Errorf("highlightableTerms(%q) = %v, want %v", c.filter, got, c.want) + break + } + } + } +} + +// A negated term marks what must be absent — there is nothing to paint, and +// painting it would be actively misleading. +func TestNegatedTermIsNotHighlighted(t *testing.T) { + terms := highlightableTerms("!worktree prune") + for _, term := range terms { + if strings.Contains(term, "worktree") { + t.Errorf("negated term %q would be highlighted", term) + } + } +} + +// Opening a conversation from anywhere other than a search result must not +// inherit a previous jump's highlight. Seven of the eight openConversation +// callers are not search jumps, so the clear has to live in the callee. +func TestOpenConversationClearsSearchHighlight(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "s.jsonl") + line := `{"type":"user","uuid":"u1","message":{"role":"user","content":[{"type":"text","text":"hello there"}]}}` + if err := os.WriteFile(path, []byte(line+"\n"), 0o644); err != nil { + t.Fatal(err) + } + + a := &App{} + a.conv.split.Folds = &FoldState{} + a.convHighlightTerms = []string{"stale"} + a.conv.split.Folds.ExtraHighlight = []string{"stale"} + + a.openConversation(session.Session{FilePath: path}) + + if len(a.convHighlightTerms) != 0 { + t.Errorf("convHighlightTerms = %v, want cleared", a.convHighlightTerms) + } + if len(a.conv.split.Folds.ExtraHighlight) != 0 { + t.Errorf("Folds.ExtraHighlight = %v, want cleared", a.conv.split.Folds.ExtraHighlight) + } +} + +func TestSearchModeStringsAreDistinct(t *testing.T) { + seen := map[string]bool{} + for _, m := range []session.SearchMode{ + session.SearchModeScan, + session.SearchModeIndex, + session.SearchModeIndexPartial, + } { + s := m.String() + if s == "" { + t.Error("empty mode string") + } + if seen[s] { + t.Errorf("duplicate mode string %q", s) + } + seen[s] = true + } +} + +// End-to-end through the real App: picking a cross-session search result must +// leave the term highlighted in the preview the jump lands on. Unit tests cover +// the paint function; this covers the wiring between search, openConversation, +// and the preview renderer. +func TestSearchJumpHighlightsPreviewEndToEnd(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "s.jsonl") + lines := []string{ + `{"type":"user","uuid":"u1","message":{"role":"user","content":[{"type":"text","text":"please prune the worktree now"}]}}`, + `{"type":"assistant","uuid":"a1","message":{"role":"assistant","content":[{"type":"text","text":"done, worktree removed"}]}}`, + } + if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")+"\n"), 0o644); err != nil { + t.Fatal(err) + } + fi, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + sess := session.Session{ + ID: "sid1", ShortID: "sid1", + ProjectPath: dir, ProjectName: "proj", + FilePath: path, ModTime: fi.ModTime(), MsgCount: 2, + } + + a := newTestApp([]session.Session{sess}) + a.searchQuery = "worktree" + + entries, err := session.LoadMessages(path) + if err != nil { + t.Fatal(err) + } + if len(entries) == 0 { + t.Fatal("no entries loaded") + } + result := session.SearchResult{ + Session: &sess, + Entry: &entries[0], + Block: &entries[0].Content[0], + } + + a.openSearchResult(result) + + if len(a.convHighlightTerms) == 0 { + t.Fatal("jump did not carry the query terms into the conversation") + } + if a.convHighlightTerms[0] != "worktree" { + t.Errorf("carried terms = %v, want [worktree]", a.convHighlightTerms) + } + + // The preview content itself must carry the highlight escapes. + a.conv.split.Show = true + a.setConvPreviewText("the worktree line in preview") + got := a.conv.split.Preview.View() + if hlCount(got) == 0 { + t.Errorf("preview has no highlight for the searched term:\n%q", stripANSI(got)) + } + + // Opening an unrelated session afterwards must not keep painting. + a.openConversation(sess) + if len(a.convHighlightTerms) != 0 { + t.Errorf("highlight leaked into a later conversation: %v", a.convHighlightTerms) + } +} diff --git a/internal/tui/messages.go b/internal/tui/messages.go index ce7d135..253d16e 100644 --- a/internal/tui/messages.go +++ b/internal/tui/messages.go @@ -253,6 +253,12 @@ func renderConversationPreview(msgs []mergedMsg, width, cursor int, expanded map text := entryFullText(e) if text != "" { wrapped := wrapText(text, textW) + // The one-line preview above is highlighted via highlightSnippet, + // but that only covers the summary. Expanding is exactly when the + // user is looking for the term in the body, so paint it here too. + if filterTerm != "" { + wrapped = highlightSearchTerms(wrapped, highlightableTerms(filterTerm), -1) + } for _, line := range strings.Split(wrapped, "\n") { row.WriteString(" " + line + "\n") } diff --git a/internal/tui/search.go b/internal/tui/search.go index 0f252e7..88aa8f3 100644 --- a/internal/tui/search.go +++ b/internal/tui/search.go @@ -44,6 +44,9 @@ func (a *App) enterSearchMode() { a.searchQuery = "" a.searchResults = nil a.searchLoading = false + // Drop any highlight carried over from a previous jump; the next result + // picked from this session sets its own. + a.convHighlightTerms = nil ti := textinput.New() ti.Placeholder = "Search all sessions..." @@ -88,29 +91,42 @@ func (a *App) executeSearch() tea.Cmd { parsed := session.ParseSearchQuery(query) - return func() tea.Msg { - results := session.SearchSessions(sessions, parsed, ctx) + // Open the index on the main loop, not inside the command: the command runs + // on its own goroutine and must not mutate App state. + if a.contentIndex == nil { + if ix, err := session.OpenIndex(a.config.ClaudeDir); err == nil { + a.contentIndex = ix + } + } + ix := a.contentIndex - go func() { - for result := range results { - // Send each result as a message (will be batched by tea runtime) - // This is a simplified approach - in production you'd batch these - _ = result + return func() tea.Msg { + // Bring the index up to date first: only transcripts whose mtime or + // size moved are re-read, so the steady-state cost is a stat per + // session. A failure here is not fatal — SearchWithIndex falls back to + // the full scan when the index is nil or unusable. + if ix != nil { + if _, err := ix.Sync(ctx, sessions, nil); err != nil && ctx.Err() != nil { + return searchBatchMsg{} } - }() - - // Collect all results synchronously for simplicity - var allResults []session.SearchResult - for result := range results { - allResults = append(allResults, result) } - return searchBatchMsg{results: allResults} + results, mode, err := session.SearchWithIndex(ctx, ix, sessions, parsed, searchResultLimit) + if err != nil { + return searchBatchMsg{} + } + return searchBatchMsg{results: results, mode: mode} } } +// searchResultLimit caps how many hits are hydrated and shown. A broad query +// can match tens of thousands of blocks; past the first few hundred the list is +// no longer something a person scrolls, and building them all costs real time. +const searchResultLimit = 500 + type searchBatchMsg struct { results []session.SearchResult + mode session.SearchMode } func (a *App) handleSearchKey(msg tea.KeyMsg) (tea.Model, tea.Cmd) { @@ -161,12 +177,24 @@ func (a *App) handleSearchKey(msg tea.KeyMsg) (tea.Model, tea.Cmd) { } func (a *App) openSearchResult(result session.SearchResult) { + // The query's text terms, kept so the match the user picked stays visible in + // the conversation they land on. + parsed := session.ParseSearchQuery(a.searchQuery) + terms := append(append([]string(nil), parsed.Terms...), parsed.Phrases...) + for i, sess := range a.sessions { if sess.ID == result.Session.ID { a.sessionList.Select(i) a.currentSess = sess a.openConversation(sess) + // Set after openConversation, which clears the highlight so that the + // other entry points cannot inherit a stale query. + a.convHighlightTerms = terms + if a.conv.split.Folds != nil { + a.conv.split.Folds.ExtraHighlight = terms + } + // Jump to the message containing the search result. Selection indices // are always in the list's visible coordinate space. targetUUID := result.Entry.UUID @@ -230,7 +258,16 @@ func (a *App) renderSearchModal(bg string) string { case a.searchQuery != "" && len(a.searchResults) == 0: sb.WriteString(dimStyle.Render("No results found")) case len(a.searchResults) > 0: - sb.WriteString(dimStyle.Render(fmt.Sprintf("%d results", len(a.searchResults))) + "\n") + // Say what was actually searched. The index does not cover tool_result + // content, so a silent "N results" would overstate the coverage. + count := fmt.Sprintf("%d results", len(a.searchResults)) + if len(a.searchResults) >= searchResultLimit { + count = fmt.Sprintf("first %d results", searchResultLimit) + } + if a.searchMode == session.SearchModeIndexPartial { + count += dimStyle.Render(" · tool output not indexed") + } + sb.WriteString(dimStyle.Render(count) + "\n") // Reserve rows already used (title + input box(3) + count + help) so the // list fits inside the modal without overflowing. listH := max(min(len(a.searchResults), bodyMaxH-6), 3) @@ -277,8 +314,9 @@ func (a *App) renderSearchModal(bg string) string { return overlayCenter(bg, modalStyle.Render(body), screenW, screenH) } -func (a *App) updateSearchResults(results []session.SearchResult) { +func (a *App) updateSearchResults(results []session.SearchResult, mode session.SearchMode) { a.searchResults = results + a.searchMode = mode a.searchLoading = false items := make([]list.Item, len(results)) diff --git a/internal/tui/splitpane.go b/internal/tui/splitpane.go index 790d29b..594b6c4 100644 --- a/internal/tui/splitpane.go +++ b/internal/tui/splitpane.go @@ -58,6 +58,10 @@ type FoldState struct { BlockStarts []int BlockVisible []bool // nil = all visible; non-nil = per-block visibility BlockFilter string // current filter expression (empty = no filter) + // ExtraHighlight are terms to paint that did not come from BlockFilter — + // currently the cross-session search query the jump into this conversation + // came from. Kept separate so clearing the block filter does not drop them. + ExtraHighlight []string HideHooks bool // true = suppress hook badges/details in render Selected foldSet // block indices selected for copy BlockSourceIdx []int // parallel to Entry.Content; -1 = unknown source @@ -562,6 +566,12 @@ func (sp *SplitPane) RefreshFoldPreview(totalW, splitRatio int) { } content := rp.content + // Paint the filter's text terms. The filter already decided which blocks are + // on screen; without this the user still has to hunt for the word inside + // them. Block-selecting tokens (is:/tool:) are skipped by highlightableTerms. + if terms := append(highlightableTerms(sp.Folds.BlockFilter), sp.Folds.ExtraHighlight...); len(terms) > 0 { + content = highlightSearchTerms(content, terms, -1) + } padLines := 0 if sp.BottomAlign && rp.lineCount < sp.Preview.Height { padLines = sp.Preview.Height - rp.lineCount