diff --git a/cmd/litefs/mount_test.go b/cmd/litefs/mount_test.go index 8208961..50073c0 100644 --- a/cmd/litefs/mount_test.go +++ b/cmd/litefs/mount_test.go @@ -298,6 +298,117 @@ func TestSingleNode_RecoverFromInitialRollback(t *testing.T) { } } +// Ensure that CommitJournal correctly computes the incremental checksum when +// pages are modified multiple times. +func TestSingleNode_JournalMultipleUpdates(t *testing.T) { + // This test only applies to the rollback journal, but it passes in WAL mode. + + cmd := runMountCommand(t, newMountCommand(t, t.TempDir(), nil)) + db := testingutil.OpenSQLDB(t, filepath.Join(cmd.Config.FUSE.Dir, "db")) + + // Disable page cache so dirty pages are written immediately. + if _, err := db.Exec(`PRAGMA cache_size = 0`); err != nil { + t.Fatal(err) + } + + if _, err := db.Exec(`CREATE TABLE t (x)`); err != nil { + t.Fatal(err) + } + for i := 0; i < 500; i++ { + if _, err := db.Exec(`INSERT INTO t VALUES (?)`, strings.Repeat("x", 60)); err != nil { + t.Fatal(err) + } + } + + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + defer func() { _ = tx.Rollback() }() + + // Update all pages (LiteFS will store the previous page checksum). + if _, err := tx.Exec(`UPDATE t SET x = ?`, strings.Repeat("y", 60)); err != nil { + t.Fatal(err) + } + // Update all pages again (LiteFS should not update the previous page checksum). + if _, err := tx.Exec(`UPDATE t SET x = ?`, strings.Repeat("z", 60)); err != nil { + t.Fatal(err) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } +} + +// Ensure that CommitJournal correctly computes the checksum when the commit +// truncates dirty pages. +func TestSingleNode_JournalTruncateDirtyPages(t *testing.T) { + if testingutil.IsWALMode() { + // The page count assertion will fail in WAL mode. + t.Skip("test only applies to rollback journal, skipping") + } + + cmd := runMountCommand(t, newMountCommand(t, t.TempDir(), nil)) + db := testingutil.OpenSQLDB(t, filepath.Join(cmd.Config.FUSE.Dir, "db")) + + // Make sure the database is truncated on commit. + if _, err := db.Exec(`PRAGMA auto_vacuum = FULL`); err != nil { + t.Fatal(err) + } + + if _, err := db.Exec(`CREATE TABLE t (id INTEGER PRIMARY KEY, x)`); err != nil { + t.Fatal(err) + } + for i := 0; i < 500; i++ { + if _, err := db.Exec(`INSERT INTO t (x) VALUES (?)`, strings.Repeat("x", 60)); err != nil { + t.Fatal(err) + } + } + + var pageCount int + if err := db.QueryRow(`PRAGMA page_count`).Scan(&pageCount); err != nil { + t.Fatal(err) + } + t.Logf("pre-commit pageCount = %d", pageCount) + if got, want := pageCount, 8; got <= want { + t.Fatalf("got %d pages, want more than %d", got, want) + } + + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + defer func() { _ = tx.Rollback() }() + + // Make all pages dirty. + if _, err := tx.Exec(`UPDATE t SET x = ?`, strings.Repeat("y", 60)); err != nil { + t.Fatal(err) + } + // Truncate the database. + if _, err := tx.Exec(`DELETE FROM t WHERE id > ?`, 100); err != nil { + t.Fatal(err) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + + var count int + if err := db.QueryRow(`SELECT COUNT(*) FROM t`).Scan(&count); err != nil { + t.Fatal(err) + } + if got, want := count, 100; got != want { + t.Fatalf("count=%d, want %d", got, want) + } + + // Make sure the database was truncated. + if err := db.QueryRow(`PRAGMA page_count`).Scan(&pageCount); err != nil { + t.Fatal(err) + } + t.Logf("post-commit pageCount = %d", pageCount) + if got, want := pageCount, 8; got >= want { + t.Fatalf("got %d pages, want less than %d", got, want) + } +} + func TestSingleNode_DropDB(t *testing.T) { cmd0 := runMountCommand(t, newMountCommand(t, t.TempDir(), nil)) dsn := filepath.Join(cmd0.Config.FUSE.Dir, "db") @@ -949,6 +1060,87 @@ func TestMultiNode_Drop(t *testing.T) { } } +// Ensure that CommitJournal correctly computes the checksum when +// switching from WAL mode to rollback journal mode. +func TestMultiNode_WALToJournal(t *testing.T) { + cmd0 := runMountCommand(t, newMountCommand(t, t.TempDir(), nil)) + waitForPrimary(t, cmd0) + cmd1 := runMountCommand(t, newMountCommand(t, t.TempDir(), cmd0)) + db0 := testingutil.OpenSQLDB(t, filepath.Join(cmd0.Config.FUSE.Dir, "db")) + db1 := testingutil.OpenSQLDB(t, filepath.Join(cmd1.Config.FUSE.Dir, "db")) + + // Switch to WAL mode. + if _, err := db0.Exec(`PRAGMA journal_mode = WAL`); err != nil { + t.Fatal(err) + } + + // Create a simple table. + if _, err := db0.Exec(`CREATE TABLE t (x)`); err != nil { + t.Fatal(err) + } + + for i := 0; i < 10; i++ { + // Write a value. + if _, err := db0.Exec(`INSERT INTO t VALUES (?)`, i); err != nil { + t.Fatal(err) + } + + // Ensure it invalidates the page on the secondary. + waitForSync(t, "db", cmd0, cmd1) + var x int + if err := db1.QueryRow(`SELECT MAX(x) FROM t`).Scan(&x); err != nil { + t.Fatal(err) + } else if got, want := x, i; got != want { + t.Fatalf("count=%d, want %d", got, want) + } + + // Switch to rollback journal mode after first write. + if i == 0 { + if _, err := db0.Exec(`PRAGMA journal_mode = ` + testingutil.JournalMode()); err != nil { + t.Fatal(err) + } + } + } +} + +// Ensure that CommitWAL correctly computes the checksum when +// switching from rollback journal mode to WAL mode. +func TestMultiNode_JournalToWAL(t *testing.T) { + cmd0 := runMountCommand(t, newMountCommand(t, t.TempDir(), nil)) + waitForPrimary(t, cmd0) + cmd1 := runMountCommand(t, newMountCommand(t, t.TempDir(), cmd0)) + db0 := testingutil.OpenSQLDB(t, filepath.Join(cmd0.Config.FUSE.Dir, "db")) + db1 := testingutil.OpenSQLDB(t, filepath.Join(cmd1.Config.FUSE.Dir, "db")) + + // Create a simple table. + if _, err := db0.Exec(`CREATE TABLE t (x)`); err != nil { + t.Fatal(err) + } + + for i := 0; i < 10; i++ { + // Write a value. + if _, err := db0.Exec(`INSERT INTO t VALUES (?)`, i); err != nil { + t.Fatal(err) + } + + // Ensure it invalidates the page on the secondary. + waitForSync(t, "db", cmd0, cmd1) + var x int + if err := db1.QueryRow(`SELECT MAX(x) FROM t`).Scan(&x); err != nil { + t.Fatal(err) + } else if got, want := x, i; got != want { + t.Fatalf("count=%d, want %d", got, want) + } + + // Switch to WAL mode after first write. + if i == 0 { + if _, err := db0.Exec(`PRAGMA journal_mode = WAL`); err != nil { + t.Fatal(err) + } + } + } +} + func TestMultiNode_LateJoinWithSnapshot(t *testing.T) { cmd0 := runMountCommand(t, newMountCommand(t, t.TempDir(), nil)) waitForPrimary(t, cmd0) diff --git a/db.go b/db.go index 94f3b2b..5abca43 100644 --- a/db.go +++ b/db.go @@ -52,15 +52,15 @@ type DB struct { blocks []ltx.Checksum // aggregated database page checksums; grouped by ChecksumBlockSize } - dirtyPageSet map[uint32]struct{} + dirtyPagePrevChksums map[uint32]ltx.Checksum // previous checksums for dirty database pages; used for rollback journal only wal struct { - offset int64 // offset of the start of the transaction - byteOrder binary.ByteOrder // determine by WAL header magic - salt1, salt2 uint32 // current WAL header salt values - chksum1, chksum2 uint32 // WAL checksum values at wal.offset - frameOffsets map[uint32]int64 // WAL frame offset of the last version of a given pgno before current tx - chksums map[uint32][]ltx.Checksum // wal page checksums + offset int64 // offset of the start of the transaction + byteOrder binary.ByteOrder // determine by WAL header magic + salt1, salt2 uint32 // current WAL header salt values + chksum1, chksum2 uint32 // WAL checksum values at wal.offset + frameOffsets map[uint32]int64 // WAL frame offset of the last version of a given pgno before current tx + chksums map[uint32]ltx.Checksum // wal page checksums } shmMu sync.Mutex // prevents updateSHM() from being called concurrently updatingSHM atomic.Bool // marks when updateSHM is being called so SHM writes are prevented @@ -99,7 +99,7 @@ func NewDB(store *Store, name string, path string) *DB { path: path, os: store.OS, - dirtyPageSet: make(map[uint32]struct{}), + dirtyPagePrevChksums: make(map[uint32]ltx.Checksum), Now: time.Now, } @@ -108,7 +108,7 @@ func NewDB(store *Store, name string, path string) *DB { db.haltLockAndGuard.Store((*haltLockAndGuard)(nil)) db.remoteHaltLock.Store((*HaltLock)(nil)) db.wal.frameOffsets = make(map[uint32]int64) - db.wal.chksums = make(map[uint32][]ltx.Checksum) + db.wal.chksums = make(map[uint32]ltx.Checksum) db.guardSets.m = make(map[uint64]*GuardSet) return db @@ -750,9 +750,6 @@ func (db *DB) CheckpointNoLock(ctx context.Context) (err error) { return fmt.Errorf("truncate wal: %w", err) } - // Clear per-page checksums within WAL. - db.wal.chksums = make(map[uint32][]ltx.Checksum) - // Update the SHM file. if err := db.updateSHM(); err != nil { return fmt.Errorf("update shm: %w", err) @@ -1092,12 +1089,25 @@ func (db *DB) WriteDatabaseAt(ctx context.Context, f *os.File, data []byte, offs return fmt.Errorf("database write must be exactly one page (%d bytes)", db.pageSize) } + pgno := uint32(offset/int64(db.pageSize)) + 1 + dbMode := db.Mode() + // Ensure we track dirty pages when switching from WAL to rollback journal. + if pgno == 1 && databaseModeFromFirstPage(data) == DBModeRollback { + dbMode = DBModeRollback + } + // Track dirty pages if we are using a rollback journal. This isn't // necessary with the write-ahead log (WAL) since pages are appended // instead of overwritten. We can determine the dirty set at commit-time. - pgno := uint32(offset/int64(db.pageSize)) + 1 - if db.Mode() == DBModeRollback { - db.dirtyPageSet[pgno] = struct{}{} + if dbMode == DBModeRollback { + // Store the previous checksum if the page isn't dirty yet, so we can compute the post-apply checksum. + if _, ok := db.dirtyPagePrevChksums[pgno]; !ok { + db.chksums.mu.Lock() + prevChksum, _ := db.pageChecksum(pgno, db.PageN()) + db.chksums.mu.Unlock() + + db.dirtyPagePrevChksums[pgno] = prevChksum + } } // Perform write on handle. @@ -1292,7 +1302,7 @@ func (db *DB) TruncateWAL(ctx context.Context, size int64) (err error) { // Clear all per-page checksums for the WAL. db.wal.frameOffsets = make(map[uint32]int64) - db.wal.chksums = make(map[uint32][]ltx.Checksum) + db.wal.chksums = make(map[uint32]ltx.Checksum) return nil } @@ -1307,7 +1317,7 @@ func (db *DB) RemoveWAL(ctx context.Context) (err error) { // Clear all per-page checksums for the WAL. db.wal.frameOffsets = make(map[uint32]int64) - db.wal.chksums = make(map[uint32][]ltx.Checksum) + db.wal.chksums = make(map[uint32]ltx.Checksum) return nil } @@ -1407,7 +1417,7 @@ func (db *DB) writeWALHeader(ctx context.Context, f *os.File, data []byte, offse db.wal.chksum1 = binary.BigEndian.Uint32(data[24:]) db.wal.chksum2 = binary.BigEndian.Uint32(data[28:]) db.wal.frameOffsets = make(map[uint32]int64) - db.wal.chksums = make(map[uint32][]ltx.Checksum) + db.wal.chksums = make(map[uint32]ltx.Checksum) // Passthrough write to underlying WAL file. _, err = f.WriteAt(data, offset) @@ -1627,6 +1637,9 @@ func (db *DB) CommitWAL(ctx context.Context) (err error) { } sort.Slice(pgnos, func(i, j int) bool { return pgnos[i] < pgnos[j] }) + // Calculate checksum after commit. + postApplyChecksum := prevPos.PostApplyChecksum + frame := make([]byte, walFrameSize) newWALChksums := make(map[uint32]ltx.Checksum) lockPgno := ltx.LockPgno(db.pageSize) @@ -1650,11 +1663,14 @@ func (db *DB) CommitWAL(ctx context.Context) (err error) { // Update per-page checksum. db.chksums.mu.Lock() - prevPageChksum, _ := db.pageChecksum(pgno, db.PageN(), nil) + prevPageChksum, _ := db.pageChecksum(pgno, db.PageN()) db.chksums.mu.Unlock() pageChksum := ltx.ChecksumPage(pgno, frame[WALFrameHeaderSize:]) newWALChksums[pgno] = pageChksum + // Update database checksum. + postApplyChecksum = ltx.ChecksumFlag | (postApplyChecksum ^ prevPageChksum ^ pageChksum) + TraceLog.Printf("[CommitWALPage(%s)]: pgno=%d chksum=%s prev=%s\n", db.name, pgno, pageChksum, prevPageChksum) } @@ -1672,7 +1688,7 @@ func (db *DB) CommitWAL(ctx context.Context) (err error) { // Clear per-page checksum. db.chksums.mu.Lock() - prevPageChksum, _ := db.pageChecksum(pgno, db.PageN(), nil) + prevPageChksum, _ := db.pageChecksum(pgno, db.PageN()) db.chksums.mu.Unlock() pageChksum := ltx.ChecksumPage(pgno, page) if pageChksum != prevPageChksum { @@ -1680,14 +1696,12 @@ func (db *DB) CommitWAL(ctx context.Context) (err error) { } newWALChksums[pgno] = 0 + // Remove page from database checksum. + postApplyChecksum = ltx.ChecksumFlag | (postApplyChecksum ^ prevPageChksum) + TraceLog.Printf("[CommitWALRemovePage(%s)]: pgno=%d prev=%s\n", db.name, pgno, prevPageChksum) } - // Calculate checksum after commit. - postApplyChecksum, err := db.checksum(commit, newWALChksums) - if err != nil { - return fmt.Errorf("compute checksum: %w", err) - } enc.SetPostApplyChecksum(postApplyChecksum) // Finish page block to compute checksum and then finish header block. @@ -1733,9 +1747,9 @@ func (db *DB) CommitWAL(ctx context.Context) (err error) { db.wal.frameOffsets[pgno] = off } - // Append new checksums onto WAL set. + // Update WAL checksums. for pgno, chksum := range newWALChksums { - db.wal.chksums[pgno] = append(db.wal.chksums[pgno], chksum) + db.wal.chksums[pgno] = chksum } // Move the WAL position forward and reset the segment size. @@ -1959,8 +1973,8 @@ func (db *DB) CommitJournal(ctx context.Context, mode JournalMode) (err error) { } // Build sorted list of dirty page numbers. - pgnos := make([]uint32, 0, len(db.dirtyPageSet)) - for pgno := range db.dirtyPageSet { + pgnos := make([]uint32, 0, len(db.dirtyPagePrevChksums)) + for pgno := range db.dirtyPagePrevChksums { if pgno <= commit { pgnos = append(pgnos, pgno) } @@ -1994,7 +2008,10 @@ func (db *DB) CommitJournal(ctx context.Context, mode JournalMode) (err error) { } // Remove WAL checksums. These shouldn't exist but remove them just in case. - db.wal.chksums = make(map[uint32][]ltx.Checksum) + db.wal.chksums = make(map[uint32]ltx.Checksum) + + // Compute new database checksum. + postApplyChecksum := prevPos.PostApplyChecksum // Copy transactions from main database to the LTX file in sorted order. buf := make([]byte, db.pageSize) @@ -2017,14 +2034,14 @@ func (db *DB) CommitJournal(ctx context.Context, mode JournalMode) (err error) { return fmt.Errorf("cannot encode ltx page: pgno=%d err=%w", pgno, err) } - // Update the mode if this is the first page and the write/read versions as set to WAL (2). - if pgno == 1 && buf[18] == 2 && buf[19] == 2 { + // Update the mode if this is the first page and the write/read versions are set to WAL (2). + if pgno == 1 && databaseModeFromFirstPage(buf) == DBModeWAL { dbMode = DBModeWAL } // Verify updated page matches in-memory checksum. db.chksums.mu.Lock() - pageChksum, ok := db.pageChecksum(pgno, commit, nil) + pageChksum, ok := db.pageChecksum(pgno, commit) db.chksums.mu.Unlock() if !ok { return fmt.Errorf("updated page checksum not found: pgno=%d", pgno) @@ -2034,6 +2051,10 @@ func (db *DB) CommitJournal(ctx context.Context, mode JournalMode) (err error) { return fmt.Errorf("updated page (%d) does not match in-memory checksum: %s <> %s (⊕%s)", pgno, bufChksum, pageChksum, bufChksum^pageChksum) } + // Update database checksum. + prevPageChksum := db.dirtyPagePrevChksums[pgno] + postApplyChecksum = ltx.ChecksumFlag | (postApplyChecksum ^ prevPageChksum ^ pageChksum) + TraceLog.Printf("[CommitJournalPage(%s)]: pgno=%d chksum=%s %s", db.name, pgno, pageChksum, errorKeyValue(err)) } @@ -2051,17 +2072,22 @@ func (db *DB) CommitJournal(ctx context.Context, mode JournalMode) (err error) { continue } - pageChksum, _ := db.pageChecksum(pgno, db.PageN(), nil) + pageChksum, _ := db.pageChecksum(pgno, db.PageN()) db.setDatabasePageChecksum(pgno, 0) + + // Use previous checksum for dirty pages. + prevPageChksum, ok := db.dirtyPagePrevChksums[pgno] + if !ok { + // Page isn't dirty, use current checksum. + prevPageChksum = pageChksum + } + // Remove page from database checksum. + postApplyChecksum = ltx.ChecksumFlag | (postApplyChecksum ^ prevPageChksum) + TraceLog.Printf("[CommitJournalRemovePage(%s)]: pgno=%d chksum=%s %s", db.name, pgno, pageChksum, errorKeyValue(err)) } }() - // Compute new database checksum. - postApplyChecksum, err := db.checksum(commit, nil) - if err != nil { - return fmt.Errorf("compute checksum: %w", err) - } enc.SetPostApplyChecksum(postApplyChecksum) // Finish page block to compute checksum and then finish header block. @@ -2252,7 +2278,7 @@ func (db *DB) Drop(ctx context.Context) (err error) { db.wal.chksum1 = 0 db.wal.chksum2 = 0 db.wal.frameOffsets = make(map[uint32]int64) - db.wal.chksums = make(map[uint32][]ltx.Checksum) + db.wal.chksums = make(map[uint32]ltx.Checksum) // Update transaction for database. pos = ltx.NewPos(enc.Header().MaxTXID, enc.Trailer().PostApplyChecksum) @@ -2373,7 +2399,7 @@ func (db *DB) invalidateJournal(mode JournalMode) error { return fmt.Errorf("sync database directory: %w", err) } - db.dirtyPageSet = make(map[uint32]struct{}) + db.dirtyPagePrevChksums = make(map[uint32]ltx.Checksum) return nil } @@ -2506,8 +2532,8 @@ func (db *DB) ApplyLTXNoLock(path string, fatalOnError bool) (retErr error) { return fmt.Errorf("decode ltx page[%d]: %w", i, err) } - // Update the mode if this is the first page and the write/read versions as set to WAL (2). - if phdr.Pgno == 1 && pageBuf[18] == 2 && pageBuf[19] == 2 { + // Update the mode if this is the first page and the write/read versions are set to WAL (2). + if phdr.Pgno == 1 && databaseModeFromFirstPage(pageBuf) == DBModeWAL { dbMode = DBModeWAL } @@ -2558,7 +2584,7 @@ func (db *DB) ApplyLTXNoLock(path string, fatalOnError bool) (retErr error) { db.mode.Store(dbMode) // Ensure checksum matches the post-apply checksum. - if chksum, err := db.checksum(dec.Header().Commit, nil); err != nil { + if chksum, err := db.checksum(dec.Header().Commit); err != nil { return fmt.Errorf("compute checksum: %w", err) } else if chksum != dec.Trailer().PostApplyChecksum { return fmt.Errorf("database checksum %s on TXID %s does not match LTX post-apply checksum %s", @@ -3215,7 +3241,7 @@ func (db *DB) recomputeBlockChksum(block uint32) { } // checksum returns the checksum of the database based on per-page checksums. -func (db *DB) checksum(pageN uint32, newWALChecksums map[uint32]ltx.Checksum) (ltx.Checksum, error) { +func (db *DB) checksum(pageN uint32) (ltx.Checksum, error) { if pageN == 0 { return ltx.ChecksumFlag, nil } @@ -3229,9 +3255,6 @@ func (db *DB) checksum(pageN uint32, newWALChecksums map[uint32]ltx.Checksum) (l for pgno := range db.wal.chksums { ignoredBlocks[pageChksumBlock(pgno)] = true } - for pgno := range newWALChecksums { - ignoredBlocks[pageChksumBlock(pgno)] = true - } var chksum ltx.Checksum for block := uint32(0); block < blockN; block++ { @@ -3252,7 +3275,7 @@ func (db *DB) checksum(pageN uint32, newWALChecksums map[uint32]ltx.Checksum) (l break } - pageChksum, ok := db.pageChecksum(pgno, pageN, newWALChecksums) + pageChksum, ok := db.pageChecksum(pgno, pageN) if !ok { return 0, fmt.Errorf("missing checksum for page %d", pgno) } @@ -3271,7 +3294,7 @@ func (db *DB) checksum(pageN uint32, newWALChecksums map[uint32]ltx.Checksum) (l // The lock page will always return a checksum of zero and a true. // // Database WRITE lock and db.chksums.mu should be held when invoked. -func (db *DB) pageChecksum(pgno, pageN uint32, newWALChecksums map[uint32]ltx.Checksum) (chksum ltx.Checksum, ok bool) { +func (db *DB) pageChecksum(pgno, pageN uint32) (chksum ltx.Checksum, ok bool) { // The lock page should never have a checksum. if pgno == ltx.LockPgno(db.pageSize) { return 0, true @@ -3283,17 +3306,9 @@ func (db *DB) pageChecksum(pgno, pageN uint32, newWALChecksums map[uint32]ltx.Ch return 0, false } - // If we're trying to calculate the checksum of an in-progress WAL transaction, - // we'll check the new checksums to be added first. - if len(newWALChecksums) > 0 { - if chksum, ok = newWALChecksums[pgno]; ok { - return chksum, true - } - } - - // Next, find the last valid checksum within committed WAL pages. - if chksums := db.wal.chksums[pgno]; len(chksums) > 0 { - return chksums[len(chksums)-1], true + // Next, check the committed WAL pages for a valid checksum. + if chksum := db.wal.chksums[pgno]; chksum != 0 { + return chksum, true } // Finally, pull the checksum from the database. diff --git a/litefs.go b/litefs.go index 028c8e9..9fd35ce 100644 --- a/litefs.go +++ b/litefs.go @@ -639,6 +639,16 @@ func readSQLiteDatabaseHeader(r io.Reader) (hdr sqliteDatabaseHeader, data []byt return hdr, b, nil } +// databaseModeFromFirstPage returns the database mode from the first page. +func databaseModeFromFirstPage(data []byte) DBMode { + // Write/read versions are set to WAL (2) + if data[18] == 2 && data[19] == 2 { + return DBModeWAL + } + + return DBModeRollback +} + // encodePageSize returns sz as a uint16. If sz is 64K, it returns 1. func encodePageSize(sz uint32) uint16 { if sz == 65536 { diff --git a/store_test.go b/store_test.go index 48ba511..3430319 100644 --- a/store_test.go +++ b/store_test.go @@ -297,6 +297,7 @@ func TestStore_PrimaryCtx(t *testing.T) { // This store will automatically close when the test ends. func newStore(tb testing.TB, leaser litefs.Leaser, client litefs.Client) *litefs.Store { store := litefs.NewStore(tb.TempDir(), true) + store.StrictVerify = true store.Leaser = leaser store.Client = client tb.Cleanup(func() {