Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 24 additions & 18 deletions __tests__/watcher.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -264,13 +264,19 @@ describe('FileWatcher', () => {
});
});

describe('lock contention degradation (#876)', () => {
it('disables auto-sync after prolonged lock contention, with bounded retries', async () => {
const syncFn = vi.fn().mockRejectedValue(new LockUnavailableError());
describe('lock contention recovery (#876)', () => {
it('keeps retrying with capped backoff and recovers after prolonged lock contention', async () => {
let attempts = 0;
const syncFn = vi.fn(async () => {
attempts += 1;
if (attempts <= 8) {
throw new LockUnavailableError();
}
return { filesChanged: 1, durationMs: 5 };
});
const onSyncComplete = vi.fn();
const onSyncError = vi.fn();
const onDegraded = vi.fn();
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
const watcher = newWatcher(syncFn, {
debounceMs: 25,
onSyncComplete,
Expand All @@ -281,22 +287,22 @@ describe('FileWatcher', () => {
await watcher.waitUntilReady();
__emitWatchEventForTests(testDir, 'src/long-lock.ts');

// 5 backoff retries (25·1,2,4,8,16 ms), then degrade on the 6th attempt.
await waitFor(() => !watcher.isActive(), 8000, 20);
// Cross the former five-retry budget and prove the watcher remains live
// with its stale entry retained while the competing writer owns the lock.
await waitFor(() => syncFn.mock.calls.length >= 7, 8000, 20);
expect(watcher.isActive()).toBe(true);
expect(watcher.isDegraded()).toBe(false);
expect(onDegraded).not.toHaveBeenCalled();
expect(watcher.getPendingFiles().some((p) => p.path === 'src/long-lock.ts')).toBe(true);

expect(syncFn.mock.calls.length).toBeGreaterThanOrEqual(6); // MAX_LOCK_RETRIES + 1
expect(watcher.isDegraded()).toBe(true);
expect(onDegraded).toHaveBeenCalledTimes(1);
expect(onDegraded).toHaveBeenCalledWith(expect.stringContaining('auto-sync disabled'));
// A held lock is neither a sync error nor a completion.
// The ninth attempt succeeds after eight lock-contention failures.
await waitFor(() => onSyncComplete.mock.calls.length > 0, 8000, 20);
expect(syncFn).toHaveBeenCalledTimes(9);
expect(onSyncError).not.toHaveBeenCalled();
expect(onSyncComplete).not.toHaveBeenCalled();
// Degrade stops the watcher, which clears pending state.
expect(watcher.getPendingFiles()).toEqual([]);
const disableWarnings = warnSpy.mock.calls.filter(
(c) => typeof c[0] === 'string' && c[0].includes('File watcher disabled')
);
expect(disableWarnings).toHaveLength(1);
expect(onSyncComplete).toHaveBeenCalledTimes(1);
expect(watcher.getPendingFiles().some((p) => p.path === 'src/long-lock.ts')).toBe(false);

watcher.stop();
});

it('does NOT degrade on brief contention — backoff resets after a clean sync', async () => {
Expand Down
4 changes: 2 additions & 2 deletions src/mcp/engine.ts
Original file line number Diff line number Diff line change
Expand Up @@ -265,8 +265,8 @@ export class MCPEngine {
},
onDegraded: (reason) => {
// Live watching gave up permanently (watch-resource exhaustion or a
// write lock held past the retry budget). Say so loudly and ONCE — the
// graph will no longer auto-update, so a long-running MCP session must
// persistent generic sync failure). Say so loudly and ONCE — the graph
// will no longer auto-update, so a long-running MCP session must
// not keep assuming it's fresh. The reason already names the remedy
// (`codegraph sync` / git sync hooks).
process.stderr.write(`[CodeGraph MCP] File watcher degraded — ${reason}\n`);
Expand Down
44 changes: 18 additions & 26 deletions src/sync/watcher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,11 @@ import { isCodeGraphDataDir } from '../directory';
import { watchDisabledReason } from './watch-policy';

/**
* Number of consecutive lock-contention retries the watcher tolerates before
* it gives up and degrades auto-sync. Brief contention (another writer for a
* few cycles) stays under this; a long-lived external writer crosses it.
* Maximum exponential-backoff step for lock contention. A competing writer is
* recoverable no matter how long it holds the lock, so retries continue at the
* capped interval until a clean sync resets the counter.
*/
const MAX_LOCK_RETRIES = 5;
const MAX_LOCK_BACKOFF_STEPS = 5;
/**
* Number of consecutive GENERIC (non-lock) sync failures the watcher tolerates
* before it degrades auto-sync. A deterministic failure — a tree-sitter
Expand Down Expand Up @@ -193,9 +193,9 @@ export interface WatchOptions {

/**
* Callback fired ONCE when live watching degrades permanently and auto-sync
* is disabled — OS watch-resource exhaustion (EMFILE/ENFILE), a write lock
* held past the retry budget, or a generic sync failure that persists past
* the retry budget (#1127). The string is an actionable, human-readable
* is disabled — OS watch-resource exhaustion (EMFILE/ENFILE), or a generic
* sync failure that persists past the retry budget (#1127). The string is an
* actionable, human-readable
* reason. Lets a host (MCP server, daemon, CLI) tell the user that the index
* will no longer auto-update instead of silently serving stale results.
*/
Expand Down Expand Up @@ -277,8 +277,8 @@ export class FileWatcher {
private inotifyLimitWarned = false;
/**
* One-way latch: the reason live watching was permanently disabled at runtime
* (watch-resource exhaustion, lock contention past the retry budget, or a
* persistent generic sync failure past the retry budget), or null while
* (watch-resource exhaustion or a persistent generic sync failure past the
* retry budget), or null while
* healthy. Set by {@link degrade}; cleared only by a fresh start().
*/
private degradedReason: string | null = null;
Expand Down Expand Up @@ -661,8 +661,8 @@ export class FileWatcher {

/**
* Permanently disable live watching after a terminal runtime failure
* (watch-resource exhaustion, lock contention past the retry budget, or a
* persistent generic sync failure past the retry budget).
* (watch-resource exhaustion or a persistent generic sync failure past the
* retry budget).
* Idempotent: logs one actionable warning, fires {@link WatchOptions.onDegraded}
* once, and stops the watcher. A subsequent start() clears the latch.
*/
Expand Down Expand Up @@ -872,24 +872,16 @@ export class FileWatcher {
this.onSyncComplete?.(result);
} catch (err) {
if (err instanceof LockUnavailableError) {
this.lockRetryCount += 1;
this.lockRetryCount = Math.min(this.lockRetryCount + 1, MAX_LOCK_BACKOFF_STEPS);
// Lock-failure no-op (another writer holds the lock). pendingFiles
// stays intact and the `finally` block reschedules with backoff. Keep
// brief contention quiet (debug-only — a long external index would
// otherwise spam stderr every cycle), but stop retrying forever: once a
// writer holds the lock past the budget, degrade auto-sync explicitly.
// stays intact and the `finally` block reschedules with bounded
// backoff. Keep contention quiet (debug-only — a long external index
// would otherwise spam stderr every cycle) and keep retrying: a writer
// that eventually releases the lock must not permanently disable sync.
logDebug('Watch sync skipped: file lock unavailable', {
pendingFiles: this.pendingFiles.size,
retryCount: this.lockRetryCount,
});
if (this.lockRetryCount > MAX_LOCK_RETRIES) {
this.degrade(
'CodeGraph file lock held by another process past the retry budget; ' +
'auto-sync disabled. Run `codegraph sync` once the other writer finishes ' +
'(or install git sync hooks) to refresh the graph.',
{ pendingFiles: this.pendingFiles.size, retryCount: this.lockRetryCount }
);
}
} else {
this.lockRetryCount = 0; // a non-lock failure isn't contention; reset that streak
this.syncFailureRetryCount += 1;
Expand Down Expand Up @@ -926,8 +918,8 @@ export class FileWatcher {
// generic sync failure — back off exponentially (debounceMs · 2^(n-1),
// capped) instead of retrying at the normal debounce cadence; a clean
// sync resets both counters so normal edits keep the fast debounce. Use
// the larger streak so interleaved failures still back off. A degrade()
// above already set `stopped`, so this won't reschedule a watcher that
// the larger streak so interleaved failures still back off. A generic-
// failure degrade() above already set `stopped`, so this won't reschedule a watcher that
// has given up.
if (this.pendingFiles.size > 0 && !this.stopped) {
const retryCount = Math.max(this.lockRetryCount, this.syncFailureRetryCount);
Expand Down