From 6a07eb83bd21b5b17fc06c04f3a540726a4273dd Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 21 Aug 2026 11:50:25 -0500 Subject: [PATCH 1/6] fsmonitor: retry late FSEvents cookies after timeout with_lock__wait_for_cookie() gives a filesystem provider one second to report a synchronization cookie. A healthy FSEvents stream can miss that deadline while macOS is under load. The daemon then returns a trivial response, and status scans the entire index even though event delivery is still making progress. 4b1c56aeed (fsmonitor: flush pending FSEvents before cookie wait, 2026-07-21) requested an asynchronous flush on every Darwin query but kept the same one-second deadline. f439708ff1 (Revert "fsmonitor: flush pending FSEvents before cookie wait", 2026-08-17) reverted it after a matched 48-query test still saw 12 timeouts in each arm. Avoid restoring that unqualified hot-path request. When the initial Darwin wait expires, request an asynchronous FSEvents flush and wait one more bounded interval. Successful queries retain the original wait and do not issue a flush or extend their deadline. The asynchronous call cannot block on the callback while the client holds main_lock. If the provider stays silent, retain the existing trivial-response fallback after the retry. Add a test-only callback delay to exercise both outcomes: a 1.2-second delay is recovered, while a 2.5-second delay still reaches the bounded fallback. --- builtin/fsmonitor--daemon.c | 31 ++++++++++++++++++++ compat/fsmonitor/fsm-darwin-gcc.h | 1 + compat/fsmonitor/fsm-listen-darwin.c | 15 ++++++++++ compat/fsmonitor/fsm-listen.h | 5 ++++ t/t7527-builtin-fsmonitor.sh | 43 ++++++++++++++++++++++++++++ 5 files changed, 95 insertions(+) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index 1c53a5af4dd6df..d243567de5b1a1 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -238,6 +238,37 @@ static enum fsmonitor_cookie_item_result with_lock__wait_for_cookie( &state->main_lock, &ts); if (err == ETIMEDOUT && cookie->result == FCIR_INIT) { +#ifdef __APPLE__ + struct timeval rescue_now; + + /* + * FSEvents may be healthy but late enough that its normal + * delivery misses our bounded wait. Flush only after that + * wait expires, so successful queries pay no extra cost. + * The asynchronous flush cannot block on the listener callback, + * which needs main_lock to publish the cookie. + */ + trace_printf_key(&trace_fsmonitor, + "cookie_wait: requesting FSEvents flush after initial timeout"); + fsm_listen__flush_async(state); + + /* + * Give the listener one more bounded interval to deliver and + * publish the cookie rather than falling back to a full index + * scan. A broken provider still reaches the existing error + * path instead of hanging a client indefinitely. + */ + gettimeofday(&rescue_now, NULL); + ts.tv_sec = rescue_now.tv_sec + 1; + ts.tv_nsec = rescue_now.tv_usec * 1000; + err = 0; + while (cookie->result == FCIR_INIT && !err) + err = pthread_cond_timedwait(&state->cookies_cond, + &state->main_lock, + &ts); +#endif + } + if (err == ETIMEDOUT && cookie->result == FCIR_INIT) { trace_printf_key(&trace_fsmonitor, "cookie_wait timed out"); cookie->result = FCIR_ERROR; diff --git a/compat/fsmonitor/fsm-darwin-gcc.h b/compat/fsmonitor/fsm-darwin-gcc.h index 959bc88f8f765a..b749012c959ca8 100644 --- a/compat/fsmonitor/fsm-darwin-gcc.h +++ b/compat/fsmonitor/fsm-darwin-gcc.h @@ -97,6 +97,7 @@ CFRunLoopRef CFRunLoopGetCurrent(void); extern CFStringRef kCFRunLoopDefaultMode; void FSEventStreamSetDispatchQueue(FSEventStreamRef stream, dispatch_queue_t q); unsigned char FSEventStreamStart(FSEventStreamRef stream); +FSEventStreamEventId FSEventStreamFlushAsync(FSEventStreamRef stream); void FSEventStreamStop(FSEventStreamRef stream); void FSEventStreamInvalidate(FSEventStreamRef stream); void FSEventStreamRelease(FSEventStreamRef stream); diff --git a/compat/fsmonitor/fsm-listen-darwin.c b/compat/fsmonitor/fsm-listen-darwin.c index f25d7cdd907af9..5ebc70902553c5 100644 --- a/compat/fsmonitor/fsm-listen-darwin.c +++ b/compat/fsmonitor/fsm-listen-darwin.c @@ -29,6 +29,7 @@ #include "fsmonitor--daemon.h" #include "fsmonitor-path-utils.h" #include "gettext.h" +#include "parse.h" #include "simple-ipc.h" #include "string-list.h" #include "trace.h" @@ -57,6 +58,8 @@ struct fsm_listen_data unsigned int stream_scheduled:1; unsigned int stream_started:1; + unsigned int test_cookie_delayed:1; + unsigned long test_cookie_delay_ms; }; static void log_flags_set(const char *path, const FSEventStreamEventFlags flag) @@ -453,6 +456,11 @@ static void fsevent_callback(ConstFSEventStreamRef streamRef UNUSED, } free(resolved); + if (cookie_list.nr && data->test_cookie_delay_ms && + !data->test_cookie_delayed) { + data->test_cookie_delayed = 1; + sleep_millisec(data->test_cookie_delay_ms); + } fsmonitor_publish(state, batch, &cookie_list); string_list_clear(&cookie_list, 0); strbuf_release(&tmp); @@ -511,6 +519,8 @@ int fsm_listen__ctor(struct fsmonitor_daemon_state *state) CALLOC_ARRAY(data, 1); state->listen_data = data; + data->test_cookie_delay_ms = git_env_ulong( + "GIT_TEST_FSMONITOR_COOKIE_DELAY_MS", 0); data->cfsr_event_path_key = CFStringCreateWithCString( NULL, "path", kCFStringEncodingUTF8); @@ -586,6 +596,11 @@ void fsm_listen__stop_async(struct fsmonitor_daemon_state *state) pthread_mutex_unlock(&data->dq_lock); } +void fsm_listen__flush_async(struct fsmonitor_daemon_state *state) +{ + FSEventStreamFlushAsync(state->listen_data->stream); +} + void fsm_listen__loop(struct fsmonitor_daemon_state *state) { struct fsm_listen_data *data; diff --git a/compat/fsmonitor/fsm-listen.h b/compat/fsmonitor/fsm-listen.h index 41650bf8972217..d58e01243b9ad9 100644 --- a/compat/fsmonitor/fsm-listen.h +++ b/compat/fsmonitor/fsm-listen.h @@ -38,6 +38,11 @@ void fsm_listen__dtor(struct fsmonitor_daemon_state *state); */ void fsm_listen__loop(struct fsmonitor_daemon_state *state); +#ifdef __APPLE__ +/* Request delivery of all FSEvents that occurred before this call. */ +void fsm_listen__flush_async(struct fsmonitor_daemon_state *state); +#endif + /* * Gently request that the fsmonitor listener thread shutdown. * It does not wait for it to stop. The caller should do a JOIN diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index 04c188ac50132a..afac37d7abfe31 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -196,6 +196,49 @@ test_expect_success 'implicit daemon start' ' test_must_fail git -C test_implicit fsmonitor--daemon status ' +test_expect_success MACOS 'rescue a delayed FSEvents cookie after timeout' ' + test_when_finished "stop_daemon_delete_repo test_delayed_cookie" && + + git init test_delayed_cookie && + ( + GIT_TEST_FSMONITOR_COOKIE_DELAY_MS=1200 && + GIT_TRACE_FSMONITOR="$PWD/delayed-cookie.trace" && + export GIT_TEST_FSMONITOR_COOKIE_DELAY_MS GIT_TRACE_FSMONITOR && + git -C test_delayed_cookie fsmonitor--daemon start \ + --start-timeout=10 + ) && + + test-tool -C test_delayed_cookie fsmonitor-client query \ + --token 0 >actual 2>error && + test_file_not_empty actual && + test_grep "cookie_wait: requesting FSEvents flush after initial timeout" \ + delayed-cookie.trace && + test_grep "cookie-seen:" delayed-cookie.trace && + test_grep ! "cookie_wait timed out$" delayed-cookie.trace && + test_must_be_empty error +' + +test_expect_success MACOS 'fall back when a delayed FSEvents cookie stays late' ' + test_when_finished "stop_daemon_delete_repo test_lost_cookie" && + + git init test_lost_cookie && + ( + GIT_TEST_FSMONITOR_COOKIE_DELAY_MS=2500 && + GIT_TRACE_FSMONITOR="$PWD/lost-cookie.trace" && + export GIT_TEST_FSMONITOR_COOKIE_DELAY_MS GIT_TRACE_FSMONITOR && + git -C test_lost_cookie fsmonitor--daemon start \ + --start-timeout=10 + ) && + + test-tool -C test_lost_cookie fsmonitor-client query \ + --token 0 >actual 2>error && + test_file_not_empty actual && + test_grep "cookie_wait: requesting FSEvents flush after initial timeout" \ + lost-cookie.trace && + test_grep "cookie_wait timed out$" lost-cookie.trace && + test_must_be_empty error +' + # Verify that the daemon has shutdown. Spin a few seconds to # make the test a little more robust during CI testing. # From 644a279cf3a957a774ecaf2fc318969fb226d75e Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 21 Aug 2026 14:48:44 -0500 Subject: [PATCH 2/6] fsmonitor: retain paths when compacting old batches The daemon currently assumes that each client which advances an FSMonitor token also updates the repository's canonical index. That does not hold for commands using GIT_INDEX_FILE. A private index can advance the daemon past the canonical index's token and cause the canonical index's next query to receive a global invalidation. Keep a deduplicated overflow batch instead of discarding old paths. Clients at the overflow sequence still get an exact delta. Older clients get a conservative union of paths, which may overreport but cannot miss a change. All paths are interned. Keep a pointer-identity hash set with the overflow batch so later compactions hash only newly retired paths, rather than rebuilding a set over the daemon's lifetime history. Add a regression which advances a private index repeatedly, verifies that compaction remains deduplicated, and then checks that a read-only canonical status reports both changed files without a trivial response. --- builtin/fsmonitor--daemon.c | 143 +++++++++++++++++++++++++++++++---- t/t7527-builtin-fsmonitor.sh | 51 +++++++++++++ 2 files changed, 178 insertions(+), 16 deletions(-) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index d243567de5b1a1..1ac2fcc5ebb1af 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -8,12 +8,14 @@ #include "environment.h" #include "gettext.h" #include "parse-options.h" + #include "fsmonitor-ll.h" #include "fsmonitor-ipc.h" #include "fsmonitor-settings.h" #include "compat/fsmonitor/fsm-health.h" #include "compat/fsmonitor/fsm-listen.h" #include "fsmonitor--daemon.h" +#include "khash.h" #include "simple-ipc.h" #include "strmap.h" @@ -30,6 +32,19 @@ static const char * const builtin_fsmonitor__daemon_usage[] = { }; #ifdef HAVE_FSMONITOR_DAEMON_BACKEND +static khint_t fsmonitor_path_hash(const char *path) +{ + return memhash(&path, sizeof(path)); +} + +static int fsmonitor_path_equal(const char *a, const char *b) +{ + return a == b; +} + +KHASH_INIT(fsmonitor_path_set, const char *, int, 0, + fsmonitor_path_hash, fsmonitor_path_equal) + /* * Global state loaded from config. */ @@ -421,6 +436,7 @@ struct fsmonitor_batch { const char **interned_paths; size_t nr, alloc; time_t pinned_time; + kh_fsmonitor_path_set_t *overflow_paths; }; static struct fsmonitor_token_data *fsmonitor_new_token_data(void) @@ -520,6 +536,7 @@ void fsmonitor_batch__free_list(struct fsmonitor_batch *batch) * are interned, so we don't own them. We only own * the array. */ + kh_destroy_fsmonitor_path_set(batch->overflow_paths); free(batch->interned_paths); free(batch); @@ -552,16 +569,93 @@ static void fsmonitor_batch__combine(struct fsmonitor_batch *batch_dest, batch_src->interned_paths[k]; } +static void fsmonitor_batch__add_overflow_path(struct fsmonitor_batch *batch, + const char *path) +{ + int added; + + kh_put_fsmonitor_path_set(batch->overflow_paths, path, &added); + if (!added) + return; + + ALLOC_GROW(batch->interned_paths, batch->nr + 1, batch->alloc); + batch->interned_paths[batch->nr++] = path; +} + +/* + * Collapse this batch and everything older than it into one deduplicated + * overflow batch. Every path is interned, so pointer identity is sufficient. + * + * Keep the set with the overflow batch. Future compactions then hash only + * newly retired paths instead of repeatedly rebuilding the complete set. + */ +static size_t fsmonitor_batch__compact_tail(struct fsmonitor_batch *batch, + size_t *input_nr) +{ + struct fsmonitor_batch compacted = { 0 }; + struct fsmonitor_batch *item, *overflow = NULL; + + *input_nr = 0; + for (item = batch; item; item = item->next) { + *input_nr = st_add(*input_nr, item->nr); + if (item->overflow_paths) { + overflow = item; + break; + } + } + + if (overflow) { + /* + * Reuse the persistent set and array from the prior overflow + * batch. Only the newly retired paths need a lookup. + */ + if (overflow->next) + BUG("overflow batch is not the batch tail"); + for (item = batch; item != overflow; item = item->next) { + size_t k; + + for (k = 0; k < item->nr; k++) + fsmonitor_batch__add_overflow_path( + overflow, item->interned_paths[k]); + } + compacted.interned_paths = overflow->interned_paths; + compacted.nr = overflow->nr; + compacted.alloc = overflow->alloc; + compacted.overflow_paths = overflow->overflow_paths; + overflow->interned_paths = NULL; + overflow->nr = overflow->alloc = 0; + overflow->overflow_paths = NULL; + } else { + compacted.overflow_paths = kh_init_fsmonitor_path_set(); + for (item = batch; item; item = item->next) { + size_t k; + + for (k = 0; k < item->nr; k++) + fsmonitor_batch__add_overflow_path( + &compacted, item->interned_paths[k]); + } + } + + free(batch->interned_paths); + batch->interned_paths = compacted.interned_paths; + batch->nr = compacted.nr; + batch->alloc = compacted.alloc; + batch->overflow_paths = compacted.overflow_paths; + + return batch->nr; +} + /* * To keep the batch list from growing unbounded in response to filesystem - * activity, we try to truncate old batches from the end of the list as - * they become irrelevant. + * activity, collapse old batches from the end of the list after a delay. * - * We assume that the .git/index will be updated with the most recent token - * any time the index is updated. And future commands will only ask for - * recent changes *since* that new token. So as tokens advance into the - * future, older batch items will never be requested/needed. So we can - * truncate them without loss of functionality. + * A repository may have multiple durable indexes with different tokens. In + * particular, advancing a private GIT_INDEX_FILE does not advance .git/index. + * We therefore cannot discard old paths just because one client asked for a + * newer token. Instead, keep their deduplicated union in an overflow batch. + * Requests older than the overflow sequence may receive extra paths, but not + * miss any. A request at that sequence excludes the overflow batch and + * remains exact. * * However, multiple commands may be talking to the daemon concurrently * or perform a slow command, so a little "token skew" is possible. @@ -580,6 +674,7 @@ static void fsmonitor_batch__combine(struct fsmonitor_batch *batch_dest, * the official list so that the caller can free it after leaving the lock. */ #define MY_TIME_DELAY_SECONDS (5 * 60) /* seconds */ +static unsigned long truncate_delay_seconds = MY_TIME_DELAY_SECONDS; static struct fsmonitor_batch *with_lock__truncate_old_batches( struct fsmonitor_daemon_state *state, @@ -589,6 +684,7 @@ static struct fsmonitor_batch *with_lock__truncate_old_batches( const struct fsmonitor_batch *batch; struct fsmonitor_batch *remainder; + size_t input_nr, unique_nr; if (!batch_marker) return NULL; @@ -597,13 +693,13 @@ static struct fsmonitor_batch *with_lock__truncate_old_batches( batch_marker->batch_seq_nr, (uint64_t)batch_marker->pinned_time); - for (batch = batch_marker; batch; batch = batch->next) { + for (batch = batch_marker->next; batch; batch = batch->next) { time_t t; - if (!batch->pinned_time) /* an overflow batch */ + if (batch->overflow_paths) continue; - t = batch->pinned_time + MY_TIME_DELAY_SECONDS; + t = batch->pinned_time + truncate_delay_seconds; if (t > batch_marker->pinned_time) /* too close to marker */ continue; @@ -613,9 +709,20 @@ static struct fsmonitor_batch *with_lock__truncate_old_batches( return NULL; truncate_past_here: + remainder = ((struct fsmonitor_batch *)batch)->next; + if (!remainder) + return NULL; + + unique_nr = fsmonitor_batch__compact_tail( + (struct fsmonitor_batch *)batch, &input_nr); + trace_printf_key(&trace_fsmonitor, + "Compact: batch %"PRIu64" covers %"PRIuMAX + " of %"PRIuMAX" paths", + batch->batch_seq_nr, + (uintmax_t)unique_nr, (uintmax_t)input_nr); + state->current_token_data->batch_tail = (struct fsmonitor_batch *)batch; - remainder = ((struct fsmonitor_batch *)batch)->next; ((struct fsmonitor_batch *)batch)->next = NULL; return remainder; @@ -940,13 +1047,14 @@ static int do_handle_client(struct fsmonitor_daemon_state *state, do_trivial = 1; } else if (requested_oldest_seq_nr < - token_data->batch_tail->batch_seq_nr) { + token_data->batch_tail->batch_seq_nr && + !token_data->batch_tail->overflow_paths) { /* * The client wants older events than we have for - * this token_id. This means that the end of our - * batch list was truncated and we cannot give the - * client a complete snapshot relative to their - * request. + * this token_id. A normal tail means that the end + * of our batch list was truncated and we cannot + * give the client a complete snapshot. An overflow + * tail conservatively contains all older paths. */ trace_printf_key(&trace_fsmonitor, "client requested truncated data"); @@ -1410,6 +1518,9 @@ static int fsmonitor_run_daemon(void) int err; memset(&state, 0, sizeof(state)); + truncate_delay_seconds = git_env_ulong( + "GIT_TEST_FSMONITOR_TRUNCATE_DELAY_SECONDS", + MY_TIME_DELAY_SECONDS); hashmap_init(&state.cookies, cookies_cmp, NULL, 0); pthread_mutex_init(&state.main_lock, NULL); diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index afac37d7abfe31..6d0fc8a26cacd0 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -239,6 +239,57 @@ test_expect_success MACOS 'fall back when a delayed FSEvents cookie stays late' test_must_be_empty error ' +test_expect_success MACOS 'private index cannot prune canonical index history' ' + test_when_finished "stop_daemon_delete_repo test_index_history" && + test_when_finished "rm -f private-index" && + + git init test_index_history && + ( + cd test_index_history && + test_commit base tracked && + test_commit other other && + git config core.untrackedCache true && + git config core.fsmonitor true && + ( + GIT_TEST_FSMONITOR_TRUNCATE_DELAY_SECONDS=0 && + export GIT_TEST_FSMONITOR_TRUNCATE_DELAY_SECONDS && + start_daemon --tf "$PWD/../index-history.trace" + ) && + + GIT_INDEX_FILE="$PWD/.git/index" \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + cp .git/index ../private-index && + cp .git/index .git/index.before && + + echo first >>tracked && + GIT_INDEX_FILE="$PWD/../private-index" git add -u && + echo second >>other && + GIT_INDEX_FILE="$PWD/../private-index" git add -u && + echo third >>tracked && + GIT_INDEX_FILE="$PWD/../private-index" git add -u && + echo fourth >>other && + GIT_INDEX_FILE="$PWD/../private-index" git add -u && + echo fifth >>tracked && + GIT_INDEX_FILE="$PWD/../private-index" git add -u && + test_cmp .git/index.before .git/index && + test_grep "Compact: batch" ../index-history.trace \ + >../index-history.compactions && + test_line_count = 3 ../index-history.compactions && + test_grep "covers 2 of 3 paths" ../index-history.compactions && + + GIT_OPTIONAL_LOCKS=0 \ + GIT_TRACE2_EVENT="$PWD/.git/canonical.trace" \ + git status --porcelain=v2 >.git/canonical && + test_line_count = 2 .git/canonical && + test_grep "^1 \.M .* tracked$" .git/canonical && + test_grep "^1 \.M .* other$" .git/canonical && + test_cmp .git/index.before .git/index && + ! test_trace2_data fsm_client query/trivial-response 1 \ + <.git/canonical.trace + ) +' + # Verify that the daemon has shutdown. Spin a few seconds to # make the test a little more robust during CI testing. # From 50819d87d863e9f5585119d11d60595aef8cf7f7 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 24 Aug 2026 11:50:10 -0500 Subject: [PATCH 3/6] fsmonitor: preserve event chronology across compaction Retired batches are collapsed into a path-only overflow set. That keeps old indexes complete, but it loses the sequence in which each path was last observed. A client that consumed an inode event can therefore see it again after another index compacts the batch list, causing repeated hard-link scans. Unpinned batches have a zero pinned time and are also eligible for compaction immediately despite the default grace period. Do not use unpinned batches as truncation boundaries. Record the newest original batch sequence for every overflow path, and filter overflow responses against the client's requested sequence. The normal batch walk remains unchanged; sequence lookups are confined to overflow responses. Cover both the default retention grace and the cross-index hard-link case. The latter persists a nonzero checkpoint, compacts through a private index, and verifies repeated canonical reads do not rescan or fall back to global invalidation. --- builtin/fsmonitor--daemon.c | 124 ++++++++++++++++++++++++++----- t/helper/test-fsmonitor-client.c | 27 +++++++ t/t7527-builtin-fsmonitor.sh | 120 ++++++++++++++++++++++++++++++ 3 files changed, 251 insertions(+), 20 deletions(-) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index 1ac2fcc5ebb1af..9ba1d77f30e657 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -42,7 +42,7 @@ static int fsmonitor_path_equal(const char *a, const char *b) return a == b; } -KHASH_INIT(fsmonitor_path_set, const char *, int, 0, +KHASH_INIT(fsmonitor_path_sequence, const char *, uint64_t, 1, fsmonitor_path_hash, fsmonitor_path_equal) /* @@ -436,7 +436,7 @@ struct fsmonitor_batch { const char **interned_paths; size_t nr, alloc; time_t pinned_time; - kh_fsmonitor_path_set_t *overflow_paths; + kh_fsmonitor_path_sequence_t *overflow_path_seqs; }; static struct fsmonitor_token_data *fsmonitor_new_token_data(void) @@ -536,7 +536,7 @@ void fsmonitor_batch__free_list(struct fsmonitor_batch *batch) * are interned, so we don't own them. We only own * the array. */ - kh_destroy_fsmonitor_path_set(batch->overflow_paths); + kh_destroy_fsmonitor_path_sequence(batch->overflow_path_seqs); free(batch->interned_paths); free(batch); @@ -570,13 +570,20 @@ static void fsmonitor_batch__combine(struct fsmonitor_batch *batch_dest, } static void fsmonitor_batch__add_overflow_path(struct fsmonitor_batch *batch, - const char *path) + const char *path, + uint64_t batch_seq_nr) { + khint_t pos; int added; - kh_put_fsmonitor_path_set(batch->overflow_paths, path, &added); - if (!added) + pos = kh_put_fsmonitor_path_sequence( + batch->overflow_path_seqs, path, &added); + if (!added) { + if (kh_value(batch->overflow_path_seqs, pos) < batch_seq_nr) + kh_value(batch->overflow_path_seqs, pos) = batch_seq_nr; return; + } + kh_value(batch->overflow_path_seqs, pos) = batch_seq_nr; ALLOC_GROW(batch->interned_paths, batch->nr + 1, batch->alloc); batch->interned_paths[batch->nr++] = path; @@ -598,7 +605,7 @@ static size_t fsmonitor_batch__compact_tail(struct fsmonitor_batch *batch, *input_nr = 0; for (item = batch; item; item = item->next) { *input_nr = st_add(*input_nr, item->nr); - if (item->overflow_paths) { + if (item->overflow_path_seqs) { overflow = item; break; } @@ -616,23 +623,26 @@ static size_t fsmonitor_batch__compact_tail(struct fsmonitor_batch *batch, for (k = 0; k < item->nr; k++) fsmonitor_batch__add_overflow_path( - overflow, item->interned_paths[k]); + overflow, item->interned_paths[k], + item->batch_seq_nr); } compacted.interned_paths = overflow->interned_paths; compacted.nr = overflow->nr; compacted.alloc = overflow->alloc; - compacted.overflow_paths = overflow->overflow_paths; + compacted.overflow_path_seqs = overflow->overflow_path_seqs; overflow->interned_paths = NULL; overflow->nr = overflow->alloc = 0; - overflow->overflow_paths = NULL; + overflow->overflow_path_seqs = NULL; } else { - compacted.overflow_paths = kh_init_fsmonitor_path_set(); + compacted.overflow_path_seqs = + kh_init_fsmonitor_path_sequence(); for (item = batch; item; item = item->next) { size_t k; for (k = 0; k < item->nr; k++) fsmonitor_batch__add_overflow_path( - &compacted, item->interned_paths[k]); + &compacted, item->interned_paths[k], + item->batch_seq_nr); } } @@ -640,7 +650,7 @@ static size_t fsmonitor_batch__compact_tail(struct fsmonitor_batch *batch, batch->interned_paths = compacted.interned_paths; batch->nr = compacted.nr; batch->alloc = compacted.alloc; - batch->overflow_paths = compacted.overflow_paths; + batch->overflow_path_seqs = compacted.overflow_path_seqs; return batch->nr; } @@ -653,9 +663,8 @@ static size_t fsmonitor_batch__compact_tail(struct fsmonitor_batch *batch, * particular, advancing a private GIT_INDEX_FILE does not advance .git/index. * We therefore cannot discard old paths just because one client asked for a * newer token. Instead, keep their deduplicated union in an overflow batch. - * Requests older than the overflow sequence may receive extra paths, but not - * miss any. A request at that sequence excludes the overflow batch and - * remains exact. + * Keep the newest original sequence number for each path so that clients do + * not receive events that they consumed before their requested checkpoint. * * However, multiple commands may be talking to the daemon concurrently * or perform a slow command, so a little "token skew" is possible. @@ -696,7 +705,7 @@ static struct fsmonitor_batch *with_lock__truncate_old_batches( for (batch = batch_marker->next; batch; batch = batch->next) { time_t t; - if (batch->overflow_paths) + if (!batch->pinned_time || batch->overflow_path_seqs) continue; t = batch->pinned_time + truncate_delay_seconds; @@ -830,6 +839,18 @@ static int fsmonitor_parse_client_token(const char *buf_token, return 0; } +static void fsmonitor_reply_overflow_paths( + const struct fsmonitor_batch *batch, + uint64_t requested_oldest_seq_nr, + int hardlink_aware_query, + ipc_server_reply_cb *reply, + struct ipc_server_reply_data *reply_data, + struct strset *shown, + struct strbuf *payload, + uint64_t *total_response_len, + intmax_t *count, + intmax_t *duplicates); + static int do_handle_client(struct fsmonitor_daemon_state *state, const char *command, ipc_server_reply_cb *reply, @@ -1048,13 +1069,14 @@ static int do_handle_client(struct fsmonitor_daemon_state *state, } else if (requested_oldest_seq_nr < token_data->batch_tail->batch_seq_nr && - !token_data->batch_tail->overflow_paths) { + !token_data->batch_tail->overflow_path_seqs) { /* * The client wants older events than we have for * this token_id. A normal tail means that the end * of our batch list was truncated and we cannot * give the client a complete snapshot. An overflow - * tail conservatively contains all older paths. + * tail retains the latest original sequence for each + * older path. */ trace_printf_key(&trace_fsmonitor, "client requested truncated data"); @@ -1103,7 +1125,8 @@ static int do_handle_client(struct fsmonitor_daemon_state *state, */ strset_init_with_options(&shown, NULL, 0); for (batch = batch_head; - batch && batch->batch_seq_nr > requested_oldest_seq_nr; + batch && batch->batch_seq_nr > requested_oldest_seq_nr && + !batch->overflow_path_seqs; batch = batch->next) { size_t k; @@ -1138,6 +1161,13 @@ static int do_handle_client(struct fsmonitor_daemon_state *state, } } } + if (batch && batch->batch_seq_nr > requested_oldest_seq_nr) { + fsmonitor_reply_overflow_paths( + batch, requested_oldest_seq_nr, hardlink_aware_query, + reply, reply_data, &shown, &payload, + &total_response_len, &count, &duplicates); + batch = batch->next; + } if (payload.len) { reply(reply_data, payload.buf, payload.len); @@ -1195,6 +1225,60 @@ static int do_handle_client(struct fsmonitor_daemon_state *state, return 0; } +static void fsmonitor_reply_overflow_paths( + const struct fsmonitor_batch *batch, + uint64_t requested_oldest_seq_nr, + int hardlink_aware_query, + ipc_server_reply_cb *reply, + struct ipc_server_reply_data *reply_data, + struct strset *shown, + struct strbuf *payload, + uint64_t *total_response_len, + intmax_t *count, + intmax_t *duplicates) +{ + size_t k; + + if (!batch->overflow_path_seqs) + BUG("expected an overflow batch"); + + for (k = 0; k < batch->nr; k++) { + const char *s = batch->interned_paths[k]; + khint_t pos = kh_get_fsmonitor_path_sequence( + batch->overflow_path_seqs, s); + size_t s_len; + + if (pos == kh_end(batch->overflow_path_seqs)) + BUG("overflow path is missing its sequence"); + if (kh_value(batch->overflow_path_seqs, pos) <= + requested_oldest_seq_nr) + continue; + + if (!hardlink_aware_query && + starts_with(s, FSMONITOR_PATH_HARDLINK_INODE_PREFIX)) + s = FSMONITOR_PATH_GLOBAL_INVALIDATE; + + if (!strset_add(shown, s)) + (*duplicates)++; + else { + trace_printf_key(&trace_fsmonitor, + "send[%"PRIuMAX"]: %s", *count, s); + + /* Each path gets written with a trailing NUL */ + s_len = strlen(s) + 1; + + if (payload->len + s_len >= LARGE_PACKET_DATA_MAX) { + reply(reply_data, payload->buf, payload->len); + *total_response_len += payload->len; + strbuf_reset(payload); + } + + strbuf_add(payload, s, s_len); + (*count)++; + } + } +} + static ipc_server_application_cb handle_client; static int handle_client(void *data, diff --git a/t/helper/test-fsmonitor-client.c b/t/helper/test-fsmonitor-client.c index 653d09455382bb..a3f9eb0bbe7bf2 100644 --- a/t/helper/test-fsmonitor-client.c +++ b/t/helper/test-fsmonitor-client.c @@ -64,6 +64,29 @@ static int do_send_query(const char *token) return 0; } +/* + * Send a protocol-v2 token without the capability and worktree-binding + * prefix used by current clients. This models an older client that does + * not understand hard-link inode events. + */ +static int do_send_legacy_query(const char *token) +{ + struct strbuf answer = STRBUF_INIT; + int ret; + + if (!token || !*token) + token = get_token_from_index(); + + ret = fsmonitor_ipc__send_command(token, &answer); + if (ret < 0) + die("could not query fsmonitor--daemon"); + + write_in_full(1, answer.buf, answer.len); + strbuf_release(&answer); + + return 0; +} + /* * Send a "flush" command to the `git-fsmonitor--daemon` (if running) * and tell it to flush its cache. @@ -221,6 +244,7 @@ int cmd__fsmonitor_client(int argc, const char **argv) const char * const fsmonitor_client_usage[] = { "test-tool fsmonitor-client query []", + "test-tool fsmonitor-client query-legacy []", "test-tool fsmonitor-client flush", "test-tool fsmonitor-client record-watch-limit", "test-tool fsmonitor-client hammer [] [] []", @@ -249,6 +273,9 @@ int cmd__fsmonitor_client(int argc, const char **argv) if (!strcmp(subcmd, "query")) return !!do_send_query(token); + if (!strcmp(subcmd, "query-legacy")) + return !!do_send_legacy_query(token); + if (!strcmp(subcmd, "flush")) return !!do_send_flush(); diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index 6d0fc8a26cacd0..e95e7931ed0642 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -239,6 +239,41 @@ test_expect_success MACOS 'fall back when a delayed FSEvents cookie stays late' test_must_be_empty error ' +test_expect_success MACOS 'fresh unpinned batches honor the retention grace' ' + test_when_finished "stop_daemon_delete_repo test_fresh_history" && + + git init test_fresh_history && + ( + cd test_fresh_history && + printf "target/\\n" >.gitignore && + printf "a\\n" >tracked-a && + printf "b\\n" >tracked-b && + git add .gitignore tracked-a tracked-b && + git commit -m base && + sane_unset GIT_TEST_FSMONITOR_TRUNCATE_DELAY_SECONDS && + start_daemon --tf "$PWD/../fresh-history.trace" && + git config core.fsmonitor true && + git config core.untrackedCache true && + git status --porcelain=v2 >.git/prime-1 && + git status --porcelain=v2 >.git/prime-2 && + test_must_be_empty .git/prime-1 && + test_must_be_empty .git/prime-2 && + mkdir target && + for i in $(test_seq 1 5250) + do + printf "x\\n" >"target/ignored-$i" || return 1 + done && + test-tool fsmonitor-client query >../fresh-history.response && + perl -0ne '\''$nr++; END { print "$nr\n" }'\'' \ + <../fresh-history.response >../fresh-history.count && + test "$(cat ../fresh-history.count)" -gt 1025 && + git status --porcelain=v2 >.git/after-burst && + printf "new\\n" >>tracked-a && + git status --porcelain=v2 >.git/after-event && + test_grep ! "Compact: batch" ../fresh-history.trace + ) +' + test_expect_success MACOS 'private index cannot prune canonical index history' ' test_when_finished "stop_daemon_delete_repo test_index_history" && test_when_finished "rm -f private-index" && @@ -290,6 +325,91 @@ test_expect_success MACOS 'private index cannot prune canonical index history' ' ) ' +test_expect_success MACOS,HARDLINKS \ + 'compaction does not replay consumed hardlink events' ' + test_when_finished "stop_daemon_delete_repo test_hardlink_history" && + test_when_finished "rm -f hardlink-private-index" && + + git init test_hardlink_history && + ( + cd test_hardlink_history && + printf "target/\\n" >.gitignore && + printf "a\\n" >tracked-a && + printf "b\\n" >tracked-b && + git add .gitignore tracked-a tracked-b && + git commit -m base && + git config core.untrackedCache true && + git config core.trustctime false && + git config core.checkStat minimal && + ( + GIT_TEST_FSMONITOR_TRUNCATE_DELAY_SECONDS=0 && + export GIT_TEST_FSMONITOR_TRUNCATE_DELAY_SECONDS && + start_daemon --tf "$PWD/../hardlink-history.trace" + ) && + git config core.fsmonitor true && + git status --porcelain=v2 >.git/prime-1 && + git status --porcelain=v2 >.git/prime-2 && + test_must_be_empty .git/prime-1 && + test_must_be_empty .git/prime-2 && + mkdir target && + printf "AAAA\\n" >target/object && + ln target/object target/object-link && + printf "BBBB\\n" >target/object-link && + GIT_TRACE2_EVENT="$PWD/.git/consume.trace" \ + git status --porcelain=v2 >.git/consume && + test_must_be_empty .git/consume && + test_trace2_data fsmonitor apply/hardlink-index-scan 1 \ + <.git/consume.trace && + git update-index --refresh --force-write-index && + test-tool dump-fsmonitor >.git/checkpoint && + checkpoint=$(sed -n "s/^fsmonitor last update //p" \ + .git/checkpoint) && + test -n "$checkpoint" && + test "${checkpoint##*:}" -gt 0 && + cp .git/index .git/index.before && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TRACE2_EVENT="$PWD/.git/control.trace" \ + git status --porcelain=v2 >.git/control && + test_must_be_empty .git/control && + ! test_trace2_data fsmonitor apply/hardlink-index-scan 1 \ + <.git/control.trace && + cp .git/index ../hardlink-private-index && + grep -c "event: //inode:" ../hardlink-history.trace \ + >.git/inodes.before && + for i in $(test_seq 1 8) + do + if test $((i % 2)) -eq 0 + then + printf "private-%s\\n" "$i" >>tracked-a + else + printf "private-%s\\n" "$i" >>tracked-b + fi && + GIT_INDEX_FILE="$PWD/../hardlink-private-index" \ + git add -u || return 1 + done && + test_cmp .git/index.before .git/index && + grep -c "event: //inode:" ../hardlink-history.trace \ + >.git/inodes.after && + test_cmp .git/inodes.before .git/inodes.after && + test_grep "Compact: batch" ../hardlink-history.trace && + for i in $(test_seq 1 5) + do + GIT_OPTIONAL_LOCKS=0 \ + GIT_TRACE2_EVENT="$PWD/.git/repeat-$i.trace" \ + git status --porcelain=v2 \ + >.git/repeat-$i || return 1 + ! test_trace2_data fsmonitor apply/hardlink-index-scan 1 \ + <.git/repeat-$i.trace || return 1 + done && + test_cmp .git/index.before .git/index && + test-tool fsmonitor-client query-legacy \ + --token "$checkpoint" >.git/legacy && + nul_to_q <.git/legacy >.git/legacy-q && + test_grep ! "Q/Q" .git/legacy-q && + test_grep ! "Q//Q" .git/legacy-q + ) +' + # Verify that the daemon has shutdown. Spin a few seconds to # make the test a little more robust during CI testing. # From 2f97399b76cba7c33ef5c0911f472462939be219 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 24 Aug 2026 11:50:41 -0500 Subject: [PATCH 4/6] t7527: query delayed cookies with valid v2 tokens The delayed-cookie tests send the v1 timestamp token "0" and only check that the response is nonempty. Both recovery and fallback can satisfy that assertion with the same trivial response, so the tests do not distinguish a rescued cookie from a token-generation reset. Send a deterministic valid v2 token instead. Verify that the 1200ms case preserves its token generation without a global invalidation, while the 2500ms case changes generation and sends the fallback invalidation. --- t/t7527-builtin-fsmonitor.sh | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index e95e7931ed0642..ea60a193d6a68a 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -202,15 +202,20 @@ test_expect_success MACOS 'rescue a delayed FSEvents cookie after timeout' ' git init test_delayed_cookie && ( GIT_TEST_FSMONITOR_COOKIE_DELAY_MS=1200 && - GIT_TRACE_FSMONITOR="$PWD/delayed-cookie.trace" && - export GIT_TEST_FSMONITOR_COOKIE_DELAY_MS GIT_TRACE_FSMONITOR && - git -C test_delayed_cookie fsmonitor--daemon start \ - --start-timeout=10 + export GIT_TEST_FSMONITOR_COOKIE_DELAY_MS && + start_daemon -C test_delayed_cookie \ + --tf "$PWD/delayed-cookie.trace" --tk true ) && + token="builtin:${fsmonitor_cookie_token_prefix}test_00000001:0" && test-tool -C test_delayed_cookie fsmonitor-client query \ - --token 0 >actual 2>error && - test_file_not_empty actual && + --token "$token" >actual 2>error && + nul_to_q actual-q && + response=$(sed -n "s/Q.*//p" actual-q) && + test "${response%:*}" = "${token%:*}" && + test_grep "^builtin:.*Q$" actual-q && + test_grep ! "Q/Q" actual-q && + test_grep ! "Q//Q" actual-q && test_grep "cookie_wait: requesting FSEvents flush after initial timeout" \ delayed-cookie.trace && test_grep "cookie-seen:" delayed-cookie.trace && @@ -224,15 +229,18 @@ test_expect_success MACOS 'fall back when a delayed FSEvents cookie stays late' git init test_lost_cookie && ( GIT_TEST_FSMONITOR_COOKIE_DELAY_MS=2500 && - GIT_TRACE_FSMONITOR="$PWD/lost-cookie.trace" && - export GIT_TEST_FSMONITOR_COOKIE_DELAY_MS GIT_TRACE_FSMONITOR && - git -C test_lost_cookie fsmonitor--daemon start \ - --start-timeout=10 + export GIT_TEST_FSMONITOR_COOKIE_DELAY_MS && + start_daemon -C test_lost_cookie \ + --tf "$PWD/lost-cookie.trace" --tk true ) && + token="builtin:${fsmonitor_cookie_token_prefix}test_00000001:0" && test-tool -C test_lost_cookie fsmonitor-client query \ - --token 0 >actual 2>error && - test_file_not_empty actual && + --token "$token" >actual 2>error && + nul_to_q actual-q && + response=$(sed -n "s/Q.*//p" actual-q) && + test "${response%:*}" != "${token%:*}" && + test_grep "Q/Q$" actual-q && test_grep "cookie_wait: requesting FSEvents flush after initial timeout" \ lost-cookie.trace && test_grep "cookie_wait timed out$" lost-cookie.trace && From 82eb070b0a64380cd7015caeb074994bc3a43358 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 24 Aug 2026 16:26:00 -0500 Subject: [PATCH 5/6] merge: preserve clean status proofs for non-ff merges 215845a7ad (fsmonitor: preserve authenticated proofs across ordinary commands, 2026-08-15) enabled the clean-status history handoff for merges, but excluded invocations where fast_forward was FF_NO. Requested merge topology does not determine whether the resulting index is semantically safe. A clean non-fast-forward merge can carry the same authenticated FSUC/FSCF state as a fast-forward merge. As a result, --no-ff, --no-ff --no-commit, and merge.ff=false all dropped FSUC and reduced the FSCF flags from 15 to 9 after a clean merge. Each subsequent read-only status invalidated the external history and rescanned the semantic manifest. Enable the handoff for every merge using the canonical index. Conflict handling still invalidates unsafe proofs, and explicit alternate indexes remain excluded. Cover all three non-fast-forward forms, repeated read-only status calls, conflicts, and alternate indexes. --- builtin/merge.c | 2 +- t/t7519-status-fsmonitor.sh | 131 ++++++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 1 deletion(-) diff --git a/builtin/merge.c b/builtin/merge.c index c7e8a31208a386..1d5de7f549dbc2 100644 --- a/builtin/merge.c +++ b/builtin/merge.c @@ -1472,7 +1472,7 @@ int cmd_merge(int argc, goto done; } - if (fast_forward != FF_NO && !getenv(INDEX_ENVIRONMENT) && + if (!getenv(INDEX_ENVIRONMENT) && !clean_status_config_read_repository(the_repository, &clean_digest)) { clean_status_enable_external_history(the_repository); clean_status_set_config_digest(the_repository, &clean_digest); diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 9454c11695077f..f511993452c078 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -2518,6 +2518,137 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ ) ' +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'clean non-fast-forward merges preserve authenticated worktree proofs' ' + test_when_finished "rm -rf clean-no-ff-proof-*" && + for mode in cli no-commit config + do + repo=clean-no-ff-proof-$mode && + test_create_repo "$repo" && + ( + cd "$repo" && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base base && + primary=$(git symbolic-ref --short HEAD) && + git switch -c side && + test_commit topic topic && + git switch "$primary" && + test_commit primary primary && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_INDEX_FILE="$PWD/.git/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_fsmonitor_full_proof .git/index paired && + case "$mode" in + cli) + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCCCCCC \ + git merge --no-ff --no-edit side + ;; + no-commit) + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCCCCCC \ + git merge --no-ff --no-commit side + ;; + config) + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCCCCCC \ + git -c merge.ff=false merge --no-edit side + ;; + esac && + test_fsmonitor_full_proof .git/index paired && + cp .git/index .git/readonly.index && + for run in 1 2 3 + do + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status-$run.trace" \ + git status --porcelain=v2 \ + >.git/status-$run && + test_cmp_bin .git/readonly.index .git/index && + ! test_trace2_data fsmonitor \ + history/external-proof-invalidated 1 \ + <.git/status-$run.trace && + ! have_t2_data_event fsmonitor \ + semantic/manifest-scan-count \ + <.git/status-$run.trace && + if test "$mode" = no-commit + then + test_grep "^1 A\\. .* topic$" \ + .git/status-$run + else + test_must_be_empty .git/status-$run + fi || return 1 + done + ) || return 1 + done +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'non-fast-forward conflicts and alternate indexes fail closed' ' + test_when_finished "rm -rf no-ff-alt-proof no-ff-conflict-proof" && + test_create_repo no-ff-alt-proof && + ( + cd no-ff-alt-proof && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base base && + primary=$(git symbolic-ref --short HEAD) && + git switch -c side && + test_commit topic topic && + git switch "$primary" && + test_commit primary primary && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_INDEX_FILE="$PWD/.git/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_fsmonitor_full_proof .git/index paired && + cp .git/index .git/alternate.index && + GIT_INDEX_FILE="$PWD/.git/alternate.index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCCCCCC \ + git merge --no-ff --no-commit side && + ! test_fsmonitor_full_proof .git/alternate.index paired \ + 2>.git/alternate.proof && + test_grep ! FSUC .git/alternate.index + ) && + test_create_repo no-ff-conflict-proof && + ( + cd no-ff-conflict-proof && + sane_unset GIT_TEST_SPLIT_INDEX && + test_write_lines base >tracked && + git add tracked && + git commit -m base && + primary=$(git symbolic-ref --short HEAD) && + git switch -c side && + test_write_lines side >tracked && + git commit -am side && + git switch "$primary" && + test_write_lines primary >tracked && + git commit -am primary && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_INDEX_FILE="$PWD/.git/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_fsmonitor_full_proof .git/index paired && + test_must_fail env \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCCCCCC \ + git merge --no-ff side && + ! test_fsmonitor_full_proof .git/index paired \ + 2>.git/conflict.proof && + test_grep ! FSUC .git/index && + git ls-files -u >.git/unmerged && + test_file_not_empty .git/unmerged + ) +' + test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ 'full status durably repairs missing mixed-writer index proofs' ' test_when_finished "rm -rf mixed-writer-missing-proofs" && From 7ccad93de82529373658d1e12ece28c6eb7e584e Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 24 Aug 2026 21:12:52 -0500 Subject: [PATCH 6/6] status: issue clean proof after repairing the index An exact clean status can repair a stale FSMonitor checkpoint or cached stat data while it scans. The repair requires an index write, so the existing issue path leaves no clean sidecar behind. Read-only callers then repeat the full scan until a second writable exact status publishes the proof. After the repair is written and resumable history is durable, install a sidecar bound to the rewritten index. Keep optional-lock-disabled commands read-only, preserve the literal exact-command restriction, and do not extend sidecar support to linked worktrees. Cover repeated read-only scans after a legacy daemon replacement, the single writable index repair in main and linked worktrees, and the next read-only sidecar hit in the main worktree. Keep option-bearing status commands ineligible for proof publication. --- .../technical/status-clean-proof.adoc | 9 +- builtin/commit.c | 27 +++- t/t7527-builtin-fsmonitor.sh | 120 ++++++++++++++++++ t/t7530-status-clean-sidecar.sh | 27 +++- 4 files changed, 170 insertions(+), 13 deletions(-) diff --git a/Documentation/technical/status-clean-proof.adoc b/Documentation/technical/status-clean-proof.adoc index fb5f24da58a133..66c2edaac297cd 100644 --- a/Documentation/technical/status-clean-proof.adoc +++ b/Documentation/technical/status-clean-proof.adoc @@ -83,9 +83,12 @@ checksum is accepted only when the pinned index is bound by the durable local-APFS identity used for raced-input checks. The sidecar is installed while the index lock remains held and after -the pinned index is rechecked. Status then rolls back the index lock, so -issuing a sidecar does not itself rewrite the index. With optional locks -disabled, status does not issue a sidecar. +the pinned index is rechecked. If the exact query first has to repair +the index's file system monitor checkpoint, status writes that repair, +refreshes the resumable history checkpoint, and then installs a proof +bound to the rewritten index. Status rolls back the lock used for the +sidecar itself, so issuing a sidecar does not itself rewrite the index. +With optional locks disabled, status does not issue a sidecar. Validation and races -------------------- diff --git a/builtin/commit.c b/builtin/commit.c index 5e8c425fa13f1d..97a9fec1a89476 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1906,6 +1906,8 @@ struct repository *repo UNUSED) int repository_inputs_changed = 0; int sidecar_provider_reset = 0; int reissue_after_write = 0; + int issue_exact_after_write = 0; + int exact_after_write_candidate = 0; int save_history_after_write = 0; int deferred_scoped_history = 0; int guarded_scoped_history_source = 0; @@ -2204,6 +2206,17 @@ struct repository *repo UNUSED) reissue_clean_sidecar && preserve_entry_changes && !external_restored && !persist_restored_boundary && !hook_exists(the_repository, "post-index-change"); + /* + * An exact query may have completed a clean scan while repairing + * the provider checkpoint or cached stat data. Bind its proof to + * the repaired index, after the resumable history is durable. + */ + exact_after_write_candidate = exact_clean_query && + preserve_entry_changes && !external_restored && + !persist_restored_boundary && + !hook_exists(the_repository, "post-index-change"); + issue_exact_after_write = + exact_after_write_candidate && external_saved; if (the_repository->index->fsmonitor_legacy_untracked_fallback && !preserve_entry_changes && !external_saved) { @@ -2251,17 +2264,23 @@ struct repository *repo UNUSED) !hook_exists(the_repository, "post-index-change") && repo_hold_locked_index(the_repository, &index_lock, 0) >= 0) { if (clean_status_save_external_history( - the_repository->index)) + the_repository->index)) { trace2_data_intmax("fsmonitor", the_repository, "history/external-postwrite-stored", 1); + if (exact_after_write_candidate) + issue_exact_after_write = 1; + } rollback_lock_file(&index_lock); } - if (reissue_after_write && + if ((reissue_after_write || issue_exact_after_write) && repo_hold_locked_index(the_repository, &index_lock, 0) >= 0) { if (clean_status_issue_sidecar( - &s, &clean_digest, &index_lock, 1)) + &s, &clean_digest, &index_lock, + reissue_after_write)) trace2_data_intmax("status", the_repository, - "clean-proof/postwrite-reissued", 1); + reissue_after_write ? + "clean-proof/postwrite-reissued" : + "clean-proof/postwrite-issued", 1); else rollback_lock_file(&index_lock); } diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index ea60a193d6a68a..027e76eb263572 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -88,6 +88,13 @@ stop_daemon_delete_repo () { rm -rf $1 } +stop_daemon_delete_linked_repo () { + r=$1 && + wt=$2 && + { maybe_timeout 30 git -C "$wt" fsmonitor--daemon stop 2>/dev/null || :; } && + rm -rf "$r" "$wt" +} + start_daemon () { r= tf= t2= tk= && @@ -2051,6 +2058,119 @@ test_expect_success 'bound query replaces a legacy daemon' ' ) ' +test_expect_success MACOS \ + 'read-only legacy upgrade waits for one writable exact repair' ' + test_when_finished \ + "stop_daemon_delete_repo legacy-read-only-upgrade" && + test_create_repo legacy-read-only-upgrade && + ( + cd legacy-read-only-upgrade && + sane_unset GIT_TEST_SPLIT_INDEX && + git config core.fsmonitor false && + for i in $(test_seq 1 64) + do + test_write_lines "$i" >"tracked-$i" || return 1 + done && + git add . && + git commit -qm base && + git config core.preloadIndex false && + git config core.untrackedCache true && + git config core.fsmonitor true && + ipc_path=$(git rev-parse --path-format=absolute \ + --git-path fsmonitor--daemon.ipc) && + test-tool simple-ipc start-daemon \ + --name="$ipc_path" --threads=1 \ + --fsmonitor-capability-superset && + git status --porcelain=v2 --untracked-files=normal \ + --no-ahead-behind >.git/prime && + test_must_be_empty .git/prime && + test_path_is_missing .git/index.csts && + test-tool simple-ipc stop-daemon --name="$ipc_path" && + test-tool simple-ipc start-daemon \ + --name="$ipc_path" --threads=1 --fsmonitor-legacy && + cp .git/index .git/index.before && + + for label in first repeat + do + GIT_OPTIONAL_LOCKS=0 \ + GIT_TRACE2_EVENT="$PWD/.git/$label.trace" \ + git status --porcelain=v2 >.git/$label && + test_must_be_empty .git/$label && + test_cmp_bin .git/index.before .git/index && + test_trace2_data index refresh/sum_lstat 64 \ + <.git/$label.trace || return 1 + done && + test_trace2_data fsm_client query/incompatible-daemon 1 \ + <.git/first.trace && + test_path_is_missing .git/index.csts && + { git fsmonitor--daemon stop 2>/dev/null || :; } && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + GIT_TRACE2_EVENT="$PWD/.git/repair.trace" \ + git status --porcelain=v2 >.git/repair && + test_must_be_empty .git/repair && + ! test_cmp_bin .git/index.before .git/index && + test-tool dump-fsmonitor >.git/fsmonitor && + test_grep "fsmonitor last update builtin:test:1" \ + .git/fsmonitor + ) +' + +test_expect_success MACOS \ + 'linked worktree legacy upgrade uses its writable index repair' ' + test_when_finished \ + "stop_daemon_delete_linked_repo legacy-linked legacy-linked-wt" && + test_create_repo legacy-linked && + ( + cd legacy-linked && + git config core.fsmonitor false && + for i in $(test_seq 1 32) + do + test_write_lines "$i" >"tracked-$i" || return 1 + done && + git add . && + git commit -qm base && + git worktree add -q -b linked ../legacy-linked-wt + ) && + git -C legacy-linked config core.preloadIndex false && + git -C legacy-linked config core.untrackedCache true && + git -C legacy-linked config core.fsmonitor true && + gitdir=$(git -C legacy-linked-wt rev-parse --absolute-git-dir) && + ipc_path=$(git -C legacy-linked-wt rev-parse --path-format=absolute \ + --git-path fsmonitor--daemon.ipc) && + test-tool simple-ipc start-daemon \ + --name="$ipc_path" --threads=1 \ + --fsmonitor-capability-superset && + git -C legacy-linked-wt status --porcelain=v2 \ + --untracked-files=normal --no-ahead-behind >linked.prime && + test_must_be_empty linked.prime && + test-tool simple-ipc stop-daemon --name="$ipc_path" && + test-tool simple-ipc start-daemon \ + --name="$ipc_path" --threads=1 --fsmonitor-legacy && + cp "$gitdir/index" linked.index.before && + for label in first repeat + do + GIT_OPTIONAL_LOCKS=0 \ + GIT_TRACE2_EVENT="$PWD/linked-$label.trace" \ + git -C legacy-linked-wt status --porcelain=v2 \ + >linked-$label && + test_must_be_empty linked-$label && + test_cmp_bin linked.index.before "$gitdir/index" && + test_trace2_data index refresh/sum_lstat 32 \ + /dev/null || :; } && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + GIT_TRACE2_EVENT="$PWD/linked-repair.trace" \ + git -C legacy-linked-wt status --porcelain=v2 >linked-repair && + test_must_be_empty linked-repair && + ! test_cmp_bin linked.index.before "$gitdir/index" && + test_path_is_missing "$gitdir/index.csts" && + test-tool -C legacy-linked-wt dump-fsmonitor >linked.fsmonitor && + test_grep "fsmonitor last update builtin:test:1" \ + linked.fsmonitor +' + test_expect_success MACOS 'bound query upgrades stale directory event daemon' ' test_when_finished \ "stop_daemon_delete_repo directory-daemon-upgrade" && diff --git a/t/t7530-status-clean-sidecar.sh b/t/t7530-status-clean-sidecar.sh index 03fbcdc68a2246..c67531a7a5fcd0 100755 --- a/t/t7530-status-clean-sidecar.sh +++ b/t/t7530-status-clean-sidecar.sh @@ -1727,7 +1727,7 @@ test_expect_success DURABLE_FSMONITOR \ ' test_expect_success DURABLE_FSMONITOR \ - 'exact status persists stat repairs before a sidecar' ' + 'exact status installs a sidecar after stat repairs' ' test_when_finished "stop_daemon external-stat-exact" && setup_repo external-stat-exact && git -C external-stat-exact config core.autocrlf false && @@ -1740,21 +1740,32 @@ test_expect_success DURABLE_FSMONITOR \ test_must_be_empty actual && test_trace2_data fsmonitor history/external-stored 1 \ actual && test_must_be_empty actual && - test_trace2_data fsmonitor history/external-stored 1 \ + test_trace2_data status clean-proof/hit 1 \ actual && test_must_be_empty actual && test_path_is_missing sidecar-shape/.git/index.csts && + bulk_status -C sidecar-shape status --porcelain=v2 \ + --untracked-files=normal --no-ahead-behind >actual && + test_must_be_empty actual && + test_path_is_missing sidecar-shape/.git/index.csts && test_env GIT_TRACE2_EVENT="$PWD/shape-branch.trace" \ bulk_status -C sidecar-shape \