diff --git a/Documentation/technical/status-clean-proof.adoc b/Documentation/technical/status-clean-proof.adoc index fb5f24da58a133..66c2edaac297cd 100644 --- a/Documentation/technical/status-clean-proof.adoc +++ b/Documentation/technical/status-clean-proof.adoc @@ -83,9 +83,12 @@ checksum is accepted only when the pinned index is bound by the durable local-APFS identity used for raced-input checks. The sidecar is installed while the index lock remains held and after -the pinned index is rechecked. Status then rolls back the index lock, so -issuing a sidecar does not itself rewrite the index. With optional locks -disabled, status does not issue a sidecar. +the pinned index is rechecked. If the exact query first has to repair +the index's file system monitor checkpoint, status writes that repair, +refreshes the resumable history checkpoint, and then installs a proof +bound to the rewritten index. Status rolls back the lock used for the +sidecar itself, so issuing a sidecar does not itself rewrite the index. +With optional locks disabled, status does not issue a sidecar. Validation and races -------------------- diff --git a/builtin/commit.c b/builtin/commit.c index 5e8c425fa13f1d..97a9fec1a89476 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1906,6 +1906,8 @@ struct repository *repo UNUSED) int repository_inputs_changed = 0; int sidecar_provider_reset = 0; int reissue_after_write = 0; + int issue_exact_after_write = 0; + int exact_after_write_candidate = 0; int save_history_after_write = 0; int deferred_scoped_history = 0; int guarded_scoped_history_source = 0; @@ -2204,6 +2206,17 @@ struct repository *repo UNUSED) reissue_clean_sidecar && preserve_entry_changes && !external_restored && !persist_restored_boundary && !hook_exists(the_repository, "post-index-change"); + /* + * An exact query may have completed a clean scan while repairing + * the provider checkpoint or cached stat data. Bind its proof to + * the repaired index, after the resumable history is durable. + */ + exact_after_write_candidate = exact_clean_query && + preserve_entry_changes && !external_restored && + !persist_restored_boundary && + !hook_exists(the_repository, "post-index-change"); + issue_exact_after_write = + exact_after_write_candidate && external_saved; if (the_repository->index->fsmonitor_legacy_untracked_fallback && !preserve_entry_changes && !external_saved) { @@ -2251,17 +2264,23 @@ struct repository *repo UNUSED) !hook_exists(the_repository, "post-index-change") && repo_hold_locked_index(the_repository, &index_lock, 0) >= 0) { if (clean_status_save_external_history( - the_repository->index)) + the_repository->index)) { trace2_data_intmax("fsmonitor", the_repository, "history/external-postwrite-stored", 1); + if (exact_after_write_candidate) + issue_exact_after_write = 1; + } rollback_lock_file(&index_lock); } - if (reissue_after_write && + if ((reissue_after_write || issue_exact_after_write) && repo_hold_locked_index(the_repository, &index_lock, 0) >= 0) { if (clean_status_issue_sidecar( - &s, &clean_digest, &index_lock, 1)) + &s, &clean_digest, &index_lock, + reissue_after_write)) trace2_data_intmax("status", the_repository, - "clean-proof/postwrite-reissued", 1); + reissue_after_write ? + "clean-proof/postwrite-reissued" : + "clean-proof/postwrite-issued", 1); else rollback_lock_file(&index_lock); } diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index 1c53a5af4dd6df..9ba1d77f30e657 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -8,12 +8,14 @@ #include "environment.h" #include "gettext.h" #include "parse-options.h" + #include "fsmonitor-ll.h" #include "fsmonitor-ipc.h" #include "fsmonitor-settings.h" #include "compat/fsmonitor/fsm-health.h" #include "compat/fsmonitor/fsm-listen.h" #include "fsmonitor--daemon.h" +#include "khash.h" #include "simple-ipc.h" #include "strmap.h" @@ -30,6 +32,19 @@ static const char * const builtin_fsmonitor__daemon_usage[] = { }; #ifdef HAVE_FSMONITOR_DAEMON_BACKEND +static khint_t fsmonitor_path_hash(const char *path) +{ + return memhash(&path, sizeof(path)); +} + +static int fsmonitor_path_equal(const char *a, const char *b) +{ + return a == b; +} + +KHASH_INIT(fsmonitor_path_sequence, const char *, uint64_t, 1, + fsmonitor_path_hash, fsmonitor_path_equal) + /* * Global state loaded from config. */ @@ -238,6 +253,37 @@ static enum fsmonitor_cookie_item_result with_lock__wait_for_cookie( &state->main_lock, &ts); if (err == ETIMEDOUT && cookie->result == FCIR_INIT) { +#ifdef __APPLE__ + struct timeval rescue_now; + + /* + * FSEvents may be healthy but late enough that its normal + * delivery misses our bounded wait. Flush only after that + * wait expires, so successful queries pay no extra cost. + * The asynchronous flush cannot block on the listener callback, + * which needs main_lock to publish the cookie. + */ + trace_printf_key(&trace_fsmonitor, + "cookie_wait: requesting FSEvents flush after initial timeout"); + fsm_listen__flush_async(state); + + /* + * Give the listener one more bounded interval to deliver and + * publish the cookie rather than falling back to a full index + * scan. A broken provider still reaches the existing error + * path instead of hanging a client indefinitely. + */ + gettimeofday(&rescue_now, NULL); + ts.tv_sec = rescue_now.tv_sec + 1; + ts.tv_nsec = rescue_now.tv_usec * 1000; + err = 0; + while (cookie->result == FCIR_INIT && !err) + err = pthread_cond_timedwait(&state->cookies_cond, + &state->main_lock, + &ts); +#endif + } + if (err == ETIMEDOUT && cookie->result == FCIR_INIT) { trace_printf_key(&trace_fsmonitor, "cookie_wait timed out"); cookie->result = FCIR_ERROR; @@ -390,6 +436,7 @@ struct fsmonitor_batch { const char **interned_paths; size_t nr, alloc; time_t pinned_time; + kh_fsmonitor_path_sequence_t *overflow_path_seqs; }; static struct fsmonitor_token_data *fsmonitor_new_token_data(void) @@ -489,6 +536,7 @@ void fsmonitor_batch__free_list(struct fsmonitor_batch *batch) * are interned, so we don't own them. We only own * the array. */ + kh_destroy_fsmonitor_path_sequence(batch->overflow_path_seqs); free(batch->interned_paths); free(batch); @@ -521,16 +569,102 @@ static void fsmonitor_batch__combine(struct fsmonitor_batch *batch_dest, batch_src->interned_paths[k]; } +static void fsmonitor_batch__add_overflow_path(struct fsmonitor_batch *batch, + const char *path, + uint64_t batch_seq_nr) +{ + khint_t pos; + int added; + + pos = kh_put_fsmonitor_path_sequence( + batch->overflow_path_seqs, path, &added); + if (!added) { + if (kh_value(batch->overflow_path_seqs, pos) < batch_seq_nr) + kh_value(batch->overflow_path_seqs, pos) = batch_seq_nr; + return; + } + kh_value(batch->overflow_path_seqs, pos) = batch_seq_nr; + + ALLOC_GROW(batch->interned_paths, batch->nr + 1, batch->alloc); + batch->interned_paths[batch->nr++] = path; +} + +/* + * Collapse this batch and everything older than it into one deduplicated + * overflow batch. Every path is interned, so pointer identity is sufficient. + * + * Keep the set with the overflow batch. Future compactions then hash only + * newly retired paths instead of repeatedly rebuilding the complete set. + */ +static size_t fsmonitor_batch__compact_tail(struct fsmonitor_batch *batch, + size_t *input_nr) +{ + struct fsmonitor_batch compacted = { 0 }; + struct fsmonitor_batch *item, *overflow = NULL; + + *input_nr = 0; + for (item = batch; item; item = item->next) { + *input_nr = st_add(*input_nr, item->nr); + if (item->overflow_path_seqs) { + overflow = item; + break; + } + } + + if (overflow) { + /* + * Reuse the persistent set and array from the prior overflow + * batch. Only the newly retired paths need a lookup. + */ + if (overflow->next) + BUG("overflow batch is not the batch tail"); + for (item = batch; item != overflow; item = item->next) { + size_t k; + + for (k = 0; k < item->nr; k++) + fsmonitor_batch__add_overflow_path( + overflow, item->interned_paths[k], + item->batch_seq_nr); + } + compacted.interned_paths = overflow->interned_paths; + compacted.nr = overflow->nr; + compacted.alloc = overflow->alloc; + compacted.overflow_path_seqs = overflow->overflow_path_seqs; + overflow->interned_paths = NULL; + overflow->nr = overflow->alloc = 0; + overflow->overflow_path_seqs = NULL; + } else { + compacted.overflow_path_seqs = + kh_init_fsmonitor_path_sequence(); + for (item = batch; item; item = item->next) { + size_t k; + + for (k = 0; k < item->nr; k++) + fsmonitor_batch__add_overflow_path( + &compacted, item->interned_paths[k], + item->batch_seq_nr); + } + } + + free(batch->interned_paths); + batch->interned_paths = compacted.interned_paths; + batch->nr = compacted.nr; + batch->alloc = compacted.alloc; + batch->overflow_path_seqs = compacted.overflow_path_seqs; + + return batch->nr; +} + /* * To keep the batch list from growing unbounded in response to filesystem - * activity, we try to truncate old batches from the end of the list as - * they become irrelevant. + * activity, collapse old batches from the end of the list after a delay. * - * We assume that the .git/index will be updated with the most recent token - * any time the index is updated. And future commands will only ask for - * recent changes *since* that new token. So as tokens advance into the - * future, older batch items will never be requested/needed. So we can - * truncate them without loss of functionality. + * A repository may have multiple durable indexes with different tokens. In + * particular, advancing a private GIT_INDEX_FILE does not advance .git/index. + * We therefore cannot discard old paths just because one client asked for a + * newer token. Instead, keep their deduplicated union in an overflow batch. + * Keep the newest original sequence number for each path so that clients do + * not receive events that they consumed before their requested checkpoint. * * However, multiple commands may be talking to the daemon concurrently * or perform a slow command, so a little "token skew" is possible. @@ -549,6 +683,7 @@ static void fsmonitor_batch__combine(struct fsmonitor_batch *batch_dest, * the official list so that the caller can free it after leaving the lock. */ #define MY_TIME_DELAY_SECONDS (5 * 60) /* seconds */ +static unsigned long truncate_delay_seconds = MY_TIME_DELAY_SECONDS; static struct fsmonitor_batch *with_lock__truncate_old_batches( struct fsmonitor_daemon_state *state, @@ -558,6 +693,7 @@ static struct fsmonitor_batch *with_lock__truncate_old_batches( const struct fsmonitor_batch *batch; struct fsmonitor_batch *remainder; + size_t input_nr, unique_nr; if (!batch_marker) return NULL; @@ -566,13 +702,13 @@ static struct fsmonitor_batch *with_lock__truncate_old_batches( batch_marker->batch_seq_nr, (uint64_t)batch_marker->pinned_time); - for (batch = batch_marker; batch; batch = batch->next) { + for (batch = batch_marker->next; batch; batch = batch->next) { time_t t; - if (!batch->pinned_time) /* an overflow batch */ + if (!batch->pinned_time || batch->overflow_path_seqs) continue; - t = batch->pinned_time + MY_TIME_DELAY_SECONDS; + t = batch->pinned_time + truncate_delay_seconds; if (t > batch_marker->pinned_time) /* too close to marker */ continue; @@ -582,9 +718,20 @@ static struct fsmonitor_batch *with_lock__truncate_old_batches( return NULL; truncate_past_here: + remainder = ((struct fsmonitor_batch *)batch)->next; + if (!remainder) + return NULL; + + unique_nr = fsmonitor_batch__compact_tail( + (struct fsmonitor_batch *)batch, &input_nr); + trace_printf_key(&trace_fsmonitor, + "Compact: batch %"PRIu64" covers %"PRIuMAX + " of %"PRIuMAX" paths", + batch->batch_seq_nr, + (uintmax_t)unique_nr, (uintmax_t)input_nr); + state->current_token_data->batch_tail = (struct fsmonitor_batch *)batch; - remainder = ((struct fsmonitor_batch *)batch)->next; ((struct fsmonitor_batch *)batch)->next = NULL; return remainder; @@ -692,6 +839,18 @@ static int fsmonitor_parse_client_token(const char *buf_token, return 0; } +static void fsmonitor_reply_overflow_paths( + const struct fsmonitor_batch *batch, + uint64_t requested_oldest_seq_nr, + int hardlink_aware_query, + ipc_server_reply_cb *reply, + struct ipc_server_reply_data *reply_data, + struct strset *shown, + struct strbuf *payload, + uint64_t *total_response_len, + intmax_t *count, + intmax_t *duplicates); + static int do_handle_client(struct fsmonitor_daemon_state *state, const char *command, ipc_server_reply_cb *reply, @@ -909,13 +1068,15 @@ static int do_handle_client(struct fsmonitor_daemon_state *state, do_trivial = 1; } else if (requested_oldest_seq_nr < - token_data->batch_tail->batch_seq_nr) { + token_data->batch_tail->batch_seq_nr && + !token_data->batch_tail->overflow_path_seqs) { /* * The client wants older events than we have for - * this token_id. This means that the end of our - * batch list was truncated and we cannot give the - * client a complete snapshot relative to their - * request. + * this token_id. A normal tail means that the end + * of our batch list was truncated and we cannot + * give the client a complete snapshot. An overflow + * tail retains the latest original sequence for each + * older path. */ trace_printf_key(&trace_fsmonitor, "client requested truncated data"); @@ -964,7 +1125,8 @@ static int do_handle_client(struct fsmonitor_daemon_state *state, */ strset_init_with_options(&shown, NULL, 0); for (batch = batch_head; - batch && batch->batch_seq_nr > requested_oldest_seq_nr; + batch && batch->batch_seq_nr > requested_oldest_seq_nr && + !batch->overflow_path_seqs; batch = batch->next) { size_t k; @@ -999,6 +1161,13 @@ static int do_handle_client(struct fsmonitor_daemon_state *state, } } } + if (batch && batch->batch_seq_nr > requested_oldest_seq_nr) { + fsmonitor_reply_overflow_paths( + batch, requested_oldest_seq_nr, hardlink_aware_query, + reply, reply_data, &shown, &payload, + &total_response_len, &count, &duplicates); + batch = batch->next; + } if (payload.len) { reply(reply_data, payload.buf, payload.len); @@ -1056,6 +1225,60 @@ static int do_handle_client(struct fsmonitor_daemon_state *state, return 0; } +static void fsmonitor_reply_overflow_paths( + const struct fsmonitor_batch *batch, + uint64_t requested_oldest_seq_nr, + int hardlink_aware_query, + ipc_server_reply_cb *reply, + struct ipc_server_reply_data *reply_data, + struct strset *shown, + struct strbuf *payload, + uint64_t *total_response_len, + intmax_t *count, + intmax_t *duplicates) +{ + size_t k; + + if (!batch->overflow_path_seqs) + BUG("expected an overflow batch"); + + for (k = 0; k < batch->nr; k++) { + const char *s = batch->interned_paths[k]; + khint_t pos = kh_get_fsmonitor_path_sequence( + batch->overflow_path_seqs, s); + size_t s_len; + + if (pos == kh_end(batch->overflow_path_seqs)) + BUG("overflow path is missing its sequence"); + if (kh_value(batch->overflow_path_seqs, pos) <= + requested_oldest_seq_nr) + continue; + + if (!hardlink_aware_query && + starts_with(s, FSMONITOR_PATH_HARDLINK_INODE_PREFIX)) + s = FSMONITOR_PATH_GLOBAL_INVALIDATE; + + if (!strset_add(shown, s)) + (*duplicates)++; + else { + trace_printf_key(&trace_fsmonitor, + "send[%"PRIuMAX"]: %s", *count, s); + + /* Each path gets written with a trailing NUL */ + s_len = strlen(s) + 1; + + if (payload->len + s_len >= LARGE_PACKET_DATA_MAX) { + reply(reply_data, payload->buf, payload->len); + *total_response_len += payload->len; + strbuf_reset(payload); + } + + strbuf_add(payload, s, s_len); + (*count)++; + } + } +} + static ipc_server_application_cb handle_client; static int handle_client(void *data, @@ -1379,6 +1602,9 @@ static int fsmonitor_run_daemon(void) int err; memset(&state, 0, sizeof(state)); + truncate_delay_seconds = git_env_ulong( + "GIT_TEST_FSMONITOR_TRUNCATE_DELAY_SECONDS", + MY_TIME_DELAY_SECONDS); hashmap_init(&state.cookies, cookies_cmp, NULL, 0); pthread_mutex_init(&state.main_lock, NULL); diff --git a/builtin/merge.c b/builtin/merge.c index c7e8a31208a386..1d5de7f549dbc2 100644 --- a/builtin/merge.c +++ b/builtin/merge.c @@ -1472,7 +1472,7 @@ int cmd_merge(int argc, goto done; } - if (fast_forward != FF_NO && !getenv(INDEX_ENVIRONMENT) && + if (!getenv(INDEX_ENVIRONMENT) && !clean_status_config_read_repository(the_repository, &clean_digest)) { clean_status_enable_external_history(the_repository); clean_status_set_config_digest(the_repository, &clean_digest); diff --git a/compat/fsmonitor/fsm-darwin-gcc.h b/compat/fsmonitor/fsm-darwin-gcc.h index 959bc88f8f765a..b749012c959ca8 100644 --- a/compat/fsmonitor/fsm-darwin-gcc.h +++ b/compat/fsmonitor/fsm-darwin-gcc.h @@ -97,6 +97,7 @@ CFRunLoopRef CFRunLoopGetCurrent(void); extern CFStringRef kCFRunLoopDefaultMode; void FSEventStreamSetDispatchQueue(FSEventStreamRef stream, dispatch_queue_t q); unsigned char FSEventStreamStart(FSEventStreamRef stream); +FSEventStreamEventId FSEventStreamFlushAsync(FSEventStreamRef stream); void FSEventStreamStop(FSEventStreamRef stream); void FSEventStreamInvalidate(FSEventStreamRef stream); void FSEventStreamRelease(FSEventStreamRef stream); diff --git a/compat/fsmonitor/fsm-listen-darwin.c b/compat/fsmonitor/fsm-listen-darwin.c index f25d7cdd907af9..5ebc70902553c5 100644 --- a/compat/fsmonitor/fsm-listen-darwin.c +++ b/compat/fsmonitor/fsm-listen-darwin.c @@ -29,6 +29,7 @@ #include "fsmonitor--daemon.h" #include "fsmonitor-path-utils.h" #include "gettext.h" +#include "parse.h" #include "simple-ipc.h" #include "string-list.h" #include "trace.h" @@ -57,6 +58,8 @@ struct fsm_listen_data unsigned int stream_scheduled:1; unsigned int stream_started:1; + unsigned int test_cookie_delayed:1; + unsigned long test_cookie_delay_ms; }; static void log_flags_set(const char *path, const FSEventStreamEventFlags flag) @@ -453,6 +456,11 @@ static void fsevent_callback(ConstFSEventStreamRef streamRef UNUSED, } free(resolved); + if (cookie_list.nr && data->test_cookie_delay_ms && + !data->test_cookie_delayed) { + data->test_cookie_delayed = 1; + sleep_millisec(data->test_cookie_delay_ms); + } fsmonitor_publish(state, batch, &cookie_list); string_list_clear(&cookie_list, 0); strbuf_release(&tmp); @@ -511,6 +519,8 @@ int fsm_listen__ctor(struct fsmonitor_daemon_state *state) CALLOC_ARRAY(data, 1); state->listen_data = data; + data->test_cookie_delay_ms = git_env_ulong( + "GIT_TEST_FSMONITOR_COOKIE_DELAY_MS", 0); data->cfsr_event_path_key = CFStringCreateWithCString( NULL, "path", kCFStringEncodingUTF8); @@ -586,6 +596,11 @@ void fsm_listen__stop_async(struct fsmonitor_daemon_state *state) pthread_mutex_unlock(&data->dq_lock); } +void fsm_listen__flush_async(struct fsmonitor_daemon_state *state) +{ + FSEventStreamFlushAsync(state->listen_data->stream); +} + void fsm_listen__loop(struct fsmonitor_daemon_state *state) { struct fsm_listen_data *data; diff --git a/compat/fsmonitor/fsm-listen.h b/compat/fsmonitor/fsm-listen.h index 41650bf8972217..d58e01243b9ad9 100644 --- a/compat/fsmonitor/fsm-listen.h +++ b/compat/fsmonitor/fsm-listen.h @@ -38,6 +38,11 @@ void fsm_listen__dtor(struct fsmonitor_daemon_state *state); */ void fsm_listen__loop(struct fsmonitor_daemon_state *state); +#ifdef __APPLE__ +/* Request delivery of all FSEvents that occurred before this call. */ +void fsm_listen__flush_async(struct fsmonitor_daemon_state *state); +#endif + /* * Gently request that the fsmonitor listener thread shutdown. * It does not wait for it to stop. The caller should do a JOIN diff --git a/t/helper/test-fsmonitor-client.c b/t/helper/test-fsmonitor-client.c index 653d09455382bb..a3f9eb0bbe7bf2 100644 --- a/t/helper/test-fsmonitor-client.c +++ b/t/helper/test-fsmonitor-client.c @@ -64,6 +64,29 @@ static int do_send_query(const char *token) return 0; } +/* + * Send a protocol-v2 token without the capability and worktree-binding + * prefix used by current clients. This models an older client that does + * not understand hard-link inode events. + */ +static int do_send_legacy_query(const char *token) +{ + struct strbuf answer = STRBUF_INIT; + int ret; + + if (!token || !*token) + token = get_token_from_index(); + + ret = fsmonitor_ipc__send_command(token, &answer); + if (ret < 0) + die("could not query fsmonitor--daemon"); + + write_in_full(1, answer.buf, answer.len); + strbuf_release(&answer); + + return 0; +} + /* * Send a "flush" command to the `git-fsmonitor--daemon` (if running) * and tell it to flush its cache. @@ -221,6 +244,7 @@ int cmd__fsmonitor_client(int argc, const char **argv) const char * const fsmonitor_client_usage[] = { "test-tool fsmonitor-client query []", + "test-tool fsmonitor-client query-legacy []", "test-tool fsmonitor-client flush", "test-tool fsmonitor-client record-watch-limit", "test-tool fsmonitor-client hammer [] [] []", @@ -249,6 +273,9 @@ int cmd__fsmonitor_client(int argc, const char **argv) if (!strcmp(subcmd, "query")) return !!do_send_query(token); + if (!strcmp(subcmd, "query-legacy")) + return !!do_send_legacy_query(token); + if (!strcmp(subcmd, "flush")) return !!do_send_flush(); diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 9454c11695077f..f511993452c078 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -2518,6 +2518,137 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ ) ' +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'clean non-fast-forward merges preserve authenticated worktree proofs' ' + test_when_finished "rm -rf clean-no-ff-proof-*" && + for mode in cli no-commit config + do + repo=clean-no-ff-proof-$mode && + test_create_repo "$repo" && + ( + cd "$repo" && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base base && + primary=$(git symbolic-ref --short HEAD) && + git switch -c side && + test_commit topic topic && + git switch "$primary" && + test_commit primary primary && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_INDEX_FILE="$PWD/.git/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_fsmonitor_full_proof .git/index paired && + case "$mode" in + cli) + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCCCCCC \ + git merge --no-ff --no-edit side + ;; + no-commit) + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCCCCCC \ + git merge --no-ff --no-commit side + ;; + config) + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCCCCCC \ + git -c merge.ff=false merge --no-edit side + ;; + esac && + test_fsmonitor_full_proof .git/index paired && + cp .git/index .git/readonly.index && + for run in 1 2 3 + do + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status-$run.trace" \ + git status --porcelain=v2 \ + >.git/status-$run && + test_cmp_bin .git/readonly.index .git/index && + ! test_trace2_data fsmonitor \ + history/external-proof-invalidated 1 \ + <.git/status-$run.trace && + ! have_t2_data_event fsmonitor \ + semantic/manifest-scan-count \ + <.git/status-$run.trace && + if test "$mode" = no-commit + then + test_grep "^1 A\\. .* topic$" \ + .git/status-$run + else + test_must_be_empty .git/status-$run + fi || return 1 + done + ) || return 1 + done +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'non-fast-forward conflicts and alternate indexes fail closed' ' + test_when_finished "rm -rf no-ff-alt-proof no-ff-conflict-proof" && + test_create_repo no-ff-alt-proof && + ( + cd no-ff-alt-proof && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base base && + primary=$(git symbolic-ref --short HEAD) && + git switch -c side && + test_commit topic topic && + git switch "$primary" && + test_commit primary primary && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_INDEX_FILE="$PWD/.git/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_fsmonitor_full_proof .git/index paired && + cp .git/index .git/alternate.index && + GIT_INDEX_FILE="$PWD/.git/alternate.index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCCCCCC \ + git merge --no-ff --no-commit side && + ! test_fsmonitor_full_proof .git/alternate.index paired \ + 2>.git/alternate.proof && + test_grep ! FSUC .git/alternate.index + ) && + test_create_repo no-ff-conflict-proof && + ( + cd no-ff-conflict-proof && + sane_unset GIT_TEST_SPLIT_INDEX && + test_write_lines base >tracked && + git add tracked && + git commit -m base && + primary=$(git symbolic-ref --short HEAD) && + git switch -c side && + test_write_lines side >tracked && + git commit -am side && + git switch "$primary" && + test_write_lines primary >tracked && + git commit -am primary && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_INDEX_FILE="$PWD/.git/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_fsmonitor_full_proof .git/index paired && + test_must_fail env \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCCCCCC \ + git merge --no-ff side && + ! test_fsmonitor_full_proof .git/index paired \ + 2>.git/conflict.proof && + test_grep ! FSUC .git/index && + git ls-files -u >.git/unmerged && + test_file_not_empty .git/unmerged + ) +' + test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ 'full status durably repairs missing mixed-writer index proofs' ' test_when_finished "rm -rf mixed-writer-missing-proofs" && diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index 04c188ac50132a..027e76eb263572 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -88,6 +88,13 @@ stop_daemon_delete_repo () { rm -rf $1 } +stop_daemon_delete_linked_repo () { + r=$1 && + wt=$2 && + { maybe_timeout 30 git -C "$wt" fsmonitor--daemon stop 2>/dev/null || :; } && + rm -rf "$r" "$wt" +} + start_daemon () { r= tf= t2= tk= && @@ -196,6 +203,228 @@ test_expect_success 'implicit daemon start' ' test_must_fail git -C test_implicit fsmonitor--daemon status ' +test_expect_success MACOS 'rescue a delayed FSEvents cookie after timeout' ' + test_when_finished "stop_daemon_delete_repo test_delayed_cookie" && + + git init test_delayed_cookie && + ( + GIT_TEST_FSMONITOR_COOKIE_DELAY_MS=1200 && + export GIT_TEST_FSMONITOR_COOKIE_DELAY_MS && + start_daemon -C test_delayed_cookie \ + --tf "$PWD/delayed-cookie.trace" --tk true + ) && + + token="builtin:${fsmonitor_cookie_token_prefix}test_00000001:0" && + test-tool -C test_delayed_cookie fsmonitor-client query \ + --token "$token" >actual 2>error && + nul_to_q actual-q && + response=$(sed -n "s/Q.*//p" actual-q) && + test "${response%:*}" = "${token%:*}" && + test_grep "^builtin:.*Q$" actual-q && + test_grep ! "Q/Q" actual-q && + test_grep ! "Q//Q" actual-q && + test_grep "cookie_wait: requesting FSEvents flush after initial timeout" \ + delayed-cookie.trace && + test_grep "cookie-seen:" delayed-cookie.trace && + test_grep ! "cookie_wait timed out$" delayed-cookie.trace && + test_must_be_empty error +' + +test_expect_success MACOS 'fall back when a delayed FSEvents cookie stays late' ' + test_when_finished "stop_daemon_delete_repo test_lost_cookie" && + + git init test_lost_cookie && + ( + GIT_TEST_FSMONITOR_COOKIE_DELAY_MS=2500 && + export GIT_TEST_FSMONITOR_COOKIE_DELAY_MS && + start_daemon -C test_lost_cookie \ + --tf "$PWD/lost-cookie.trace" --tk true + ) && + + token="builtin:${fsmonitor_cookie_token_prefix}test_00000001:0" && + test-tool -C test_lost_cookie fsmonitor-client query \ + --token "$token" >actual 2>error && + nul_to_q actual-q && + response=$(sed -n "s/Q.*//p" actual-q) && + test "${response%:*}" != "${token%:*}" && + test_grep "Q/Q$" actual-q && + test_grep "cookie_wait: requesting FSEvents flush after initial timeout" \ + lost-cookie.trace && + test_grep "cookie_wait timed out$" lost-cookie.trace && + test_must_be_empty error +' + +test_expect_success MACOS 'fresh unpinned batches honor the retention grace' ' + test_when_finished "stop_daemon_delete_repo test_fresh_history" && + + git init test_fresh_history && + ( + cd test_fresh_history && + printf "target/\\n" >.gitignore && + printf "a\\n" >tracked-a && + printf "b\\n" >tracked-b && + git add .gitignore tracked-a tracked-b && + git commit -m base && + sane_unset GIT_TEST_FSMONITOR_TRUNCATE_DELAY_SECONDS && + start_daemon --tf "$PWD/../fresh-history.trace" && + git config core.fsmonitor true && + git config core.untrackedCache true && + git status --porcelain=v2 >.git/prime-1 && + git status --porcelain=v2 >.git/prime-2 && + test_must_be_empty .git/prime-1 && + test_must_be_empty .git/prime-2 && + mkdir target && + for i in $(test_seq 1 5250) + do + printf "x\\n" >"target/ignored-$i" || return 1 + done && + test-tool fsmonitor-client query >../fresh-history.response && + perl -0ne '\''$nr++; END { print "$nr\n" }'\'' \ + <../fresh-history.response >../fresh-history.count && + test "$(cat ../fresh-history.count)" -gt 1025 && + git status --porcelain=v2 >.git/after-burst && + printf "new\\n" >>tracked-a && + git status --porcelain=v2 >.git/after-event && + test_grep ! "Compact: batch" ../fresh-history.trace + ) +' + +test_expect_success MACOS 'private index cannot prune canonical index history' ' + test_when_finished "stop_daemon_delete_repo test_index_history" && + test_when_finished "rm -f private-index" && + + git init test_index_history && + ( + cd test_index_history && + test_commit base tracked && + test_commit other other && + git config core.untrackedCache true && + git config core.fsmonitor true && + ( + GIT_TEST_FSMONITOR_TRUNCATE_DELAY_SECONDS=0 && + export GIT_TEST_FSMONITOR_TRUNCATE_DELAY_SECONDS && + start_daemon --tf "$PWD/../index-history.trace" + ) && + + GIT_INDEX_FILE="$PWD/.git/index" \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + cp .git/index ../private-index && + cp .git/index .git/index.before && + + echo first >>tracked && + GIT_INDEX_FILE="$PWD/../private-index" git add -u && + echo second >>other && + GIT_INDEX_FILE="$PWD/../private-index" git add -u && + echo third >>tracked && + GIT_INDEX_FILE="$PWD/../private-index" git add -u && + echo fourth >>other && + GIT_INDEX_FILE="$PWD/../private-index" git add -u && + echo fifth >>tracked && + GIT_INDEX_FILE="$PWD/../private-index" git add -u && + test_cmp .git/index.before .git/index && + test_grep "Compact: batch" ../index-history.trace \ + >../index-history.compactions && + test_line_count = 3 ../index-history.compactions && + test_grep "covers 2 of 3 paths" ../index-history.compactions && + + GIT_OPTIONAL_LOCKS=0 \ + GIT_TRACE2_EVENT="$PWD/.git/canonical.trace" \ + git status --porcelain=v2 >.git/canonical && + test_line_count = 2 .git/canonical && + test_grep "^1 \.M .* tracked$" .git/canonical && + test_grep "^1 \.M .* other$" .git/canonical && + test_cmp .git/index.before .git/index && + ! test_trace2_data fsm_client query/trivial-response 1 \ + <.git/canonical.trace + ) +' + +test_expect_success MACOS,HARDLINKS \ + 'compaction does not replay consumed hardlink events' ' + test_when_finished "stop_daemon_delete_repo test_hardlink_history" && + test_when_finished "rm -f hardlink-private-index" && + + git init test_hardlink_history && + ( + cd test_hardlink_history && + printf "target/\\n" >.gitignore && + printf "a\\n" >tracked-a && + printf "b\\n" >tracked-b && + git add .gitignore tracked-a tracked-b && + git commit -m base && + git config core.untrackedCache true && + git config core.trustctime false && + git config core.checkStat minimal && + ( + GIT_TEST_FSMONITOR_TRUNCATE_DELAY_SECONDS=0 && + export GIT_TEST_FSMONITOR_TRUNCATE_DELAY_SECONDS && + start_daemon --tf "$PWD/../hardlink-history.trace" + ) && + git config core.fsmonitor true && + git status --porcelain=v2 >.git/prime-1 && + git status --porcelain=v2 >.git/prime-2 && + test_must_be_empty .git/prime-1 && + test_must_be_empty .git/prime-2 && + mkdir target && + printf "AAAA\\n" >target/object && + ln target/object target/object-link && + printf "BBBB\\n" >target/object-link && + GIT_TRACE2_EVENT="$PWD/.git/consume.trace" \ + git status --porcelain=v2 >.git/consume && + test_must_be_empty .git/consume && + test_trace2_data fsmonitor apply/hardlink-index-scan 1 \ + <.git/consume.trace && + git update-index --refresh --force-write-index && + test-tool dump-fsmonitor >.git/checkpoint && + checkpoint=$(sed -n "s/^fsmonitor last update //p" \ + .git/checkpoint) && + test -n "$checkpoint" && + test "${checkpoint##*:}" -gt 0 && + cp .git/index .git/index.before && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TRACE2_EVENT="$PWD/.git/control.trace" \ + git status --porcelain=v2 >.git/control && + test_must_be_empty .git/control && + ! test_trace2_data fsmonitor apply/hardlink-index-scan 1 \ + <.git/control.trace && + cp .git/index ../hardlink-private-index && + grep -c "event: //inode:" ../hardlink-history.trace \ + >.git/inodes.before && + for i in $(test_seq 1 8) + do + if test $((i % 2)) -eq 0 + then + printf "private-%s\\n" "$i" >>tracked-a + else + printf "private-%s\\n" "$i" >>tracked-b + fi && + GIT_INDEX_FILE="$PWD/../hardlink-private-index" \ + git add -u || return 1 + done && + test_cmp .git/index.before .git/index && + grep -c "event: //inode:" ../hardlink-history.trace \ + >.git/inodes.after && + test_cmp .git/inodes.before .git/inodes.after && + test_grep "Compact: batch" ../hardlink-history.trace && + for i in $(test_seq 1 5) + do + GIT_OPTIONAL_LOCKS=0 \ + GIT_TRACE2_EVENT="$PWD/.git/repeat-$i.trace" \ + git status --porcelain=v2 \ + >.git/repeat-$i || return 1 + ! test_trace2_data fsmonitor apply/hardlink-index-scan 1 \ + <.git/repeat-$i.trace || return 1 + done && + test_cmp .git/index.before .git/index && + test-tool fsmonitor-client query-legacy \ + --token "$checkpoint" >.git/legacy && + nul_to_q <.git/legacy >.git/legacy-q && + test_grep ! "Q/Q" .git/legacy-q && + test_grep ! "Q//Q" .git/legacy-q + ) +' + # Verify that the daemon has shutdown. Spin a few seconds to # make the test a little more robust during CI testing. # @@ -1829,6 +2058,119 @@ test_expect_success 'bound query replaces a legacy daemon' ' ) ' +test_expect_success MACOS \ + 'read-only legacy upgrade waits for one writable exact repair' ' + test_when_finished \ + "stop_daemon_delete_repo legacy-read-only-upgrade" && + test_create_repo legacy-read-only-upgrade && + ( + cd legacy-read-only-upgrade && + sane_unset GIT_TEST_SPLIT_INDEX && + git config core.fsmonitor false && + for i in $(test_seq 1 64) + do + test_write_lines "$i" >"tracked-$i" || return 1 + done && + git add . && + git commit -qm base && + git config core.preloadIndex false && + git config core.untrackedCache true && + git config core.fsmonitor true && + ipc_path=$(git rev-parse --path-format=absolute \ + --git-path fsmonitor--daemon.ipc) && + test-tool simple-ipc start-daemon \ + --name="$ipc_path" --threads=1 \ + --fsmonitor-capability-superset && + git status --porcelain=v2 --untracked-files=normal \ + --no-ahead-behind >.git/prime && + test_must_be_empty .git/prime && + test_path_is_missing .git/index.csts && + test-tool simple-ipc stop-daemon --name="$ipc_path" && + test-tool simple-ipc start-daemon \ + --name="$ipc_path" --threads=1 --fsmonitor-legacy && + cp .git/index .git/index.before && + + for label in first repeat + do + GIT_OPTIONAL_LOCKS=0 \ + GIT_TRACE2_EVENT="$PWD/.git/$label.trace" \ + git status --porcelain=v2 >.git/$label && + test_must_be_empty .git/$label && + test_cmp_bin .git/index.before .git/index && + test_trace2_data index refresh/sum_lstat 64 \ + <.git/$label.trace || return 1 + done && + test_trace2_data fsm_client query/incompatible-daemon 1 \ + <.git/first.trace && + test_path_is_missing .git/index.csts && + { git fsmonitor--daemon stop 2>/dev/null || :; } && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + GIT_TRACE2_EVENT="$PWD/.git/repair.trace" \ + git status --porcelain=v2 >.git/repair && + test_must_be_empty .git/repair && + ! test_cmp_bin .git/index.before .git/index && + test-tool dump-fsmonitor >.git/fsmonitor && + test_grep "fsmonitor last update builtin:test:1" \ + .git/fsmonitor + ) +' + +test_expect_success MACOS \ + 'linked worktree legacy upgrade uses its writable index repair' ' + test_when_finished \ + "stop_daemon_delete_linked_repo legacy-linked legacy-linked-wt" && + test_create_repo legacy-linked && + ( + cd legacy-linked && + git config core.fsmonitor false && + for i in $(test_seq 1 32) + do + test_write_lines "$i" >"tracked-$i" || return 1 + done && + git add . && + git commit -qm base && + git worktree add -q -b linked ../legacy-linked-wt + ) && + git -C legacy-linked config core.preloadIndex false && + git -C legacy-linked config core.untrackedCache true && + git -C legacy-linked config core.fsmonitor true && + gitdir=$(git -C legacy-linked-wt rev-parse --absolute-git-dir) && + ipc_path=$(git -C legacy-linked-wt rev-parse --path-format=absolute \ + --git-path fsmonitor--daemon.ipc) && + test-tool simple-ipc start-daemon \ + --name="$ipc_path" --threads=1 \ + --fsmonitor-capability-superset && + git -C legacy-linked-wt status --porcelain=v2 \ + --untracked-files=normal --no-ahead-behind >linked.prime && + test_must_be_empty linked.prime && + test-tool simple-ipc stop-daemon --name="$ipc_path" && + test-tool simple-ipc start-daemon \ + --name="$ipc_path" --threads=1 --fsmonitor-legacy && + cp "$gitdir/index" linked.index.before && + for label in first repeat + do + GIT_OPTIONAL_LOCKS=0 \ + GIT_TRACE2_EVENT="$PWD/linked-$label.trace" \ + git -C legacy-linked-wt status --porcelain=v2 \ + >linked-$label && + test_must_be_empty linked-$label && + test_cmp_bin linked.index.before "$gitdir/index" && + test_trace2_data index refresh/sum_lstat 32 \ + /dev/null || :; } && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + GIT_TRACE2_EVENT="$PWD/linked-repair.trace" \ + git -C legacy-linked-wt status --porcelain=v2 >linked-repair && + test_must_be_empty linked-repair && + ! test_cmp_bin linked.index.before "$gitdir/index" && + test_path_is_missing "$gitdir/index.csts" && + test-tool -C legacy-linked-wt dump-fsmonitor >linked.fsmonitor && + test_grep "fsmonitor last update builtin:test:1" \ + linked.fsmonitor +' + test_expect_success MACOS 'bound query upgrades stale directory event daemon' ' test_when_finished \ "stop_daemon_delete_repo directory-daemon-upgrade" && diff --git a/t/t7530-status-clean-sidecar.sh b/t/t7530-status-clean-sidecar.sh index 03fbcdc68a2246..c67531a7a5fcd0 100755 --- a/t/t7530-status-clean-sidecar.sh +++ b/t/t7530-status-clean-sidecar.sh @@ -1727,7 +1727,7 @@ test_expect_success DURABLE_FSMONITOR \ ' test_expect_success DURABLE_FSMONITOR \ - 'exact status persists stat repairs before a sidecar' ' + 'exact status installs a sidecar after stat repairs' ' test_when_finished "stop_daemon external-stat-exact" && setup_repo external-stat-exact && git -C external-stat-exact config core.autocrlf false && @@ -1740,21 +1740,32 @@ test_expect_success DURABLE_FSMONITOR \ test_must_be_empty actual && test_trace2_data fsmonitor history/external-stored 1 \ actual && test_must_be_empty actual && - test_trace2_data fsmonitor history/external-stored 1 \ + test_trace2_data status clean-proof/hit 1 \ actual && test_must_be_empty actual && test_path_is_missing sidecar-shape/.git/index.csts && + bulk_status -C sidecar-shape status --porcelain=v2 \ + --untracked-files=normal --no-ahead-behind >actual && + test_must_be_empty actual && + test_path_is_missing sidecar-shape/.git/index.csts && test_env GIT_TRACE2_EVENT="$PWD/shape-branch.trace" \ bulk_status -C sidecar-shape \