Fix the case where lazygit struggles with index.lock in very large repos

pkg/commands/git_commands/file_loader.go

@@ -1,7 +1,11 @@
 package git_commands
 
 import (
+	"encoding/binary"
 	"fmt"
+	"io"
+	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 
@@ -45,10 +49,30 @@ type GetStatusFileOptions struct {
 }
 
 func (self *FileLoader) GetStatusFiles(opts GetStatusFileOptions) []*models.File {
-	// check if config wants us ignoring untracked files
+	// Decide how to pass --untracked-files to git status.
+	//
+	// If the user has explicitly set status.showUntrackedFiles, always honor it.
+	// Otherwise default to "all" so that individual files inside newly-created
+	// untracked directories show up in the files panel. The exception is very
+	// large repos: git can only use its untracked cache in "normal" mode, so in
+	// "all" mode it does a full recursive readdir of the entire worktree on every
+	// status. In a repo with hundreds of thousands of files that takes several
+	// seconds and holds index.lock long enough that a concurrent git command
+	// (e.g. the checkout that triggered this refresh) can fail with
+	// "Unable to create '.../index.lock': File exists". In "normal" mode git uses
+	// the untracked cache and stays fast, at the cost of showing a brand-new
+	// untracked directory as a single entry rather than listing each file within
+	// it. See https://github.com/jesseduffield/lazygit/issues/5906.
 	untrackedFilesSetting := self.config.GetShowUntrackedFiles()
-
-	if opts.ForceShowUntracked || untrackedFilesSetting == "" {
+	if untrackedFilesSetting == "" {
+		untrackedFilesSetting = "all"
+		if self.repoTooLargeForUntrackedFilesAll() {
+			untrackedFilesSetting = "normal"
+		}
+	}
+	// The "show untracked files" filter is an explicit, transient user request to
+	// see untracked files, so honor it even in a large repo.
+	if opts.ForceShowUntracked {
 		untrackedFilesSetting = "all"
 	}
 	untrackedFilesArg := fmt.Sprintf("--untracked-files=%s", untrackedFilesSetting)
@@ -189,6 +213,49 @@ func (self *FileLoader) getFileDiffs() (map[string]FileDiff, error) {
 	return fileDiffs, nil
 }
 
+// untrackedFilesAllMaxTrackedFiles is the tracked-file count above which we stop
+// defaulting --untracked-files to "all" (see GetStatusFiles): git's untracked
+// cache is only used in "normal" mode, so "all" becomes prohibitively slow in
+// very large repos. We use the number of entries in the index as a cheap proxy
+// for the size of the working tree (and thus the cost of an "all" scan).
+const untrackedFilesAllMaxTrackedFiles = 100_000
+
+// repoTooLargeForUntrackedFilesAll reports whether the repository is large enough
+// that we should prefer "--untracked-files=normal" over "all" by default. It
+// reads the tracked-entry count straight from the index header, which is
+// essentially free. If the count can't be determined it returns false, so we
+// keep the previous default of "all".
+func (self *FileLoader) repoTooLargeForUntrackedFilesAll() bool {
+	if self.repoPaths == nil {
+		return false
+	}
+	count, ok := trackedFileCountFromIndex(filepath.Join(self.repoPaths.WorktreeGitDirPath(), "index"))
+	return ok && count > untrackedFilesAllMaxTrackedFiles
+}
+
+// trackedFileCountFromIndex returns the number of entries recorded in the git
+// index at indexPath, read from its 12-byte header: the 4-byte signature "DIRC",
+// a 4-byte version, and a 4-byte big-endian entry count.
+//
+// Limitation: with a split index (core.splitIndex) this reads only the main
+// index file, which holds a small delta.
+func trackedFileCountFromIndex(indexPath string) (int, bool) {
+	f, err := os.Open(indexPath)
+	if err != nil {
+		return 0, false
+	}
+	defer f.Close()
+
+	var header [12]byte
+	if _, err := io.ReadFull(f, header[:]); err != nil {
+		return 0, false
+	}
+	if string(header[:4]) != "DIRC" {
+		return 0, false
+	}
+	return int(binary.BigEndian.Uint32(header[8:12])), true
+}
+
 // GitStatus returns the file status of the repo
 type GitStatusOptions struct {
 	NoRenames         bool

pkg/commands/git_commands/file_loader_test.go

@@ -1,6 +1,9 @@
 package git_commands
 
 import (
+	"encoding/binary"
+	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/jesseduffield/lazygit/pkg/commands/models"
@@ -314,3 +317,181 @@ type FakeFileLoaderConfig struct {
 func (self *FakeFileLoaderConfig) GetShowUntrackedFiles() string {
 	return self.showUntrackedFiles
 }
+
+// writeFakeIndexHeader writes a minimal git index file (just the 12-byte header:
+// "DIRC" + version + big-endian entry count) at <worktreeDir>/.git/index, which
+// is where MockRepoPaths expects it.
+func writeFakeIndexHeader(t *testing.T, worktreeDir string, entryCount uint32) {
+	t.Helper()
+	gitDir := filepath.Join(worktreeDir, ".git")
+	if err := os.MkdirAll(gitDir, 0o700); err != nil {
+		t.Fatalf("failed to create .git dir: %v", err)
+	}
+	header := make([]byte, 12)
+	copy(header, "DIRC")
+	binary.BigEndian.PutUint32(header[4:8], 2) // index format version
+	binary.BigEndian.PutUint32(header[8:12], entryCount)
+	if err := os.WriteFile(filepath.Join(gitDir, "index"), header, 0o600); err != nil {
+		t.Fatalf("failed to write index: %v", err)
+	}
+}
+
+// TestFileLoaderUntrackedFilesArg covers how GetStatusFiles chooses the
+// --untracked-files argument: an explicit status.showUntrackedFiles setting is
+// always honored, an unset setting defaults to "all" but downgrades to "normal"
+// in large repos, and the ForceShowUntracked filter forces "all" regardless.
+func TestFileLoaderUntrackedFilesArg(t *testing.T) {
+	type scenario struct {
+		testName string
+		// git config status.showUntrackedFiles ("" means unset)
+		showUntrackedFiles string
+		// number of entries to record in the fake index header
+		indexEntryCount uint32
+		// if false, no index file is written (simulates an unreadable/missing index)
+		writeIndex         bool
+		forceShowUntracked bool
+		expectedArg        string
+	}
+
+	scenarios := []scenario{
+		{
+			testName:        "unset config, large repo -> normal",
+			indexEntryCount: untrackedFilesAllMaxTrackedFiles + 1,
+			writeIndex:      true,
+			expectedArg:     "--untracked-files=normal",
+		},
+		{
+			testName:        "unset config, small repo -> all",
+			indexEntryCount: 100,
+			writeIndex:      true,
+			expectedArg:     "--untracked-files=all",
+		},
+		{
+			testName:        "unset config, count exactly at threshold -> all (strict >)",
+			indexEntryCount: untrackedFilesAllMaxTrackedFiles,
+			writeIndex:      true,
+			expectedArg:     "--untracked-files=all",
+		},
+		{
+			testName:    "unset config, missing/unreadable index -> all (fail-safe)",
+			writeIndex:  false,
+			expectedArg: "--untracked-files=all",
+		},
+		{
+			testName:           "explicit 'all' honored even in large repo",
+			showUntrackedFiles: "all",
+			indexEntryCount:    untrackedFilesAllMaxTrackedFiles + 1,
+			writeIndex:         true,
+			expectedArg:        "--untracked-files=all",
+		},
+		{
+			testName:           "explicit 'no' honored even in large repo",
+			showUntrackedFiles: "no",
+			indexEntryCount:    untrackedFilesAllMaxTrackedFiles + 1,
+			writeIndex:         true,
+			expectedArg:        "--untracked-files=no",
+		},
+		{
+			testName:           "explicit 'normal' honored in small repo",
+			showUntrackedFiles: "normal",
+			indexEntryCount:    100,
+			writeIndex:         true,
+			expectedArg:        "--untracked-files=normal",
+		},
+		{
+			testName:           "ForceShowUntracked forces all in large repo",
+			indexEntryCount:    untrackedFilesAllMaxTrackedFiles + 1,
+			writeIndex:         true,
+			forceShowUntracked: true,
+			expectedArg:        "--untracked-files=all",
+		},
+		{
+			testName:           "ForceShowUntracked overrides explicit 'no'",
+			showUntrackedFiles: "no",
+			indexEntryCount:    100,
+			writeIndex:         true,
+			forceShowUntracked: true,
+			expectedArg:        "--untracked-files=all",
+		},
+	}
+
+	for _, s := range scenarios {
+		t.Run(s.testName, func(t *testing.T) {
+			worktreeDir := t.TempDir()
+			if s.writeIndex {
+				writeFakeIndexHeader(t, worktreeDir, s.indexEntryCount)
+			}
+
+			runner := oscommands.NewFakeRunner(t).
+				ExpectGitArgs([]string{"status", s.expectedArg, "--porcelain", "-z", "--find-renames=50%"}, "", nil)
+			cmd := oscommands.NewDummyCmdObjBuilder(runner)
+
+			userConfig := &config.UserConfig{}
+			userConfig.Git.RenameSimilarityThreshold = 50
+
+			loader := &FileLoader{
+				GitCommon: buildGitCommon(commonDeps{
+					appState:   &config.AppState{},
+					userConfig: userConfig,
+					repoPaths:  MockRepoPaths(worktreeDir),
+				}),
+				cmd:         cmd,
+				config:      &FakeFileLoaderConfig{showUntrackedFiles: s.showUntrackedFiles},
+				getFileType: func(string) string { return "file" },
+			}
+
+			loader.GetStatusFiles(GetStatusFileOptions{ForceShowUntracked: s.forceShowUntracked})
+			runner.CheckForMissingCalls()
+		})
+	}
+}
+
+func TestTrackedFileCountFromIndex(t *testing.T) {
+	validHeader := func(count uint32) []byte {
+		b := make([]byte, 12)
+		copy(b, "DIRC")
+		binary.BigEndian.PutUint32(b[4:8], 2)
+		binary.BigEndian.PutUint32(b[8:12], count)
+		return b
+	}
+	writeIndex := func(t *testing.T, content []byte) string {
+		t.Helper()
+		path := filepath.Join(t.TempDir(), "index")
+		if err := os.WriteFile(path, content, 0o600); err != nil {
+			t.Fatalf("failed to write index: %v", err)
+		}
+		return path
+	}
+
+	t.Run("valid header returns entry count", func(t *testing.T) {
+		count, ok := trackedFileCountFromIndex(writeIndex(t, validHeader(123456)))
+		assert.True(t, ok)
+		assert.Equal(t, 123456, count)
+	})
+
+	t.Run("trailing entry bytes don't affect the count", func(t *testing.T) {
+		count, ok := trackedFileCountFromIndex(writeIndex(t, append(validHeader(42), []byte("trailing entry data")...)))
+		assert.True(t, ok)
+		assert.Equal(t, 42, count)
+	})
+
+	t.Run("missing file returns not-ok", func(t *testing.T) {
+		count, ok := trackedFileCountFromIndex(filepath.Join(t.TempDir(), "does-not-exist"))
+		assert.False(t, ok)
+		assert.Equal(t, 0, count)
+	})
+
+	t.Run("truncated header returns not-ok", func(t *testing.T) {
+		count, ok := trackedFileCountFromIndex(writeIndex(t, []byte("DIRC")))
+		assert.False(t, ok)
+		assert.Equal(t, 0, count)
+	})
+
+	t.Run("wrong signature returns not-ok", func(t *testing.T) {
+		bad := validHeader(999)
+		copy(bad, "XXXX")
+		count, ok := trackedFileCountFromIndex(writeIndex(t, bad))
+		assert.False(t, ok)
+		assert.Equal(t, 0, count)
+	})
+}