From 8d33ee35aefd7b11d01cdd859e31c66b9d174636 Mon Sep 17 00:00:00 2001 From: yushan Date: Tue, 22 Sep 2026 19:07:38 +0000 Subject: [PATCH] feat(itg): support excluded file regexes in incremental hash computation Targets whose name matches one of the configured exclusion regexes now get an empty hash instead of being computed, mirroring the existing isExcluded behavior in core/targethasher/graph.go used by the full-graph hashing path. UpdateGraphInput gains an ExcludedRegex field that is threaded through computeAvailableHashes and the recursive computeHashes/computeInvalidatedHashes so incremental updates respect the same exclusions as a full recompute, and excluded targets short-circuit before their dependencies are resolved. Co-Authored-By: Claude Sonnet 5 --- core/itg/graph/update.go | 41 +++++++--- core/itg/graph/update_test.go | 137 ++++++++++++++++++++++++++++++---- 2 files changed, 155 insertions(+), 23 deletions(-) diff --git a/core/itg/graph/update.go b/core/itg/graph/update.go index 9d0849b9..a7041421 100644 --- a/core/itg/graph/update.go +++ b/core/itg/graph/update.go @@ -18,6 +18,7 @@ import ( "context" "crypto/sha1" "fmt" + "regexp" "slices" "strings" @@ -38,6 +39,7 @@ type UpdateGraphInput struct { WorkspaceRoot string FullHashRepos StringSet UseBzlmod bool + ExcludedRegex []*regexp.Regexp } // UpdateGraph updates the dependency relationships and hashes of targets in the graph. @@ -85,7 +87,7 @@ func (g *OptimizedGraph) UpdateGraph( } } // compute hashes for source file, package group, and rule common targets - if err := computeAvailableHashes(ctx, sourceHasher, targets); err != nil { + if err := computeAvailableHashes(ctx, sourceHasher, targets, input.ExcludedRegex); err != nil { return err } @@ -94,19 +96,19 @@ func (g *OptimizedGraph) UpdateGraph( return err } - return g.computeInvalidatedHashes(ctx, allInvalidated) + return g.computeInvalidatedHashes(ctx, allInvalidated, input.ExcludedRegex) } // computeInvalidatedHashes computes invalidated target hashes in the same // deterministic traversal order used by full graph hashing. -func (g *OptimizedGraph) computeInvalidatedHashes(ctx context.Context, invalidated IntSet) error { +func (g *OptimizedGraph) computeInvalidatedHashes(ctx context.Context, invalidated IntSet, excludedRegex []*regexp.Regexp) error { // Prioritize configured targets that could create cycles. for _, target := range sequentialHashTargets { id, ok := g.TargetNameToID[target] if !ok || !invalidated.Contains(id) { continue } - if _, err := g.computeHashes(ctx, id); err != nil { + if _, err := g.computeHashes(ctx, id, excludedRegex); err != nil { return err } } @@ -123,7 +125,7 @@ func (g *OptimizedGraph) computeInvalidatedHashes(ctx context.Context, invalidat if !ok || len(target.ReverseDeps) != 0 { continue } - if _, err := g.computeHashes(ctx, id); err != nil { + if _, err := g.computeHashes(ctx, id, excludedRegex); err != nil { return err } } @@ -132,7 +134,7 @@ func (g *OptimizedGraph) computeInvalidatedHashes(ctx context.Context, invalidat // root-unreachable cycles. The first lexicographic member is their canonical // cycle breaker, matching full graph hashing's cyclic-target fallback. for _, id := range ids { - if _, err := g.computeHashes(ctx, id); err != nil { + if _, err := g.computeHashes(ctx, id, excludedRegex); err != nil { return err } } @@ -175,8 +177,14 @@ func computeAvailableHashes( ctx context.Context, hasher targethasher.SourceHasher, targets map[string]*targethasher.Target, + excludedRegex []*regexp.Regexp, ) error { for name, target := range targets { + if isExcludedTarget(name, excludedRegex) { + target.Hash = []byte{} + target.HashWithoutDeps = []byte{} + continue + } var hash []byte var hashWithoutDeps []byte switch target.RuleType { @@ -210,7 +218,7 @@ func computeAvailableHashes( } // computeHashes computes hashes recursively for the given target ID. -func (g *OptimizedGraph) computeHashes(ctx context.Context, id int) ([]byte, error) { +func (g *OptimizedGraph) computeHashes(ctx context.Context, id int, excludedRegex []*regexp.Regexp) ([]byte, error) { if ctx.Err() != nil { return nil, context.Cause(ctx) } @@ -227,6 +235,12 @@ func (g *OptimizedGraph) computeHashes(ctx context.Context, id int) ([]byte, err return target.Hash, nil } + if isExcludedTarget(g.TargetIDToString[id], excludedRegex) { + target.Hash = []byte{} + target.HashWithoutDeps = []byte{} + return target.Hash, nil + } + // mark as visiting to handle cycles target.Hash = []byte{} var hash []byte @@ -239,7 +253,7 @@ func (g *OptimizedGraph) computeHashes(ctx context.Context, id int) ([]byte, err singleDep = dep break } - dephash, err := g.computeHashes(ctx, singleDep) + dephash, err := g.computeHashes(ctx, singleDep, excludedRegex) if err != nil { return nil, err } @@ -255,7 +269,7 @@ func (g *OptimizedGraph) computeHashes(ctx context.Context, id int) ([]byte, err return strings.Compare(g.TargetIDToString[i], g.TargetIDToString[j]) }) for _, dep := range depIDs { - dephash, err := g.computeHashes(ctx, dep) + dephash, err := g.computeHashes(ctx, dep, excludedRegex) if err != nil { return nil, err } @@ -268,3 +282,12 @@ func (g *OptimizedGraph) computeHashes(ctx context.Context, id int) ([]byte, err } return hash, nil } + +func isExcludedTarget(name string, excludedRegex []*regexp.Regexp) bool { + for _, re := range excludedRegex { + if re.MatchString(name) { + return true + } + } + return false +} diff --git a/core/itg/graph/update_test.go b/core/itg/graph/update_test.go index 19b79b34..87dc68e3 100644 --- a/core/itg/graph/update_test.go +++ b/core/itg/graph/update_test.go @@ -17,6 +17,7 @@ package graph import ( "context" "crypto/sha1" + "regexp" "testing" buildpb "github.com/bazelbuild/buildtools/build_proto" @@ -57,7 +58,7 @@ func TestComputeAvailableHashes(t *testing.T) { }, } - require.NoError(t, computeAvailableHashes(ctx, hasher, targets)) + require.NoError(t, computeAvailableHashes(ctx, hasher, targets, nil)) assert.Equal(t, expected, targets[name].Hash) assert.Equal(t, ctx, hasher.ctx) }) @@ -69,7 +70,7 @@ func TestComputeAvailableHashes(t *testing.T) { name: {Name: name, RuleType: targethasher.PackageGroup}, } - require.NoError(t, computeAvailableHashes(context.Background(), &fakeSourceHasher{}, targets)) + require.NoError(t, computeAvailableHashes(context.Background(), &fakeSourceHasher{}, targets, nil)) h := sha1.New() h.Write([]byte(name)) @@ -83,7 +84,7 @@ func TestComputeAvailableHashes(t *testing.T) { name: {Name: name, RuleType: targethasher.ExternalRuleType}, } - require.NoError(t, computeAvailableHashes(context.Background(), &fakeSourceHasher{}, targets)) + require.NoError(t, computeAvailableHashes(context.Background(), &fakeSourceHasher{}, targets, nil)) assert.Nil(t, targets[name].Hash, "external rule targets should not get a hash here") }) @@ -94,7 +95,7 @@ func TestComputeAvailableHashes(t *testing.T) { name: {Name: name, RuleType: targethasher.GeneratedFileType}, } - require.NoError(t, computeAvailableHashes(context.Background(), &fakeSourceHasher{}, targets)) + require.NoError(t, computeAvailableHashes(context.Background(), &fakeSourceHasher{}, targets, nil)) assert.Nil(t, targets[name].Hash, "generated file hash is resolved later") }) @@ -111,7 +112,7 @@ func TestComputeAvailableHashes(t *testing.T) { }, } - require.NoError(t, computeAvailableHashes(context.Background(), &fakeSourceHasher{}, targets)) + require.NoError(t, computeAvailableHashes(context.Background(), &fakeSourceHasher{}, targets, nil)) assert.NotNil(t, targets[name].HashWithoutDeps, "rule should have HashWithoutDeps after hashing") assert.Nil(t, targets[name].Hash, "full hash is not computed here — deps are needed") }) @@ -123,9 +124,85 @@ func TestComputeAvailableHashes(t *testing.T) { "//pkg:f": {Name: "//pkg:f", RuleType: targethasher.SourceFileType, SourceFile: &buildpb.SourceFile{}}, } - err := computeAvailableHashes(context.Background(), hasher, targets) + err := computeAvailableHashes(context.Background(), hasher, targets, nil) assert.Error(t, err) }) + + t.Run("excluded source file target is skipped without calling hasher", func(t *testing.T) { + t.Parallel() + name := "//pkg:secret.go" + hasher := &fakeSourceHasher{err: assert.AnError} // would fail the test if called + targets := map[string]*targethasher.Target{ + name: {Name: name, RuleType: targethasher.SourceFileType, SourceFile: &buildpb.SourceFile{}}, + } + + err := computeAvailableHashes(context.Background(), hasher, targets, []*regexp.Regexp{regexp.MustCompile("secret")}) + require.NoError(t, err) + assert.Equal(t, []byte{}, targets[name].Hash) + assert.Equal(t, []byte{}, targets[name].HashWithoutDeps) + assert.Nil(t, hasher.ctx, "hasher should never be invoked for an excluded target") + }) + + t.Run("excluded rule target gets empty hash instead of HashWithoutDeps", func(t *testing.T) { + t.Parallel() + name := "//pkg:lib" + ruleName := name + ruleClass := "go_library" + targets := map[string]*targethasher.Target{ + name: { + Name: name, + RuleType: "go_library", + Rule: &buildpb.Rule{Name: &ruleName, RuleClass: &ruleClass}, + }, + } + + err := computeAvailableHashes(context.Background(), &fakeSourceHasher{}, targets, []*regexp.Regexp{regexp.MustCompile("//pkg:lib")}) + require.NoError(t, err) + assert.Equal(t, []byte{}, targets[name].Hash) + assert.Equal(t, []byte{}, targets[name].HashWithoutDeps) + }) + + t.Run("non-matching excludedRegex leaves hashing unaffected", func(t *testing.T) { + t.Parallel() + name := "//pkg:__pkg__" + targets := map[string]*targethasher.Target{ + name: {Name: name, RuleType: targethasher.PackageGroup}, + } + + err := computeAvailableHashes(context.Background(), &fakeSourceHasher{}, targets, []*regexp.Regexp{regexp.MustCompile("no-match")}) + require.NoError(t, err) + + h := sha1.New() + h.Write([]byte(name)) + assert.Equal(t, h.Sum(nil), targets[name].Hash) + }) +} + +func TestIsExcludedTarget(t *testing.T) { + t.Parallel() + + t.Run("empty regex list never excludes", func(t *testing.T) { + t.Parallel() + assert.False(t, isExcludedTarget("//pkg:anything", nil)) + }) + + t.Run("matches when any regex in the list matches", func(t *testing.T) { + t.Parallel() + regexes := []*regexp.Regexp{ + regexp.MustCompile("^//other:"), + regexp.MustCompile("^//pkg:target$"), + } + assert.True(t, isExcludedTarget("//pkg:target", regexes)) + }) + + t.Run("returns false when no regex matches", func(t *testing.T) { + t.Parallel() + regexes := []*regexp.Regexp{ + regexp.MustCompile("^//other:"), + regexp.MustCompile("^//another:"), + } + assert.False(t, isExcludedTarget("//pkg:target", regexes)) + }) } // --- computeHashes --- @@ -143,7 +220,7 @@ func TestComputeHashes(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - _, err := g.computeHashes(ctx, aID) + _, err := g.computeHashes(ctx, aID, nil) assert.ErrorIs(t, err, context.Canceled) }) @@ -155,7 +232,7 @@ func TestComputeHashes(t *testing.T) { }) aID := g.TargetNameToID["//pkg:a"] - got, err := g.computeHashes(context.Background(), aID) + got, err := g.computeHashes(context.Background(), aID, nil) require.NoError(t, err) assert.Equal(t, hash, got) }) @@ -164,7 +241,7 @@ func TestComputeHashes(t *testing.T) { t.Parallel() g := OptimizeGraph(nil) - _, err := g.computeHashes(context.Background(), 9999) + _, err := g.computeHashes(context.Background(), 9999, nil) assert.Error(t, err) }) @@ -176,7 +253,7 @@ func TestComputeHashes(t *testing.T) { }) id := g.TargetNameToID["//external:repo"] - got, err := g.computeHashes(context.Background(), id) + got, err := g.computeHashes(context.Background(), id, nil) require.NoError(t, err) assert.Equal(t, hash, got) }) @@ -188,7 +265,7 @@ func TestComputeHashes(t *testing.T) { }) id := g.TargetNameToID["//pkg:f.go"] - _, err := g.computeHashes(context.Background(), id) + _, err := g.computeHashes(context.Background(), id, nil) assert.Error(t, err, "source file should already have its hash set") }) @@ -206,7 +283,7 @@ func TestComputeHashes(t *testing.T) { g.OptimizedTargets[ruleID].Hash = depHash g.OptimizedTargets[genID].Hash = nil // force recompute - got, err := g.computeHashes(context.Background(), genID) + got, err := g.computeHashes(context.Background(), genID, nil) require.NoError(t, err) assert.Equal(t, depHash, got, "generated file should inherit its dep's hash") }) @@ -223,7 +300,7 @@ func TestComputeHashes(t *testing.T) { // lib has no final hash yet (only HashWithoutDeps) g.OptimizedTargets[libID].Hash = nil - got, err := g.computeHashes(context.Background(), libID) + got, err := g.computeHashes(context.Background(), libID, nil) require.NoError(t, err) // Expected: sha1(hwod || depHash) (single dep, already sorted) @@ -232,6 +309,38 @@ func TestComputeHashes(t *testing.T) { h.Write(depHash) assert.Equal(t, h.Sum(nil), got) }) + + t.Run("excluded target short-circuits before dep traversal", func(t *testing.T) { + t.Parallel() + g := OptimizeGraph(map[string]*targethasher.Target{ + "//pkg:lib": { + Name: "//pkg:lib", + RuleType: "go_library", + HashWithoutDeps: []byte{0x01}, + Deps: []string{"//pkg:missing"}, // dangling: never added to the graph + }, + }) + libID := g.TargetNameToID["//pkg:lib"] + g.OptimizedTargets[libID].Hash = nil // force recompute path + + got, err := g.computeHashes(context.Background(), libID, []*regexp.Regexp{regexp.MustCompile("//pkg:lib")}) + require.NoError(t, err, "excluded targets should short-circuit before resolving deps") + assert.Equal(t, []byte{}, got) + + // Sanity check: without the exclusion, the same dangling dep does error. + g2 := OptimizeGraph(map[string]*targethasher.Target{ + "//pkg:lib": { + Name: "//pkg:lib", + RuleType: "go_library", + HashWithoutDeps: []byte{0x01}, + Deps: []string{"//pkg:missing"}, + }, + }) + libID2 := g2.TargetNameToID["//pkg:lib"] + g2.OptimizedTargets[libID2].Hash = nil + _, err = g2.computeHashes(context.Background(), libID2, nil) + assert.Error(t, err, "unexcluded target with a dangling dep should still error") + }) } func TestComputeInvalidatedHashesCycleOrderInvariance(t *testing.T) { @@ -316,7 +425,7 @@ func TestComputeInvalidatedHashesCycleOrderInvariance(t *testing.T) { invalidated.Insert(graph.TargetNameToID[name]) } - err := graph.computeInvalidatedHashes(ctx, invalidated) + err := graph.computeInvalidatedHashes(ctx, invalidated, nil) require.NoError(rt, err) for name, expected := range expectedHashes { actual := graph.OptimizedTargets[graph.TargetNameToID[name]].Hash