Skip to content

feat(itg): support excluded file regexes in incremental hash computation #317

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
yushan8 wants to merge 1 commit into
base: main
Choose a base branch
from
+155 −23
Open

feat(itg): support excluded file regexes in incremental hash computation #317

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 32 additions & 9 deletions core/itg/graph/update.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"context"
"crypto/sha1"
"fmt"
"regexp"
"slices"
"strings"

Expand All @@ -38,6 +39,7 @@ type UpdateGraphInput struct {
WorkspaceRoot string
FullHashRepos StringSet
UseBzlmod bool
ExcludedRegex []*regexp.Regexp
}

// UpdateGraph updates the dependency relationships and hashes of targets in the graph.
Expand Down Expand Up @@ -85,7 +87,7 @@ func (g *OptimizedGraph) UpdateGraph(
}
}
// compute hashes for source file, package group, and rule common targets
if err := computeAvailableHashes(ctx, sourceHasher, targets); err != nil {
if err := computeAvailableHashes(ctx, sourceHasher, targets, input.ExcludedRegex); err != nil {
return err
}

Expand All @@ -94,19 +96,19 @@ func (g *OptimizedGraph) UpdateGraph(
return err
}

return g.computeInvalidatedHashes(ctx, allInvalidated)
return g.computeInvalidatedHashes(ctx, allInvalidated, input.ExcludedRegex)
}

// computeInvalidatedHashes computes invalidated target hashes in the same
// deterministic traversal order used by full graph hashing.
func (g *OptimizedGraph) computeInvalidatedHashes(ctx context.Context, invalidated IntSet) error {
func (g *OptimizedGraph) computeInvalidatedHashes(ctx context.Context, invalidated IntSet, excludedRegex []*regexp.Regexp) error {
// Prioritize configured targets that could create cycles.
for _, target := range sequentialHashTargets {
id, ok := g.TargetNameToID[target]
if !ok || !invalidated.Contains(id) {
continue
}
if _, err := g.computeHashes(ctx, id); err != nil {
if _, err := g.computeHashes(ctx, id, excludedRegex); err != nil {
return err
}
}
Expand All @@ -123,7 +125,7 @@ func (g *OptimizedGraph) computeInvalidatedHashes(ctx context.Context, invalidat
if !ok || len(target.ReverseDeps) != 0 {
continue
}
if _, err := g.computeHashes(ctx, id); err != nil {
if _, err := g.computeHashes(ctx, id, excludedRegex); err != nil {
return err
}
}
Expand All @@ -132,7 +134,7 @@ func (g *OptimizedGraph) computeInvalidatedHashes(ctx context.Context, invalidat
// root-unreachable cycles. The first lexicographic member is their canonical
// cycle breaker, matching full graph hashing's cyclic-target fallback.
for _, id := range ids {
if _, err := g.computeHashes(ctx, id); err != nil {
if _, err := g.computeHashes(ctx, id, excludedRegex); err != nil {
return err
}
}
Expand Down Expand Up @@ -175,8 +177,14 @@ func computeAvailableHashes(
ctx context.Context,
hasher targethasher.SourceHasher,
targets map[string]*targethasher.Target,
excludedRegex []*regexp.Regexp,
) error {
for name, target := range targets {
if isExcludedTarget(name, excludedRegex) {
target.Hash = []byte{}
target.HashWithoutDeps = []byte{}
continue
}
var hash []byte
var hashWithoutDeps []byte
switch target.RuleType {
Expand Down Expand Up @@ -210,7 +218,7 @@ func computeAvailableHashes(
}

// computeHashes computes hashes recursively for the given target ID.
func (g *OptimizedGraph) computeHashes(ctx context.Context, id int) ([]byte, error) {
func (g *OptimizedGraph) computeHashes(ctx context.Context, id int, excludedRegex []*regexp.Regexp) ([]byte, error) {
if ctx.Err() != nil {
return nil, context.Cause(ctx)
}
Expand All @@ -227,6 +235,12 @@ func (g *OptimizedGraph) computeHashes(ctx context.Context, id int) ([]byte, err
return target.Hash, nil
}

if isExcludedTarget(g.TargetIDToString[id], excludedRegex) {
target.Hash = []byte{}
target.HashWithoutDeps = []byte{}
return target.Hash, nil
}

// mark as visiting to handle cycles
target.Hash = []byte{}
var hash []byte
Expand All @@ -239,7 +253,7 @@ func (g *OptimizedGraph) computeHashes(ctx context.Context, id int) ([]byte, err
singleDep = dep
break
}
dephash, err := g.computeHashes(ctx, singleDep)
dephash, err := g.computeHashes(ctx, singleDep, excludedRegex)
if err != nil {
return nil, err
}
Expand All @@ -255,7 +269,7 @@ func (g *OptimizedGraph) computeHashes(ctx context.Context, id int) ([]byte, err
return strings.Compare(g.TargetIDToString[i], g.TargetIDToString[j])
})
for _, dep := range depIDs {
dephash, err := g.computeHashes(ctx, dep)
dephash, err := g.computeHashes(ctx, dep, excludedRegex)
if err != nil {
return nil, err
}
Expand All @@ -268,3 +282,12 @@ func (g *OptimizedGraph) computeHashes(ctx context.Context, id int) ([]byte, err
}
return hash, nil
}

func isExcludedTarget(name string, excludedRegex []*regexp.Regexp) bool {
for _, re := range excludedRegex {
if re.MatchString(name) {
return true
}
}
return false
}
137 changes: 123 additions & 14 deletions core/itg/graph/update_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ package graph
import (
"context"
"crypto/sha1"
"regexp"
"testing"

buildpb "github.com/bazelbuild/buildtools/build_proto"
Expand Down Expand Up @@ -57,7 +58,7 @@ func TestComputeAvailableHashes(t *testing.T) {
},
}

require.NoError(t, computeAvailableHashes(ctx, hasher, targets))
require.NoError(t, computeAvailableHashes(ctx, hasher, targets, nil))
assert.Equal(t, expected, targets[name].Hash)
assert.Equal(t, ctx, hasher.ctx)
})
Expand All @@ -69,7 +70,7 @@ func TestComputeAvailableHashes(t *testing.T) {
name: {Name: name, RuleType: targethasher.PackageGroup},
}

require.NoError(t, computeAvailableHashes(context.Background(), &fakeSourceHasher{}, targets))
require.NoError(t, computeAvailableHashes(context.Background(), &fakeSourceHasher{}, targets, nil))

h := sha1.New()
h.Write([]byte(name))
Expand All @@ -83,7 +84,7 @@ func TestComputeAvailableHashes(t *testing.T) {
name: {Name: name, RuleType: targethasher.ExternalRuleType},
}

require.NoError(t, computeAvailableHashes(context.Background(), &fakeSourceHasher{}, targets))
require.NoError(t, computeAvailableHashes(context.Background(), &fakeSourceHasher{}, targets, nil))
assert.Nil(t, targets[name].Hash, "external rule targets should not get a hash here")
})

Expand All @@ -94,7 +95,7 @@ func TestComputeAvailableHashes(t *testing.T) {
name: {Name: name, RuleType: targethasher.GeneratedFileType},
}

require.NoError(t, computeAvailableHashes(context.Background(), &fakeSourceHasher{}, targets))
require.NoError(t, computeAvailableHashes(context.Background(), &fakeSourceHasher{}, targets, nil))
assert.Nil(t, targets[name].Hash, "generated file hash is resolved later")
})

Expand All @@ -111,7 +112,7 @@ func TestComputeAvailableHashes(t *testing.T) {
},
}

require.NoError(t, computeAvailableHashes(context.Background(), &fakeSourceHasher{}, targets))
require.NoError(t, computeAvailableHashes(context.Background(), &fakeSourceHasher{}, targets, nil))
assert.NotNil(t, targets[name].HashWithoutDeps, "rule should have HashWithoutDeps after hashing")
assert.Nil(t, targets[name].Hash, "full hash is not computed here — deps are needed")
})
Expand All @@ -123,9 +124,85 @@ func TestComputeAvailableHashes(t *testing.T) {
"//pkg:f": {Name: "//pkg:f", RuleType: targethasher.SourceFileType, SourceFile: &buildpb.SourceFile{}},
}

err := computeAvailableHashes(context.Background(), hasher, targets)
err := computeAvailableHashes(context.Background(), hasher, targets, nil)
assert.Error(t, err)
})

t.Run("excluded source file target is skipped without calling hasher", func(t *testing.T) {
t.Parallel()
name := "//pkg:secret.go"
hasher := &fakeSourceHasher{err: assert.AnError} // would fail the test if called
targets := map[string]*targethasher.Target{
name: {Name: name, RuleType: targethasher.SourceFileType, SourceFile: &buildpb.SourceFile{}},
}

err := computeAvailableHashes(context.Background(), hasher, targets, []*regexp.Regexp{regexp.MustCompile("secret")})
require.NoError(t, err)
assert.Equal(t, []byte{}, targets[name].Hash)
assert.Equal(t, []byte{}, targets[name].HashWithoutDeps)
assert.Nil(t, hasher.ctx, "hasher should never be invoked for an excluded target")
})

t.Run("excluded rule target gets empty hash instead of HashWithoutDeps", func(t *testing.T) {
t.Parallel()
name := "//pkg:lib"
ruleName := name
ruleClass := "go_library"
targets := map[string]*targethasher.Target{
name: {
Name: name,
RuleType: "go_library",
Rule: &buildpb.Rule{Name: &ruleName, RuleClass: &ruleClass},
},
}

err := computeAvailableHashes(context.Background(), &fakeSourceHasher{}, targets, []*regexp.Regexp{regexp.MustCompile("//pkg:lib")})
require.NoError(t, err)
assert.Equal(t, []byte{}, targets[name].Hash)
assert.Equal(t, []byte{}, targets[name].HashWithoutDeps)
})

t.Run("non-matching excludedRegex leaves hashing unaffected", func(t *testing.T) {
t.Parallel()
name := "//pkg:__pkg__"
targets := map[string]*targethasher.Target{
name: {Name: name, RuleType: targethasher.PackageGroup},
}

err := computeAvailableHashes(context.Background(), &fakeSourceHasher{}, targets, []*regexp.Regexp{regexp.MustCompile("no-match")})
require.NoError(t, err)

h := sha1.New()
h.Write([]byte(name))
assert.Equal(t, h.Sum(nil), targets[name].Hash)
})
}

func TestIsExcludedTarget(t *testing.T) {
t.Parallel()

t.Run("empty regex list never excludes", func(t *testing.T) {
t.Parallel()
assert.False(t, isExcludedTarget("//pkg:anything", nil))
})

t.Run("matches when any regex in the list matches", func(t *testing.T) {
t.Parallel()
regexes := []*regexp.Regexp{
regexp.MustCompile("^//other:"),
regexp.MustCompile("^//pkg:target$"),
}
assert.True(t, isExcludedTarget("//pkg:target", regexes))
})

t.Run("returns false when no regex matches", func(t *testing.T) {
t.Parallel()
regexes := []*regexp.Regexp{
regexp.MustCompile("^//other:"),
regexp.MustCompile("^//another:"),
}
assert.False(t, isExcludedTarget("//pkg:target", regexes))
})
}

// --- computeHashes ---
Expand All @@ -143,7 +220,7 @@ func TestComputeHashes(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()

_, err := g.computeHashes(ctx, aID)
_, err := g.computeHashes(ctx, aID, nil)
assert.ErrorIs(t, err, context.Canceled)
})

Expand All @@ -155,7 +232,7 @@ func TestComputeHashes(t *testing.T) {
})
aID := g.TargetNameToID["//pkg:a"]

got, err := g.computeHashes(context.Background(), aID)
got, err := g.computeHashes(context.Background(), aID, nil)
require.NoError(t, err)
assert.Equal(t, hash, got)
})
Expand All @@ -164,7 +241,7 @@ func TestComputeHashes(t *testing.T) {
t.Parallel()
g := OptimizeGraph(nil)

_, err := g.computeHashes(context.Background(), 9999)
_, err := g.computeHashes(context.Background(), 9999, nil)
assert.Error(t, err)
})

Expand All @@ -176,7 +253,7 @@ func TestComputeHashes(t *testing.T) {
})
id := g.TargetNameToID["//external:repo"]

got, err := g.computeHashes(context.Background(), id)
got, err := g.computeHashes(context.Background(), id, nil)
require.NoError(t, err)
assert.Equal(t, hash, got)
})
Expand All @@ -188,7 +265,7 @@ func TestComputeHashes(t *testing.T) {
})
id := g.TargetNameToID["//pkg:f.go"]

_, err := g.computeHashes(context.Background(), id)
_, err := g.computeHashes(context.Background(), id, nil)
assert.Error(t, err, "source file should already have its hash set")
})

Expand All @@ -206,7 +283,7 @@ func TestComputeHashes(t *testing.T) {
g.OptimizedTargets[ruleID].Hash = depHash
g.OptimizedTargets[genID].Hash = nil // force recompute

got, err := g.computeHashes(context.Background(), genID)
got, err := g.computeHashes(context.Background(), genID, nil)
require.NoError(t, err)
assert.Equal(t, depHash, got, "generated file should inherit its dep's hash")
})
Expand All @@ -223,7 +300,7 @@ func TestComputeHashes(t *testing.T) {
// lib has no final hash yet (only HashWithoutDeps)
g.OptimizedTargets[libID].Hash = nil

got, err := g.computeHashes(context.Background(), libID)
got, err := g.computeHashes(context.Background(), libID, nil)
require.NoError(t, err)

// Expected: sha1(hwod || depHash) (single dep, already sorted)
Expand All @@ -232,6 +309,38 @@ func TestComputeHashes(t *testing.T) {
h.Write(depHash)
assert.Equal(t, h.Sum(nil), got)
})

t.Run("excluded target short-circuits before dep traversal", func(t *testing.T) {
t.Parallel()
g := OptimizeGraph(map[string]*targethasher.Target{
"//pkg:lib": {
Name: "//pkg:lib",
RuleType: "go_library",
HashWithoutDeps: []byte{0x01},
Deps: []string{"//pkg:missing"}, // dangling: never added to the graph
},
})
libID := g.TargetNameToID["//pkg:lib"]
g.OptimizedTargets[libID].Hash = nil // force recompute path

got, err := g.computeHashes(context.Background(), libID, []*regexp.Regexp{regexp.MustCompile("//pkg:lib")})
require.NoError(t, err, "excluded targets should short-circuit before resolving deps")
assert.Equal(t, []byte{}, got)

// Sanity check: without the exclusion, the same dangling dep does error.
g2 := OptimizeGraph(map[string]*targethasher.Target{
"//pkg:lib": {
Name: "//pkg:lib",
RuleType: "go_library",
HashWithoutDeps: []byte{0x01},
Deps: []string{"//pkg:missing"},
},
})
libID2 := g2.TargetNameToID["//pkg:lib"]
g2.OptimizedTargets[libID2].Hash = nil
_, err = g2.computeHashes(context.Background(), libID2, nil)
assert.Error(t, err, "unexcluded target with a dangling dep should still error")
})
}

func TestComputeInvalidatedHashesCycleOrderInvariance(t *testing.T) {
Expand Down Expand Up @@ -316,7 +425,7 @@ func TestComputeInvalidatedHashesCycleOrderInvariance(t *testing.T) {
invalidated.Insert(graph.TargetNameToID[name])
}

err := graph.computeInvalidatedHashes(ctx, invalidated)
err := graph.computeInvalidatedHashes(ctx, invalidated, nil)
require.NoError(rt, err)
for name, expected := range expectedHashes {
actual := graph.OptimizedTargets[graph.TargetNameToID[name]].Hash
Expand Down
Loading