chore(release-prep): cleanup links in test fixtures & enforce CodeQL eligibility checks

.gitattributes

@@ -0,0 +1,2 @@
+# Recorded by scripts/regenerate-corpus-fixtures.sh and never edited by hand; collapsed in GitHub diffs.
+testdata/corpus/fixtures/**/*.sarif linguist-generated=true

cmd/codeql-abom/check.go

@@ -57,7 +57,9 @@ func runCheck(ctx context.Context, args []string, stdout, stderr io.Writer) erro
 			return fmt.Errorf("check: %w", err)
 		}
 		github = &lazyGitHub{build: func() *catalog.Client {
-			return checkReader(ctx, *apiURL, readerTokens(app), processPools)
+			c := checkReader(ctx, *apiURL, readerTokens(app), processPools)
+			protectSources(s, c, stderr)
+			return c
 		}}
 	}
 	build := checkBuild()

cmd/codeql-abom/codescanning.go

@@ -20,5 +20,8 @@ func wireCodeScanning(ctx context.Context, loop *supervisor.Supervisor, s store.
 	// The build a refused read falls back to costs less than waiting out a rate limit.
 	client.MaxRetries = 0
 	client.REST = processPools.Pool(work.PoolFetching, work.UnitREST)
+	if policy := loop.Expander.SourcePolicy; policy != nil {
+		client.SourcePolicy = policy.Check
+	}
 	loop.Expander.CodeScanning = client
 }

cmd/codeql-abom/discover.go

@@ -149,6 +149,7 @@ func githubClient(ctx context.Context, apiURL string, s store.Backend, tokens gh
 	c.Pace = githubPace
 	c.REST = processPools.Pool(work.PoolEnumeration, work.UnitREST)
 	c.GraphQL = processPools.Pool(work.PoolEnumeration, work.UnitGraphQL)
+	protectSources(s, c, os.Stderr)
 	return c
 }
 

cmd/codeql-abom/expand.go

@@ -324,7 +324,7 @@ func newExpander(
 		cleanup()
 		return nil, nil, nil, err
 	}
-	return &expand.Expander{
+	e := &expand.Expander{
 		Queue:    queue,
 		Store:    s,
 		Ingester: ingest.New(s, w),
@@ -341,7 +341,9 @@ func newExpander(
 		Cache:     cache,
 		Databases: databases,
 		OnEvent:   logFailures(stdout),
-	}, cache, cleanup, nil
+	}
+	wireSourcePolicy(context.Background(), e, s, defaultAPIURL(), cfg.tokens, stdout)
+	return e, cache, cleanup, nil
 }
 
 func runExpandRounds(

cmd/codeql-abom/serve.go

@@ -28,6 +28,7 @@ import (
 	"github.com/advanced-security/codeql-abom/internal/runner"
 	"github.com/advanced-security/codeql-abom/internal/serve"
 	"github.com/advanced-security/codeql-abom/internal/setup"
+	"github.com/advanced-security/codeql-abom/internal/sourcepolicy"
 	"github.com/advanced-security/codeql-abom/internal/store"
 	"github.com/advanced-security/codeql-abom/internal/supervisor"
 	"github.com/advanced-security/codeql-abom/internal/toolchain"
@@ -261,7 +262,11 @@ func newProcessor(
 	}
 	if reader != nil {
 		processor.Branches, processor.Databases = reader.ProtectedBranches, reader.FindDatabase
-		processor.Setup = setup.Reader{Store: s, Catalog: reader}.Read
+		var policy *sourcepolicy.Policy
+		if loop.Expander != nil {
+			policy = loop.Expander.SourcePolicy
+		}
+		processor.Setup = sourceSetupReader(s, policy, setup.Reader{Store: s, Catalog: reader}.Read)
 		processor.Dependencies = reader.CompareDependencies
 	}
 	if loop.Enumerate != nil {
@@ -369,6 +374,7 @@ func runServe(ctx context.Context, args []string, stdout, stderr io.Writer) (res
 		return fmt.Errorf("serve: %w", err)
 	}
 	wireTrees(ctx, loop, cfg.apiURL, tokens)
+	wireSourcePolicy(ctx, loop.Expander, s, cfg.apiURL, tokens, stderr)
 	wireCodeScanning(ctx, loop, s, cfg)
 	if err := primeServedStore(s, loop.Queue, stdout, stderr); err != nil {
 		return err
@@ -630,7 +636,11 @@ func newConfiguredServer(
 	srv.EnumerateBranches = reader.ProtectedBranches
 	srv.EnumerateRepository = reader.ReadRepository
 	setupReader := setup.Reader{Store: s, Catalog: reader}
-	srv.EnumerateSetup = setupReader.Read
+	if policy := loop.Expander.SourcePolicy; policy != nil {
+		reader.SourcePolicy = policy.Check
+		srv.InvalidateSources = policy.InvalidateAll
+	}
+	srv.EnumerateSetup = sourceSetupReader(s, loop.Expander.SourcePolicy, setupReader.Read)
 	srv.EnumerateConfigurations = setupReader.ReadConfigurations
 	srv.CodeScanningWrites = codeScanningWrites(ctx, cfg, s, setupReader)
 	srv.FindDatabase = reader.FindDatabase
@@ -640,7 +650,11 @@ func newConfiguredServer(
 	srv.GroupProperty = cfg.groupProperty
 	srv.QueueOwned = func() (int, error) { return expand.SeedOwnedRepositories(s, loop.Queue) }
 	srv.QueueClosure = loop.Queue.Closure
-	verifier := lockVerifier(s, tokens)
+	var sourceCheck func(context.Context, string) error
+	if policy := loop.Expander.SourcePolicy; policy != nil {
+		sourceCheck = policy.Check
+	}
+	verifier := lockVerifierWithPolicy(s, tokens, sourceCheck, cfg.apiURL)
 	fetchTree, locks := verifier.Fetch, processPools.Pool(work.PoolLocks, work.UnitGit)
 	verifier.Fetch = func(ctx context.Context, dest, owner, repo, commit string) error {
 		if err := locks.Take(ctx); err != nil {

cmd/codeql-abom/sourcepolicy.go

@@ -0,0 +1,103 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/url"
+	"strings"
+
+	"github.com/advanced-security/codeql-abom/internal/catalog"
+	"github.com/advanced-security/codeql-abom/internal/expand"
+	"github.com/advanced-security/codeql-abom/internal/ghapp"
+	"github.com/advanced-security/codeql-abom/internal/sourcepolicy"
+	"github.com/advanced-security/codeql-abom/internal/store"
+	"github.com/advanced-security/codeql-abom/internal/work"
+)
+
+func protectSources(s store.Backend, reader *catalog.Client, log io.Writer) *sourcepolicy.Policy {
+	p := &sourcepolicy.Policy{
+		Read: reader, Owned: s.RepositoryIsOwned,
+		Evidence: func(repository string) (string, error) {
+			rows, err := s.SetupClasses(repository)
+			if err != nil {
+				return "", err
+			}
+			for _, r := range rows {
+				switch {
+				case r.Unreadable == "" && r.State == "configured":
+					return "default setup is configured", nil
+				case r.ConfigurationDefaultSet == "enabled" &&
+					(r.AttachmentStatus == "attached" || r.AttachmentStatus == "enforced"):
+					return "an applied security configuration enables scanning", nil
+				case r.AnalysesUnreadable == "" && (r.AdvancedKey != "" || r.ActionsDefaultAt != ""):
+					return "code scanning has reported analyses", nil
+				}
+			}
+			return "", nil
+		},
+		Warn: func(repository, reason string) {
+			fmt.Fprintf(log, "source policy: %s: %s\n", repository, reason)
+		},
+	}
+	reader.SourcePolicy = p.Check
+	return p
+}
+
+func wireSourcePolicy(
+	ctx context.Context, e *expand.Expander, s store.Backend, apiURL string, tokens ghapp.TokenSource, log io.Writer,
+) {
+	reader := newReader(ctx, apiURL, tokens)
+	reader.Pace = githubPace
+	reader.REST = processPools.Pool(work.PoolFetching, work.UnitREST)
+	reader.MaxRetries = 0
+	p := protectSources(s, reader, log)
+	e.SourcePolicy = p
+	if checkout, ok := e.Fetcher.(*expand.Checkout); ok {
+		bindSourcePolicy(checkout, apiURL, p.Check)
+	}
+}
+
+func bindSourcePolicy(checkout *expand.Checkout, apiURL string, check func(context.Context, string) error) {
+	if apiURL == "" {
+		apiURL = ghapp.DefaultAPIURL
+	}
+	u, err := url.Parse(apiURL)
+	if err != nil || u.Host == "" || u.User != nil || (u.Scheme != "https" && u.Scheme != "http") {
+		checkout.SourcePolicy = func(context.Context, string) error {
+			return fmt.Errorf("source policy: configure an HTTP(S) GitHub API URL without embedded credentials")
+		}
+		return
+	}
+	checkout.BaseURL = u.Scheme + "://" + u.Host
+	if strings.EqualFold(u.Host, "api.github.com") {
+		checkout.BaseURL = "https://github.com"
+	}
+	checkout.SourcePolicy = check
+}
+
+func sourceSetupReader(s store.Backend, policy *sourcepolicy.Policy,
+	read func(context.Context, string) error,
+) func(context.Context, string) error {
+	return func(ctx context.Context, repository string) error {
+		if err := read(ctx, repository); err != nil {
+			return err
+		}
+		if policy == nil {
+			return nil
+		}
+		policy.Invalidate(repository)
+		skipped, err := s.SourcePolicySkipped(repository)
+		if err != nil || !skipped {
+			return err
+		}
+		if err := policy.Check(ctx, repository); err != nil {
+			if sourcepolicy.IsDenied(err) {
+				return nil
+			}
+			return err
+		}
+		_, err = s.RetrySourcePolicy(repository)
+		return err
+	}
+}

cmd/codeql-abom/sourcepolicy_corpus_test.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"context"
+	"io"
+	"os"
+	"reflect"
+	"slices"
+	"testing"
+
+	"github.com/advanced-security/codeql-abom/internal/sourcepolicy"
+	"github.com/advanced-security/codeql-abom/internal/store"
+)
+
+func TestRemoteSourcePolicyAgainstRecordedCorpus(t *testing.T) {
+	path := os.Getenv("ABOM_SOURCE_POLICY_CORPUS")
+	if path == "" {
+		t.Skip("set ABOM_SOURCE_POLICY_CORPUS to a real corpus backup for a read-only policy comparison")
+	}
+	s, err := store.OpenBackend(path, store.Options{ReadOnly: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer s.Close()
+	before, err := s.Summary()
+	if err != nil {
+		t.Fatal(err)
+	}
+	names, err := s.RepositoryNames()
+	if err != nil || len(names) == 0 {
+		t.Fatalf("corpus names = %d, %v", len(names), err)
+	}
+	reader := newReader(context.Background(), defaultAPIURL(), nil)
+	reader.MaxRetries = 0
+	p := protectSources(s, reader, io.Discard)
+	warnings := 0
+	p.Warn = func(_, _ string) { warnings++ }
+	sorted := make([]string, 0, len(names))
+	for _, name := range names {
+		sorted = append(sorted, name)
+	}
+	slices.Sort(sorted)
+	allowed, denied := 0, 0
+	for _, name := range sorted {
+		err := p.Check(context.Background(), name)
+		switch {
+		case sourcepolicy.IsDenied(err):
+			denied++
+			t.Log(err)
+		case err != nil:
+			t.Fatal(err)
+		default:
+			allowed++
+		}
+	}
+	after, err := s.Summary()
+	if err != nil || !reflect.DeepEqual(before, after) {
+		t.Fatalf("source checks changed corpus evidence: before=%+v after=%+v err=%v", before, after, err)
+	}
+	if allowed == 0 {
+		t.Fatal("no source allowed; the real positive path was not exercised")
+	}
+	t.Logf("Before: %d repositories had no sourcing guard. After: allowed=%d (warnings=%d), skipped=%d, API requests=%d",
+		len(sorted), allowed, warnings, denied, reader.Stats().Requests)
+	t.Logf("Corpus before/after unchanged: %+v", after)
+}