feat(eval): exact scoring baseline for generic-feature migration

.github/workflows/ci.yml

@@ -27,6 +27,9 @@ concurrency:
   group: ci-${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  GOAMD64: v1
+
 jobs:
   build-and-unit-test:
     name: build, vet, unit tests
@@ -42,7 +45,15 @@ jobs:
       - name: lint (gofmt + go vet)
         run: make lint
       - name: unit tests (no DB)
+        env:
+          ABUSEKIT_GOLDEN_EXPECT_PROFILE: amd64-fma
         run: make test
+      - name: exact golden replay without optional FMA
+        env:
+          GOAMD64: v1
+          GODEBUG: cpu.fma=off
+          ABUSEKIT_GOLDEN_EXPECT_PROFILE: amd64-no-fma
+        run: go test ./eval ./cmd/abusekit -run TestGolden -count=1
 
   gate:
     name: eval harness gate
@@ -90,3 +101,17 @@ jobs:
           # report green while testing nothing.
           ABUSEKIT_REQUIRE_DB: "1"
         run: make test-db
+
+  golden-arm64:
+    name: exact golden replay (ARM64)
+    runs-on: ubuntu-24.04-arm
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache-dependency-path: go.sum
+      - name: ARM64 baseline and mutation tests
+        env:
+          ABUSEKIT_GOLDEN_EXPECT_PROFILE: arm64
+        run: go test ./eval ./cmd/abusekit -run TestGolden -count=1

README.md

@@ -214,3 +214,18 @@ targets, and `AGENTS.md` for the worktree/lint/corpus conventions that apply to
 go run ./cmd/abusekit migrate --database-url postgres://e2a:e2a@localhost:5433/abusekit_dev
 go run ./cmd/abusekit serve --check --database-url postgres://e2a:e2a@localhost:5433/abusekit_dev
 ```
+
+### Exact migration replay
+
+The golden replay checks every committed event fixture and synthetic subject
+after each event and scheduled rescore:
+
+```sh
+abusekit eval --golden --brands-extra eval/fixtures/test_brands.yaml \
+  --golden-check eval/golden/reference-flat.jsonl
+```
+
+On ARM64, select `eval/golden/reference-flat-arm64.jsonl`; on AMD64 without FMA,
+select `eval/golden/reference-flat-amd64-no-fma.jsonl`. The replay pins
+float64 bits, rule hashes, scorer versions, and rescore times without a database
+or vendor calls. See [the baseline contract](eval/golden/README.md).

cmd/abusekit/eval_cmd.go

@@ -22,6 +22,8 @@ import (
 // convention (envOr) so a deployment's existing ABUSEKIT_RULES_CONFIG
 // etc. work unchanged for the harness too.
 type evalFlags struct {
+	golden      bool
+	goldenCheck string
 	ruleConfigPaths
 	brandsExtraPath string
 	webmailPath     string
@@ -47,6 +49,8 @@ var errHelp = errors.New("eval: help requested")
 func parseEvalFlags(args []string) (evalFlags, error) {
 	fs := flag.NewFlagSet("eval", flag.ContinueOnError)
 	var f evalFlags
+	fs.BoolVar(&f.golden, "golden", false, "replay every event and timer with exact float bits (local scorers only)")
+	fs.StringVar(&f.goldenCheck, "golden-check", "", "compare golden JSONL against this reference; exit 1 on drift")
 	fs.StringVar(&f.rulesPath, "rules", envOr("ABUSEKIT_RULES_CONFIG", "config/rules.yaml"), "path to rules.yaml")
 	fs.StringVar(&f.vendorsPath, "vendors", envOr("ABUSEKIT_VENDORS_CONFIG", "config/vendors.yaml"), "path to vendors.yaml")
 	fs.StringVar(&f.weightsPath, "weights", envOr("ABUSEKIT_LOCAL_WEIGHTS", "config/local_weights.yaml"), "path to the local scorer's weights YAML")
@@ -75,6 +79,28 @@ func parseEvalFlags(args []string) (evalFlags, error) {
 		}
 		return evalFlags{}, exitCode2(err)
 	}
+	if f.golden {
+		if fs.NArg() != 0 {
+			return evalFlags{}, exitCode2(fmt.Errorf("unexpected positional arguments in golden mode"))
+		}
+		var incompatible string
+		fs.Visit(func(v *flag.Flag) {
+			switch v.Name {
+			case "labels", "rule", "scorer", "slice", "split", "cassettes", "record", "skip-invalid", "floors", "prompt-version":
+				incompatible = v.Name
+			}
+		})
+		if incompatible != "" {
+			return evalFlags{}, exitCode2(fmt.Errorf("--%s is incompatible with --golden", incompatible))
+		}
+		if f.dataset == "" {
+			f.dataset = "eval/fixtures"
+		}
+		return f, nil
+	}
+	if f.goldenCheck != "" {
+		return evalFlags{}, exitCode2(fmt.Errorf("--golden-check requires --golden"))
+	}
 	if f.dataset == "" {
 		return evalFlags{}, exitCode2(fmt.Errorf("--dataset is required"))
 	}
@@ -121,6 +147,9 @@ func runEval(args []string) error {
 	if err != nil {
 		return exitCode2(fmt.Errorf("load webmail config: %w", err))
 	}
+	if f.golden {
+		return runGolden(f, cfg, brands, webmail)
+	}
 	rule, ok := ruleByName(cfg, f.rule)
 	if !ok {
 		return exitCode2(fmt.Errorf("unknown rule %q (known: %s)", f.rule, strings.Join(ruleNames(cfg), ", ")))

cmd/abusekit/golden_cmd.go

@@ -0,0 +1,173 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/tokencanopy/abusekit/eval"
+	"github.com/tokencanopy/abusekit/internal/config"
+	"github.com/tokencanopy/abusekit/internal/feature"
+)
+
+func runGolden(f evalFlags, cfg *config.Config, brands feature.BrandSet, webmail feature.WebmailSet) error {
+	info, err := os.Stat(f.dataset)
+	if err != nil {
+		return exitCode2(err)
+	}
+	if err := validateGoldenOutput(f, info.IsDir()); err != nil {
+		return exitCode2(err)
+	}
+	var out bytes.Buffer
+	if info.IsDir() {
+		err = eval.WriteGoldenFixtures(context.Background(), &out, f.dataset, cfg, brands, webmail)
+	} else {
+		input, openErr := os.Open(f.dataset)
+		if openErr != nil {
+			return exitCode2(openErr)
+		}
+		err = eval.WriteGolden(context.Background(), &out, filepath.Base(f.dataset), input, cfg, brands, webmail)
+		input.Close()
+	}
+	if err != nil {
+		return exitCode2(err)
+	}
+	if f.goldenCheck != "" {
+		want, err := os.ReadFile(f.goldenCheck)
+		if err != nil {
+			return exitCode2(err)
+		}
+		if !bytes.Equal(out.Bytes(), want) {
+			return exitCode1(fmt.Errorf("golden drift from %s: first differing line %d", f.goldenCheck, firstDifferentLine(out.Bytes(), want)))
+		}
+	}
+	if f.out != "" {
+		if err := writeGoldenFile(f.out, out.Bytes()); err != nil {
+			return exitCode2(err)
+		}
+	} else if f.goldenCheck == "" {
+		if _, err := os.Stdout.Write(out.Bytes()); err != nil {
+			return exitCode2(err)
+		}
+	}
+	return nil
+}
+
+func firstDifferentLine(a, b []byte) int {
+	n := len(a)
+	if len(b) < n {
+		n = len(b)
+	}
+	line := 1
+	for i := 0; i < n; i++ {
+		if a[i] != b[i] {
+			return line
+		}
+		if a[i] == '\n' {
+			line++
+		}
+	}
+	return line
+}
+
+// resolvedPath also resolves parent-directory symlinks for a new output file.
+func resolvedPath(path string) (string, error) {
+	abs, err := filepath.Abs(path)
+	if err != nil {
+		return "", err
+	}
+	resolved, err := filepath.EvalSymlinks(abs)
+	if err == nil {
+		return resolved, nil
+	}
+	if !errors.Is(err, os.ErrNotExist) {
+		return "", err
+	}
+	parent := filepath.Dir(abs)
+	if parent == abs {
+		return "", err
+	}
+	parent, err = resolvedPath(parent)
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(parent, filepath.Base(abs)), nil
+}
+
+func validateGoldenOutput(f evalFlags, directory bool) error {
+	if f.out == "" {
+		return nil
+	}
+	output, err := resolvedPath(f.out)
+	if err != nil {
+		return err
+	}
+	inputs := []string{f.dataset, f.goldenCheck, f.rulesPath, f.vendorsPath, f.weightsPath, f.brandsPath, f.brandsExtraPath, f.webmailPath}
+	if directory {
+		dir, err := resolvedPath(f.dataset)
+		if err != nil {
+			return err
+		}
+		rel, err := filepath.Rel(dir, output)
+		if err != nil {
+			return err
+		}
+		if rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
+			return fmt.Errorf("golden output must be outside the fixture directory")
+		}
+		paths, err := filepath.Glob(filepath.Join(f.dataset, "*.jsonl"))
+		if err != nil {
+			return err
+		}
+		inputs = append(inputs, paths...)
+		inputs = append(inputs, filepath.Join(f.dataset, "synthetic", "events.jsonl"))
+	}
+	outputInfo, err := os.Stat(f.out)
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+	for _, input := range inputs {
+		if input == "" {
+			continue
+		}
+		resolved, err := resolvedPath(input)
+		if err != nil {
+			return err
+		}
+		inputInfo, err := os.Stat(input)
+		if err != nil && !errors.Is(err, os.ErrNotExist) {
+			return err
+		}
+		if output == resolved || (inputInfo != nil && outputInfo != nil && os.SameFile(inputInfo, outputInfo)) {
+			return fmt.Errorf("golden output must differ from every input and reference")
+		}
+	}
+	return nil
+}
+
+func writeGoldenFile(path string, data []byte) error {
+	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+		return err
+	}
+	f, err := os.CreateTemp(filepath.Dir(path), ".golden-*")
+	if err != nil {
+		return err
+	}
+	defer os.Remove(f.Name())
+	if _, err = f.Write(data); err != nil {
+		f.Close()
+		return err
+	}
+	if err = f.Chmod(0644); err != nil {
+		f.Close()
+		return err
+	}
+	if err = f.Close(); err != nil {
+		return err
+	}
+	return os.Rename(f.Name(), path)
+}

cmd/abusekit/golden_cmd_test.go

@@ -0,0 +1,98 @@
+package main
+
+import (
+	"errors"
+	"math"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/tokencanopy/abusekit/internal/model/local"
+
+	"github.com/tokencanopy/abusekit/eval"
+)
+
+func goldenArgs(t *testing.T, extra ...string) []string {
+	t.Helper()
+	root := repoRoot(t)
+	return append([]string{"--golden", "--dataset", filepath.Join(root, "eval", "fixtures"), "--rules", filepath.Join(root, "config", "rules.yaml"), "--vendors", filepath.Join(root, "config", "vendors.yaml"), "--weights", filepath.Join(root, "config", "local_weights.yaml"), "--brands", filepath.Join(root, "config", "brands.yaml"), "--brands-extra", filepath.Join(root, "eval", "fixtures", "test_brands.yaml"), "--webmail", filepath.Join(root, "config", "webmail.yaml")}, extra...)
+}
+
+func TestGoldenLastBitWeightMutationFails(t *testing.T) {
+	root := repoRoot(t)
+	referenceName := "reference-flat.jsonl"
+	if profile := eval.GoldenProfile(); profile != "amd64-fma" {
+		referenceName = "reference-flat-" + profile + ".jsonl"
+	}
+	reference := filepath.Join(root, "eval", "golden", referenceName)
+	if err := runEval(goldenArgs(t, "--golden-check", reference)); err != nil {
+		t.Fatalf("unmodified weights must pass first: %v", err)
+	}
+	path := filepath.Join(root, "config", "local_weights.yaml")
+	w, err := local.LoadWeightsFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	old := w.Weight["resource_velocity_1h"]
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	changed := strings.Replace(string(raw), "resource_velocity_1h: "+strconv.FormatFloat(old, 'g', -1, 64), "resource_velocity_1h: "+strconv.FormatFloat(math.Float64frombits(math.Float64bits(old)^1), 'g', -1, 64), 1)
+	if changed == string(raw) {
+		t.Fatal("test did not mutate weight")
+	}
+	mutated := filepath.Join(t.TempDir(), "weights.yaml")
+	os.WriteFile(mutated, []byte(changed), 0600)
+	err = runEval(goldenArgs(t, "--weights", mutated, "--golden-check", reference))
+	var exit *exitError
+	if !errors.As(err, &exit) || exit.code != 1 || !strings.Contains(err.Error(), "golden drift") {
+		t.Fatalf("want golden drift exit 1, got %v", err)
+	}
+}
+
+func TestGoldenFlagsRejectIgnoredOptions(t *testing.T) {
+	for _, args := range [][]string{{"--golden", "--skip-invalid"}, {"--golden", "--scorer", "fake"}, {"--golden-check", "x"}} {
+		if _, err := parseEvalFlags(args); err == nil {
+			t.Fatalf("accepted %v", args)
+		}
+	}
+}
+
+func TestGoldenOutputCannotOverwriteSources(t *testing.T) {
+	for _, kind := range []string{"directory-fixture", "symlink", "hardlink"} {
+		t.Run(kind, func(t *testing.T) {
+			dir := t.TempDir()
+			fixtures := filepath.Join(dir, "fixtures")
+			os.MkdirAll(filepath.Join(fixtures, "synthetic"), 0755)
+			src := filepath.Join(fixtures, "case.jsonl")
+			data := []byte(`{"id":"x","subject":"acct_x","type":"subject.created","at":"2031-01-01T00:00:00Z","data":{}}` + "\n")
+			os.WriteFile(src, data, 0600)
+			os.WriteFile(filepath.Join(fixtures, "synthetic", "events.jsonl"), data, 0600)
+			dataset, output := fixtures, src
+			if kind != "directory-fixture" {
+				dataset = src
+				output = filepath.Join(dir, "alias.jsonl")
+				var err error
+				if kind == "symlink" {
+					err = os.Symlink(src, output)
+				} else {
+					err = os.Link(src, output)
+				}
+				if err != nil {
+					t.Fatal(err)
+				}
+			}
+			err := runEval(goldenArgs(t, "--dataset", dataset, "--out", output))
+			if err == nil {
+				t.Fatal("accepted output alias of an input")
+			}
+			got, _ := os.ReadFile(src)
+			if string(got) != string(data) {
+				t.Fatal("input was modified")
+			}
+		})
+	}
+}