S2b: send-volume, webmail, recipient and subject-brand features

Makefile

@@ -42,6 +42,16 @@ fmt:
 # interval bound of the reference run)". Never touches Postgres, a
 # vendor, or the network: the local scorer needs no cassette (S4).
 #
+# --brands-extra eval/fixtures/test_brands.yaml (S2b's F9 TODO): the
+# committed corpus's abusive_subject_lure family mentions a FICTIONAL
+# brand in its subject lines (never a real one, per this repo's hygiene
+# rule for fabricated lure prose) — merging that test-only file in is
+# what lets subject_brand_match recognize it here, the same way it's
+# merged for every internal/worker replay fixture that needs a brand
+# match. --webmail defaults to config/webmail.yaml (abusekit eval's own
+# default, same as `serve`), which the abusive_webmail_blast family needs
+# no extra flag for.
+#
 # `go build ./...` (the `build` target above) deliberately writes no
 # binary when it matches more than one package (Go's own default), so
 # gate builds cmd/abusekit explicitly to a throwaway path instead of
@@ -59,6 +69,7 @@ gate:
 	@./.gate-abusekit eval \
 		--dataset eval/fixtures/synthetic/events.jsonl \
 		--labels eval/fixtures/synthetic/labels.jsonl \
+		--brands-extra eval/fixtures/test_brands.yaml \
 		--rule new_account_velocity --scorer local --slice full \
 		--floors eval/floors.yaml \
 		--out .gate-run.json; \

README.md

@@ -78,6 +78,22 @@ included both; a fix round found the erasure semantics unsafe to ship (see the P
 section) and pulled both back out, preserving that implementation on `feat/s3b-list-erasure` for a
 proper S3b design pass.
 
+### Configuration flags
+
+`cmd/abusekit serve`/`serve --check` load their config from a set of flags (each with an
+`ABUSEKIT_*` env var equivalent): `--rules` (`config/rules.yaml`), `--vendors`
+(`config/vendors.yaml`), `--weights` (the local scorer's `config/local_weights.yaml`), `--brands`
+(`config/brands.yaml`), `--keys` (required, no default), `--database-url` (required, no default).
+Two are S2b additions:
+
+- `--webmail` (env `ABUSEKIT_WEBMAIL_CONFIG`, default `config/webmail.yaml`) — the public list of
+  consumer webmail provider domains `webmail_recipient_share`/`webmail_sends_1h` match against.
+- `--brands-extra` (env `ABUSEKIT_BRANDS_EXTRA_CONFIG`, **no default**) — an optional path to a
+  private, `config/brands.yaml`-shaped brand list, merged (`feature.MergeBrandSets`) alongside the
+  shipped public `--brands` list. Empty (the default) merges in nothing. This is how an operator
+  extends brand matching with names that shouldn't live in this public repo (AGENTS.md's data-
+  boundary rule) — e.g. a customer's own brand, or one under an NDA — without forking the binary.
+
 ### Go client (`pkg/abusekit`)
 
 ```go

cmd/abusekit/eval_cmd.go

@@ -23,18 +23,20 @@ import (
 // etc. work unchanged for the harness too.
 type evalFlags struct {
 	ruleConfigPaths
-	dataset       string
-	labels        string
-	rule          string
-	scorer        string
-	slice         string
-	split         string
-	out           string
-	cassettesDir  string
-	record        bool
-	skipInvalid   bool
-	floorsPath    string
-	promptVersion string
+	brandsExtraPath string
+	webmailPath     string
+	dataset         string
+	labels          string
+	rule            string
+	scorer          string
+	slice           string
+	split           string
+	out             string
+	cassettesDir    string
+	record          bool
+	skipInvalid     bool
+	floorsPath      string
+	promptVersion   string
 }
 
 // errHelp is returned by parseEvalFlags when -h/--help was given (fix
@@ -49,6 +51,12 @@ func parseEvalFlags(args []string) (evalFlags, error) {
 	fs.StringVar(&f.vendorsPath, "vendors", envOr("ABUSEKIT_VENDORS_CONFIG", "config/vendors.yaml"), "path to vendors.yaml")
 	fs.StringVar(&f.weightsPath, "weights", envOr("ABUSEKIT_LOCAL_WEIGHTS", "config/local_weights.yaml"), "path to the local scorer's weights YAML")
 	fs.StringVar(&f.brandsPath, "brands", envOr("ABUSEKIT_BRANDS_CONFIG", "config/brands.yaml"), "path to brands.yaml")
+	// Both of these mirror `serve`'s own flags exactly (same names, same
+	// env vars, same defaults — parseServeFlags in main.go) so the harness
+	// scores against the SAME brand/webmail configuration a real
+	// deployment runs on, not a silently different one.
+	fs.StringVar(&f.brandsExtraPath, "brands-extra", os.Getenv("ABUSEKIT_BRANDS_EXTRA_CONFIG"), "optional path to a private, brands.yaml-shaped extra brand list, merged with --brands (env ABUSEKIT_BRANDS_EXTRA_CONFIG; empty disables it)")
+	fs.StringVar(&f.webmailPath, "webmail", envOr("ABUSEKIT_WEBMAIL_CONFIG", "config/webmail.yaml"), "path to webmail.yaml")
 	fs.StringVar(&f.dataset, "dataset", "", "path to a label-snapshot corpus JSONL (design §4.6), or — with --labels also set — an event-replay events JSONL (required)")
 	fs.StringVar(&f.labels, "labels", "", "path to an event-replay labels JSONL; when set, --dataset is read as the matching events file (design §4.6's second corpus shape)")
 	fs.StringVar(&f.rule, "rule", "", "the rule name (from rules.yaml) to score against (required)")
@@ -99,6 +107,20 @@ func runEval(args []string) error {
 	if err != nil {
 		return exitCode2(fmt.Errorf("load rule config: %w", err))
 	}
+	// Mirrors boot's own sequence in main.go: brands-extra is optional (an
+	// empty path merges in nothing), webmail has a default the same as
+	// --brands does.
+	if f.brandsExtraPath != "" {
+		extra, err := feature.LoadBrandsFile(f.brandsExtraPath)
+		if err != nil {
+			return exitCode2(fmt.Errorf("load brands-extra config: %w", err))
+		}
+		brands = feature.MergeBrandSets(brands, extra)
+	}
+	webmail, err := feature.LoadWebmailFile(f.webmailPath)
+	if err != nil {
+		return exitCode2(fmt.Errorf("load webmail config: %w", err))
+	}
 	rule, ok := ruleByName(cfg, f.rule)
 	if !ok {
 		return exitCode2(fmt.Errorf("unknown rule %q (known: %s)", f.rule, strings.Join(ruleNames(cfg), ", ")))
@@ -109,7 +131,7 @@ func runEval(args []string) error {
 		return exitCode2(err)
 	}
 
-	dataset, datasetSHA, labelsSHA, skippedRows, err := loadEvalDataset(f.dataset, f.labels, rule.BenignLabel, brands, f.skipInvalid)
+	dataset, datasetSHA, labelsSHA, skippedRows, err := loadEvalDataset(f.dataset, f.labels, rule.BenignLabel, brands, webmail, f.skipInvalid)
 	if err != nil {
 		return exitCode2(err)
 	}
@@ -254,7 +276,7 @@ func resolveScorerNames(registry *model.Registry, flagValue string) ([]string, e
 // the partial Dataset (every row that DID parse) is returned alongside
 // the row errors instead of an error, for the caller to report as
 // `skipped_rows`.
-func loadEvalDataset(datasetPath, labelsPath, benignLabel string, brands feature.BrandSet, skipInvalid bool) (dataset eval.Dataset, datasetSHA, labelsSHA string, skipped []eval.RowError, err error) {
+func loadEvalDataset(datasetPath, labelsPath, benignLabel string, brands feature.BrandSet, webmail feature.WebmailSet, skipInvalid bool) (dataset eval.Dataset, datasetSHA, labelsSHA string, skipped []eval.RowError, err error) {
 	datasetSHA, err = eval.SHA256File(datasetPath)
 	if err != nil {
 		return eval.Dataset{}, "", "", nil, fmt.Errorf("hash --dataset %s: %w", datasetPath, err)
@@ -294,7 +316,7 @@ func loadEvalDataset(datasetPath, labelsPath, benignLabel string, brands feature
 	dataset, rowErrs, err := eval.LoadReplayDataset(eval.ReplayInput{
 		EventsPath: datasetPath, Events: eventsF,
 		LabelsPath: labelsPath, Labels: labelsF,
-	}, brands, benignLabel)
+	}, brands, webmail, benignLabel)
 	if err != nil {
 		if !skipInvalid {
 			return eval.Dataset{}, "", "", nil, schemaCLIError(rowErrs, err)

cmd/abusekit/eval_cmd_test.go

@@ -26,6 +26,7 @@ func syntheticCorpusArgs(t *testing.T, extra ...string) []string {
 		"--vendors", filepath.Join(root, "config", "vendors.yaml"),
 		"--weights", filepath.Join(root, "config", "local_weights.yaml"),
 		"--brands", filepath.Join(root, "config", "brands.yaml"),
+		"--webmail", filepath.Join(root, "config", "webmail.yaml"),
 		"--rule", "new_account_velocity",
 		"--scorer", "local",
 		"--slice", "full",
@@ -145,6 +146,7 @@ func TestRunEval_UnknownRuleAndScorerAreBadInput(t *testing.T) {
 		"--vendors", filepath.Join(root, "config", "vendors.yaml"),
 		"--weights", filepath.Join(root, "config", "local_weights.yaml"),
 		"--brands", filepath.Join(root, "config", "brands.yaml"),
+		"--webmail", filepath.Join(root, "config", "webmail.yaml"),
 	}
 	t.Run("unknown rule", func(t *testing.T) {
 		args := append(append([]string{}, base...), "--rule", "does_not_exist", "--scorer", "local")
@@ -284,6 +286,7 @@ func TestRunEval_SchemaErrorsReportOwnFileAndLine(t *testing.T) {
 		"--vendors", filepath.Join(root, "config", "vendors.yaml"),
 		"--weights", filepath.Join(root, "config", "local_weights.yaml"),
 		"--brands", filepath.Join(root, "config", "brands.yaml"),
+		"--webmail", filepath.Join(root, "config", "webmail.yaml"),
 		"--rule", "new_account_velocity", "--scorer", "local",
 		"--out", filepath.Join(t.TempDir(), "run.json"),
 	}
@@ -324,6 +327,7 @@ func TestRunEval_SkipInvalidWritesSkippedRows(t *testing.T) {
 		"--vendors", filepath.Join(root, "config", "vendors.yaml"),
 		"--weights", filepath.Join(root, "config", "local_weights.yaml"),
 		"--brands", filepath.Join(root, "config", "brands.yaml"),
+		"--webmail", filepath.Join(root, "config", "webmail.yaml"),
 		"--rule", "new_account_velocity", "--scorer", "local", "--skip-invalid",
 		"--out", out,
 	}
@@ -381,6 +385,7 @@ func TestRunEval_NullOptionalFieldsLoadAndDontChangeVerdicts(t *testing.T) {
 			"--vendors", filepath.Join(root, "config", "vendors.yaml"),
 			"--weights", filepath.Join(root, "config", "local_weights.yaml"),
 			"--brands", filepath.Join(root, "config", "brands.yaml"),
+			"--webmail", filepath.Join(root, "config", "webmail.yaml"),
 			"--rule", "new_account_velocity", "--scorer", "local",
 			"--out", out,
 		}

cmd/abusekit/main.go

@@ -90,16 +90,18 @@ func run(args []string) error {
 }
 
 type serveConfig struct {
-	check         bool
-	dev           bool
-	databaseURL   string
-	rulesPath     string
-	vendorsPath   string
-	weightsPath   string
-	brandsPath    string
-	keysPath      string
-	metricsListen string
-	listenAddr    string
+	check           bool
+	dev             bool
+	databaseURL     string
+	rulesPath       string
+	vendorsPath     string
+	weightsPath     string
+	brandsPath      string
+	brandsExtraPath string
+	webmailPath     string
+	keysPath        string
+	metricsListen   string
+	listenAddr      string
 }
 
 // minKeySecretBytes and devSecretPrefix are the fix round's B2 guards
@@ -124,6 +126,13 @@ func parseServeFlags(args []string) (serveConfig, error) {
 	fs.StringVar(&c.vendorsPath, "vendors", envOr("ABUSEKIT_VENDORS_CONFIG", "config/vendors.yaml"), "path to vendors.yaml")
 	fs.StringVar(&c.weightsPath, "weights", envOr("ABUSEKIT_LOCAL_WEIGHTS", "config/local_weights.yaml"), "path to the local scorer's weights YAML")
 	fs.StringVar(&c.brandsPath, "brands", envOr("ABUSEKIT_BRANDS_CONFIG", "config/brands.yaml"), "path to brands.yaml")
+	// S2b: brands-extra is OPTIONAL and has no default path at all (unlike
+	// --brands) — an operator's private brand list lives outside this
+	// public repo (AGENTS.md's data-boundary rule), so there is no
+	// checked-in file a default could ever point at. Empty (the default)
+	// means "no private brand list", not an error.
+	fs.StringVar(&c.brandsExtraPath, "brands-extra", os.Getenv("ABUSEKIT_BRANDS_EXTRA_CONFIG"), "optional path to a private, brands.yaml-shaped extra brand list, merged with --brands (env ABUSEKIT_BRANDS_EXTRA_CONFIG; empty disables it — S2b)")
+	fs.StringVar(&c.webmailPath, "webmail", envOr("ABUSEKIT_WEBMAIL_CONFIG", "config/webmail.yaml"), "path to webmail.yaml (S2b)")
 	// B2 fix round: NO default keys path. A silently-defaulted
 	// config/keys.yaml is exactly the fail-open behavior this guards
 	// against -- every deployment must say explicitly where its keys live.
@@ -218,6 +227,7 @@ func runServeWithContext(ctx context.Context, c serveConfig) error {
 		Config:    cfg,
 		Neighbors: deps.neighbors,
 		Brands:    deps.brands,
+		Webmail:   deps.webmail,
 		Metrics:   deps.metrics,
 		Budgets:   deps.budgets,
 		Logger:    slog.Default(),
@@ -258,7 +268,7 @@ func runServeWithContext(ctx context.Context, c serveConfig) error {
 	// requests first means an evaluate call that's already claimed a
 	// subject gets to finish its round through a still-running worker
 	// rather than racing its own shutdown.
-	apiSrv, apiAddr, err := startAPIServer(c.listenAddr, s, w, cfg, deps.keys, deps.neighbors, deps.brands)
+	apiSrv, apiAddr, err := startAPIServer(c.listenAddr, s, w, cfg, deps.keys, deps.neighbors, deps.brands, deps.webmail)
 	if err != nil {
 		return fmt.Errorf("start api server: %w", err)
 	}
@@ -299,13 +309,13 @@ const (
 // convention), returning (nil, "", nil) — a test constructing a
 // serveConfig by hand (leaving listenAddr at its zero value) gets no
 // listener at all, never a port collision.
-func startAPIServer(addr string, s *store.Store, w *worker.Worker, cfg *config.Config, keys map[string]config.Key, neighbors feature.Neighbors, brands feature.BrandSet) (*http.Server, string, error) {
+func startAPIServer(addr string, s *store.Store, w *worker.Worker, cfg *config.Config, keys map[string]config.Key, neighbors feature.Neighbors, brands feature.BrandSet, webmail feature.WebmailSet) (*http.Server, string, error) {
 	if addr == "" {
 		return nil, "", nil
 	}
 	srv, err := serve.New(serve.Deps{
 		Store: s, Worker: w, Config: cfg, Keys: keys,
-		Neighbors: neighbors, Brands: brands, Logger: slog.Default(),
+		Neighbors: neighbors, Brands: brands, Webmail: webmail, Logger: slog.Default(),
 	})
 	if err != nil {
 		return nil, "", fmt.Errorf("construct http server: %w", err)
@@ -381,6 +391,7 @@ const (
 type bootDeps struct {
 	neighbors feature.Neighbors
 	brands    feature.BrandSet
+	webmail   feature.WebmailSet
 	metrics   *worker.Metrics
 	budgets   *worker.Budgets
 	// keys is design §4.3's per-producer/operator credential set (S3),
@@ -406,6 +417,21 @@ func boot(ctx context.Context, c serveConfig) (*store.Store, *config.Config, boo
 	if err != nil {
 		return nil, nil, bootDeps{}, err
 	}
+	// S2b: brands-extra is optional (see parseServeFlags' own comment) —
+	// an empty path merges in nothing, MergeBrandSets(brands, BrandSet{})
+	// behaves identically to brands alone.
+	if c.brandsExtraPath != "" {
+		extra, err := feature.LoadBrandsFile(c.brandsExtraPath)
+		if err != nil {
+			return nil, nil, bootDeps{}, fmt.Errorf("load brands-extra config: %w", err)
+		}
+		brands = feature.MergeBrandSets(brands, extra)
+	}
+
+	webmail, err := feature.LoadWebmailFile(c.webmailPath)
+	if err != nil {
+		return nil, nil, bootDeps{}, fmt.Errorf("load webmail config: %w", err)
+	}
 
 	keysData, err := os.ReadFile(c.keysPath)
 	if err != nil {
@@ -433,6 +459,7 @@ func boot(ctx context.Context, c serveConfig) (*store.Store, *config.Config, boo
 	deps := bootDeps{
 		neighbors: feature.NewStoreNeighbors(s, feature.Config{}), // default link-kind policy (S1 fix round) until a tenant-specific override exists
 		brands:    brands,
+		webmail:   webmail,
 		budgets:   worker.NewPersistedBudgets(s, defaultPerAdapterDailyBudget, worker.DefaultPerSubjectDailyBudget, defaultPerTenantDailyBudget),
 		metrics:   metrics,
 		keys:      keys,

cmd/abusekit/main_test.go

@@ -38,6 +38,7 @@ func shippedConfig(t *testing.T) serveConfig {
 		vendorsPath: filepath.Join(root, "config", "vendors.yaml"),
 		weightsPath: filepath.Join(root, "config", "local_weights.yaml"),
 		brandsPath:  filepath.Join(root, "config", "brands.yaml"),
+		webmailPath: filepath.Join(root, "config", "webmail.yaml"),
 		keysPath:    filepath.Join(root, "config", "keys.yaml"),
 		dev:         true,
 	}
@@ -75,6 +76,12 @@ func TestParseServeFlags_Defaults(t *testing.T) {
 	if c.metricsListen != "127.0.0.1:9099" {
 		t.Errorf("metricsListen = %q, want the default 127.0.0.1:9099 (R8 round 2)", c.metricsListen)
 	}
+	if c.webmailPath != "config/webmail.yaml" {
+		t.Errorf("webmailPath = %q, want the default config/webmail.yaml (S2b)", c.webmailPath)
+	}
+	if c.brandsExtraPath != "" {
+		t.Errorf("brandsExtraPath = %q, want empty by default (S2b: no committed file for a private brand list to default to)", c.brandsExtraPath)
+	}
 }
 
 // TestParseServeFlags_RequiresKeysPath is B2: no default keys path — a
@@ -153,6 +160,54 @@ func TestBoot_RejectsInvalidRules(t *testing.T) {
 	}
 }
 
+// TestBoot_RejectsMissingBrandsExtraFile is S2b's analogue of
+// TestBoot_RejectsMissingRulesFile: --brands-extra, once set, is
+// validated the same way every other config path is — before ever
+// touching Postgres.
+func TestBoot_RejectsMissingBrandsExtraFile(t *testing.T) {
+	c := shippedConfig(t)
+	c.brandsExtraPath = filepath.Join(t.TempDir(), "does-not-exist.yaml")
+	if _, _, _, err := boot(context.Background(), c); err == nil {
+		t.Fatalf("expected boot to fail with a missing brands-extra file")
+	}
+}
+
+// TestBoot_RejectsMissingWebmailFile is S2b's analogue for --webmail.
+func TestBoot_RejectsMissingWebmailFile(t *testing.T) {
+	c := shippedConfig(t)
+	c.webmailPath = filepath.Join(t.TempDir(), "does-not-exist.yaml")
+	if _, _, _, err := boot(context.Background(), c); err == nil {
+		t.Fatalf("expected boot to fail with a missing webmail file")
+	}
+}
+
+// TestBoot_MergesBrandsExtra is S2b: --brands-extra, when set, actually
+// merges into the brand set boot constructs — matched here against a
+// brand name that config/brands.yaml does NOT ship, so a false pass
+// (the shipped list alone happening to already match) is impossible.
+// Needs Postgres (boot only returns populated deps on a full success);
+// skips cleanly like every other DB-backed test here.
+func TestBoot_MergesBrandsExtra(t *testing.T) {
+	c := shippedConfig(t)
+	c.databaseURL = testDBURL(t)
+
+	extra := filepath.Join(t.TempDir(), "brands-extra.yaml")
+	if err := os.WriteFile(extra, []byte("brands:\n  - name: Zzyzxcorp\n"), 0o644); err != nil {
+		t.Fatalf("write brands-extra file: %v", err)
+	}
+	c.brandsExtraPath = extra
+
+	s, _, deps, err := boot(context.Background(), c)
+	if err != nil {
+		t.Fatalf("boot: %v", err)
+	}
+	defer s.Close()
+
+	if !deps.brands.Matches("Zzyzxcorp") {
+		t.Errorf("expected --brands-extra's entry to be merged into boot's brand set")
+	}
+}
+
 // TestRunServe_CheckSucceeds is S17's end-to-end path: connect, migrate,
 // validate the shipped config, and return with no error and no blocking —
 // including closing the store's pool on the way out (defer'd inside
@@ -169,6 +224,7 @@ func TestRunServe_CheckSucceeds(t *testing.T) {
 		"--vendors", c.vendorsPath,
 		"--weights", c.weightsPath,
 		"--brands", c.brandsPath,
+		"--webmail", c.webmailPath,
 		"--keys", c.keysPath,
 	}
 	if err := runServe(args); err != nil {
@@ -225,12 +281,12 @@ func TestRunServeWithContext_ServesTheAPI(t *testing.T) {
 		t.Fatalf("boot: %v", err)
 	}
 
-	w, err := worker.New(worker.Deps{Store: s, Config: cfg, Neighbors: deps.neighbors, Brands: deps.brands})
+	w, err := worker.New(worker.Deps{Store: s, Config: cfg, Neighbors: deps.neighbors, Brands: deps.brands, Webmail: deps.webmail})
 	if err != nil {
 		s.Close()
 		t.Fatalf("worker.New: %v", err)
 	}
-	apiSrv, apiAddr, err := startAPIServer(c.listenAddr, s, w, cfg, deps.keys, deps.neighbors, deps.brands)
+	apiSrv, apiAddr, err := startAPIServer(c.listenAddr, s, w, cfg, deps.keys, deps.neighbors, deps.brands, deps.webmail)
 	if err != nil {
 		s.Close()
 		t.Fatalf("startAPIServer: %v", err)