feat(harbor): the agent's first baseline eval is budget-free

vero/src/vero/harbor/cli.py

@@ -120,8 +120,14 @@ def finalize_cmd(token_file, output):
     from vero.harbor.auth import read_admin_token
 
     token = read_admin_token(token_file)
-    reward = _request("POST", "/finalize", headers={"Authorization": f"Bearer {token}"})
+    resp = _request("POST", "/finalize", headers={"Authorization": f"Bearer {token}"})
+    # finalize returns {"rewards": {...}, "baseline": {...}}. Only the rewards are
+    # the reward.json payload the outer harness consumes; the baseline outcome is
+    # echoed to stdout (the trial's stdout survives teardown, the admin volume does
+    # not) so a baseline skip or failure is durably recorded. Tolerate the older
+    # bare-rewards shape so a mixed-version sidecar still writes a valid reward.json.
+    rewards = resp["rewards"] if isinstance(resp, dict) and "rewards" in resp else resp
     out = Path(output)
     out.parent.mkdir(parents=True, exist_ok=True)
-    out.write_text(json.dumps(reward))
-    click.echo(json.dumps(reward, indent=2))
+    out.write_text(json.dumps(rewards))
+    click.echo(json.dumps(resp, indent=2))

vero/src/vero/harbor/protocol.py

@@ -51,6 +51,10 @@ class StatusSummary:
     submit_enabled: bool
     # per (split, dataset_id): tier + whether the agent may evaluate it + remaining budget
     splits: list[dict] = field(default_factory=list)
+    # the seeded baseline sha and whether its one budget-free reference eval
+    # is still available (None/False when the task has no recorded baseline)
+    base_commit: str | None = None
+    free_baseline_available: bool = False
 
     def to_dict(self) -> dict:
         return asdict(self)
@@ -98,6 +102,8 @@ def build_status(
     submit_enabled: bool,
     budget: dict[tuple[str, str], SplitBudget],
     split_accesses: list[SplitAccess],
+    base_commit: str | None = None,
+    free_baseline_available: bool = False,
 ) -> StatusSummary:
     """Build the agent-facing status from the budget ledger + split tiers.
 
@@ -117,4 +123,9 @@ def build_status(
                 "remaining_run_budget": b.remaining_run_budget,
             }
         )
-    return StatusSummary(submit_enabled=submit_enabled, splits=splits)
+    return StatusSummary(
+        submit_enabled=submit_enabled,
+        splits=splits,
+        base_commit=base_commit,
+        free_baseline_available=free_baseline_available,
+    )

vero/src/vero/harbor/serve.py

@@ -71,6 +71,12 @@ class ServeConfig(BaseModel):
     # write it to <admin_volume>/baseline.json: makes regressions visible
     # (an optimized candidate can score WORSE than the untouched baseline).
     score_baseline: bool = False
+    # Total attempts for the finalize baseline eval (>=1): a transient nested-run
+    # failure once silently dropped the regression check.
+    baseline_score_attempts: int = 2
+    # auto_best never ships a candidate that fails to beat the untouched baseline
+    # on the selection split; it reverts to base_commit instead (needs base_commit).
+    auto_best_baseline_floor: bool = True
 
     # volumes / token
     agent_volume: str
@@ -221,6 +227,7 @@ async def build_components(config: ServeConfig) -> tuple[EvaluationSidecar, Veri
         agent_volume=Path(config.agent_volume),
         admin_volume=Path(config.admin_volume),
         submit_enabled=config.submit_enabled,
+        base_commit=config.base_commit,
     )
     verifier = Verifier(
         engine=engine,
@@ -232,6 +239,8 @@ async def build_components(config: ServeConfig) -> tuple[EvaluationSidecar, Veri
         selection_task=config.task,
         selection_dataset_id=config.dataset_id,
         score_baseline=config.score_baseline,
+        baseline_score_attempts=config.baseline_score_attempts,
+        auto_best_baseline_floor=config.auto_best_baseline_floor,
     )
 
     token = generate_token()

vero/src/vero/harbor/server.py

@@ -55,21 +55,48 @@ def __init__(
         agent_volume: Path,
         admin_volume: Path,
         submit_enabled: bool = False,
+        base_commit: str | None = None,
     ):
         self.engine = engine
         self.split_accesses = split_accesses
         self.agent_repo_path = Path(agent_repo_path)
         self.agent_volume = Path(agent_volume)
         self.admin_volume = Path(admin_volume)
         self.submit_enabled = submit_enabled
+        self.base_commit = base_commit
+        self._free_baseline_used = False
 
     # ------------------------------------------------------------------
     # Handlers (the HTTP layer resolves `admin` from auth and calls these)
     # ------------------------------------------------------------------
 
     async def evaluate(self, req: EvalRequest, *, admin: bool = False) -> EvalSummary:
         sha = await self._transfer_commit(req.commit)
-        exp = await self.engine.evaluate(replace(req, commit=sha), admin=admin)
+        # The agent's FIRST eval of the seeded baseline is budget-free. The
+        # baseline is the reference every candidate is implicitly compared to,
+        # yet it can never win selection (auto_best excludes base_commit), so
+        # metering it forces a choice between optimizing blind and paying a
+        # budgeted eval for a commit that cannot be selected (observed live:
+        # an optimizer that skipped the reference could not tell a no-op edit
+        # from an improvement and quit with budget unspent). Capped at one:
+        # later baseline evals debit normally, so free compute is bounded.
+        free_baseline = (
+            not admin
+            and self.base_commit is not None
+            and sha == self.base_commit
+            and not self._free_baseline_used
+        )
+        exp = await self.engine.evaluate(
+            replace(req, commit=sha), admin=admin or free_baseline
+        )
+        # Consume the free slot only after the eval actually succeeds. Setting it
+        # before the await would burn the one free baseline on a transient engine
+        # failure (timeout, infra), forcing the agent to pay for the retry, which is
+        # the exact failure mode this feature prevents. Safe in the single-threaded
+        # asyncio loop: no await runs between the check above and this write.
+        if free_baseline:
+            self._free_baseline_used = True
+        # Route with the agent's real tier even when the eval was unmetered.
         result_path = self._route_results(exp, admin=admin)
         budget_remaining = None
         if not admin:
@@ -99,6 +126,10 @@ def status(self) -> StatusSummary:
             submit_enabled=self.submit_enabled,
             budget=self.engine.budget.status(),
             split_accesses=self.split_accesses,
+            base_commit=self.base_commit,
+            free_baseline_available=(
+                self.base_commit is not None and not self._free_baseline_used
+            ),
         )
 
     # ------------------------------------------------------------------

vero/src/vero/harbor/verifier.py

@@ -53,6 +53,8 @@ def __init__(
         selection_dataset_id: str | None = None,
         rescore_top_k: int = 3,
         score_baseline: bool = False,
+        baseline_score_attempts: int = 2,
+        auto_best_baseline_floor: bool = True,
     ):
         self.engine = engine
         self.admin_volume = Path(admin_volume)
@@ -67,9 +69,27 @@ def __init__(
         self.selection_dataset_id = selection_dataset_id
         self.rescore_top_k = rescore_top_k
         self.score_baseline = score_baseline
+        # auto_best selection floor: never ship a candidate that fails to beat the
+        # untouched baseline on the selection split. Without it, auto_best (which
+        # excludes base_commit from the candidate pool) selects the least-bad
+        # candidate even when every candidate regressed, shipping a regression
+        # (observed live: a weak inner model, every candidate below baseline).
+        self.auto_best_baseline_floor = auto_best_baseline_floor
+        # Baseline scoring is retried this many times total before its outcome is
+        # reported as an error; the nested eval can fail transiently (a nested
+        # harbor run crashing right after a large eval), and a single blip must
+        # not silently drop the regression check.
+        self._baseline_score_attempts = max(1, baseline_score_attempts)
 
-    async def finalize(self) -> dict[str, float]:
-        """Select the commit and score it on every target -> {reward_key: score}.
+    async def finalize(self) -> dict:
+        """Select the commit, score it on every target, and score the baseline.
+
+        Returns a wrapper ``{"rewards": {reward_key: score}, "baseline": {...}}``.
+        ``rewards`` is the reward.json payload the outer harness consumes (the CLI
+        writes only that to reward.json); ``baseline`` is the outcome of baseline
+        scoring, surfaced here because it is otherwise invisible: the admin volume
+        it used to be written to does not survive teardown, and the finalize
+        response echoed to the trial's stdout is the only host-durable channel.
 
         A run in which the optimizer produced no scorable candidate (never
         submitted in ``submit`` mode; no non-baseline experiments on the
@@ -89,7 +109,8 @@ async def finalize(self) -> dict[str, float]:
                 len(self.targets),
                 default_minimum_score,
             )
-            return {t.reward_key: float(default_minimum_score) for t in self.targets}
+            rewards = {t.reward_key: float(default_minimum_score) for t in self.targets}
+            return {"rewards": rewards, "baseline": {"skipped": "no candidate commit"}}
         logger.info(f"Verifier selected commit {sha} (mode={self.reward_mode})")
         rewards: dict[str, float] = {}
         for target in self.targets:
@@ -104,56 +125,90 @@ async def finalize(self) -> dict[str, float]:
             rewards[target.reward_key] = (
                 float(score) if score is not None else default_minimum_score
             )
-        await self._maybe_score_baseline(rewards)
-        return rewards
+        baseline = await self._maybe_score_baseline(rewards)
+        return {"rewards": rewards, "baseline": baseline}
 
-    async def _maybe_score_baseline(self, rewards: dict[str, float]) -> None:
-        """Admin-score the unmodified baseline on every target and persist it.
+    async def _maybe_score_baseline(self, rewards: dict[str, float]) -> dict:
+        """Admin-score the unmodified baseline on every target and report it.
 
         An optimized candidate can score WORSE than the untouched baseline
         (observed live: a weak inner model went 0.3 -> 0.2 after optimization);
         without this, the regression is invisible because auto_best excludes the
-        baseline from selection and nothing else ever scores it. Written to
-        <admin_volume>/baseline.json (NOT into reward.json, whose keys the outer
-        harness consumes) and logged next to the candidate's rewards. Failures
-        here never fail the trial.
+        baseline from selection and nothing else ever scores it.
+
+        Returns a structured outcome (``{"scores": ...}`` / ``{"error": ...}`` /
+        ``{"skipped": ...}``) that ``finalize`` surfaces in its response, so a
+        skip or failure is durably recorded rather than lost. A live trial once
+        skipped this silently: the nested baseline eval failed transiently and
+        the only record (a log line) died with the container at teardown. So the
+        eval is retried once, and any failure is returned instead of swallowed.
+        Baseline scoring still never fails the trial (reward.json is unaffected).
+        A best-effort copy is also written to <admin_volume>/baseline.json for
+        in-cluster debugging while the sidecar is alive.
         """
         if not self.score_baseline:
-            return
+            return {"skipped": "score_baseline is disabled"}
         if not self.base_commit:
             # Misconfiguration must not be a silent no-op: the operator asked
             # for baseline scoring and would otherwise never learn it is off.
             logger.warning(
                 "score_baseline=True but base_commit is not set; skipping "
                 "baseline scoring."
             )
-            return
-        try:
-            baselines: dict[str, float] = {}
-            for target in self.targets:
-                exp = await self.engine.evaluate_admin(
-                    task=target.task,
-                    dataset_id=target.dataset_id,
-                    split=target.split,
-                    commit=self.base_commit,
-                    sample_ids=target.sample_ids,
-                )
-                score = exp.result.score()
-                baselines[target.reward_key] = (
-                    float(score) if score is not None else default_minimum_score
-                )
-            self.admin_volume.mkdir(parents=True, exist_ok=True)
-            (self.admin_volume / "baseline.json").write_text(
-                json.dumps(baselines, indent=2)
-            )
-            for key, value in rewards.items():
-                base = baselines.get(key)
-                tag = " (REGRESSION vs baseline)" if base is not None and value < base else ""
-                logger.info(
-                    "finalize: %s=%s baseline=%s%s", key, value, base, tag
+            return {"skipped": "base_commit is not set"}
+
+        last_error: Exception | None = None
+        for attempt in range(1, self._baseline_score_attempts + 1):
+            try:
+                baselines: dict[str, float] = {}
+                for target in self.targets:
+                    exp = await self.engine.evaluate_admin(
+                        task=target.task,
+                        dataset_id=target.dataset_id,
+                        split=target.split,
+                        commit=self.base_commit,
+                        sample_ids=target.sample_ids,
+                    )
+                    score = exp.result.score()
+                    baselines[target.reward_key] = (
+                        float(score) if score is not None else default_minimum_score
+                    )
+                # Best-effort local copy (admin volume does not survive teardown;
+                # the return value is the durable record).
+                try:
+                    self.admin_volume.mkdir(parents=True, exist_ok=True)
+                    (self.admin_volume / "baseline.json").write_text(
+                        json.dumps(baselines, indent=2)
+                    )
+                except OSError:
+                    logger.warning("could not write baseline.json to the admin volume")
+                for key, value in rewards.items():
+                    base = baselines.get(key)
+                    tag = (
+                        " (REGRESSION vs baseline)"
+                        if base is not None and value < base
+                        else ""
+                    )
+                    logger.info("finalize: %s=%s baseline=%s%s", key, value, base, tag)
+                return {"scores": baselines, "attempts": attempt}
+            except Exception as exc:  # noqa: BLE001 - never fail the trial on baseline scoring
+                last_error = exc
+                logger.warning(
+                    "baseline scoring attempt %d/%d failed: %s",
+                    attempt,
+                    self._baseline_score_attempts,
+                    exc,
                 )
-        except Exception:
-            logger.exception("baseline scoring failed; reward.json is unaffected")
+        logger.exception(
+            "baseline scoring failed after %d attempt(s); reward.json is unaffected",
+            self._baseline_score_attempts,
+            exc_info=last_error,
+        )
+        return {
+            "error": str(last_error),
+            "error_type": type(last_error).__name__ if last_error else None,
+            "attempts": self._baseline_score_attempts,
+        }
 
     async def _select_commit(self) -> str:
         if self.reward_mode == "submit":
@@ -236,4 +291,36 @@ async def _best_from_db(self) -> str:
             )
         # Highest admin score wins; ties break to the earliest shortlist position.
         rescored.sort(key=lambda t: (-t[0], t[1]))
-        return rescored[0][2]
+        best_score, _, best_commit = rescored[0]
+
+        # Selection floor: never ship a candidate that fails to beat the untouched
+        # baseline on the selection split. auto_best excludes base_commit from the
+        # candidate pool, so without this it selects the least-bad candidate even
+        # when every candidate regressed. Revert to the seed instead. Strict '>' so
+        # a statistical tie also reverts: if the optimizer cannot show an
+        # improvement, shipping the seed is the safe outcome. Needs a base_commit to
+        # compare against; costs one extra admin eval on the selection split.
+        if self.auto_best_baseline_floor and self.base_commit is not None:
+            base_dataset_id = self.selection_dataset_id
+            if base_dataset_id is None:
+                base_dataset_id = shortlist.iloc[0].get("dataset_subset_dataset_id")
+            base_exp = await self.engine.evaluate_admin(
+                task=self.selection_task,
+                dataset_id=base_dataset_id,
+                split=self.selection_split,
+                commit=self.base_commit,
+            )
+            base_s = base_exp.result.score()
+            base_score = float(base_s) if base_s is not None else default_minimum_score
+            if best_score <= base_score:
+                logger.info(
+                    "auto_best floor: best candidate %s (admin_score=%s) does not beat "
+                    "baseline %s (admin_score=%s); reverting to base_commit.",
+                    best_commit, best_score, self.base_commit, base_score,
+                )
+                return self.base_commit
+            logger.info(
+                "auto_best floor: best candidate %s (%s) beats baseline (%s); keeping it.",
+                best_commit, best_score, base_score,
+            )
+        return best_commit

vero/tests/test_harbor_app.py

@@ -75,15 +75,18 @@ def test_budget_exceeded_maps_to_429(self):
 class TestAdminEndpoint:
     def test_finalize_requires_token(self):
         verifier = MagicMock()
-        verifier.finalize = AsyncMock(return_value={"reward": 1.0})
+        # Mock mirrors the real finalize contract: {"rewards": ..., "baseline": ...}
+        # (the CLI extracts "rewards" for reward.json and echoes the rest to stdout).
+        payload = {"rewards": {"reward": 1.0}, "baseline": {"scores": {"reward": 0.8}}}
+        verifier.finalize = AsyncMock(return_value=payload)
         client = _client(verifier=verifier)
 
         assert client.post("/finalize").status_code == 403  # no token
         assert client.post("/finalize", headers={"Authorization": "Bearer wrong"}).status_code == 403
         verifier.finalize.assert_not_awaited()
 
         r = client.post("/finalize", headers={"Authorization": f"Bearer {TOKEN}"})
-        assert r.status_code == 200 and r.json() == {"reward": 1.0}
+        assert r.status_code == 200 and r.json() == payload
         verifier.finalize.assert_awaited_once()