fix: [no-ticket] harbor - access-tier integrity (free-baseline privilege, k-anonymity floor, trusted nested CLI)

vero/src/vero/evaluation/engine.py

@@ -146,8 +146,18 @@ def resolve_samples(self, req: EvalRequest) -> tuple[list[int] | None, int]:
     # Evaluation
     # ------------------------------------------------------------------
 
-    async def evaluate(self, req: EvalRequest, *, admin: bool = False) -> Experiment:
-        """Meter (unless admin) and run one evaluation; return the full Experiment.
+    async def evaluate(
+        self, req: EvalRequest, *, admin: bool = False, free: bool = False
+    ) -> Experiment:
+        """Meter (unless admin or free) and run one evaluation; return the full
+        Experiment.
+
+        ``admin`` and ``free`` are distinct authorities and must stay separate:
+        ``admin`` grants ACCESS (bypasses the tier gate and the ledger), while
+        ``free`` only waives the BUDGET debit for an otherwise ordinary agent
+        eval (the sidecar's one free baseline eval). A free eval still runs as
+        the agent: conflating the two let the free-baseline path evaluate
+        ``no_access`` splits and return their aggregate score to the agent.
 
         ``no_access`` gating is EXPLICIT and fail-closed: when ``split_accesses``
         is configured, the split's tier is resolved (an unlisted split defaults
@@ -163,7 +173,7 @@ async def evaluate(self, req: EvalRequest, *, admin: bool = False) -> Experiment
                     f"and cannot be evaluated by the agent."
                 )
         sample_ids, n = self.resolve_samples(req)
-        if not admin:
+        if not admin and not free:
             await self.budget.reserve(req.dataset_id, req.split, n)
         return await self.evaluator.evaluate(
             commit=req.commit,
@@ -183,22 +193,39 @@ async def evaluate_admin(
         split: str,
         commit: str,
         sample_ids: list[int] | None = None,
+        model: str | None = None,
     ) -> Experiment:
         """Admin/verifier evaluation: explicit ``task``, no budget, no allowlist.
 
         Unlike :meth:`evaluate` (which is bound to ``default_task`` and metered),
         this scores an arbitrary ``(task, dataset_id, split)`` — including held-out
         tasks/splits the agent never had access to. Used by the verifier to score
         the selected commit on its configured targets.
+
+        ``model`` overrides the executor model for this one eval (rides
+        ``task_params`` so the eval strategy can honor it; the Mode-B
+        HarborRunner does, the Mode-A vero-task path ignores it). Used for
+        transfer targets: scoring the champion under a model it was NOT
+        optimized on.
         """
+        params = self.run_constraints
+        if model is not None:
+            params = params.model_copy(
+                update={
+                    "task_params": {
+                        **(params.task_params or {}),
+                        "harbor_model_override": model,
+                    }
+                }
+            )
         return await self.evaluator.evaluate(
             commit=commit,
             dataset_id=dataset_id,
             split=split,
             task=task,
             sample_ids=sample_ids,
             db=self.db,
-            evaluation_parameters=self.run_constraints,
+            evaluation_parameters=params,
         )
 
     def status(self) -> dict[tuple[str, str], SplitBudget]:

vero/src/vero/harbor/app.py

@@ -17,7 +17,7 @@
 from vero.evaluation.engine import EvalRequest
 from vero.exceptions import ExperimentBudgetExceeded, InvalidSplitError
 from vero.harbor.auth import check_admin
-from vero.harbor.server import SubmitDisabledError
+from vero.harbor.server import KAnonymityError, SubmitDisabledError
 from vero.harbor.verifier import NoCandidateError
 
 if TYPE_CHECKING:
@@ -58,6 +58,10 @@ def create_app(
         SubmitDisabledError,
         lambda r, e: JSONResponse(status_code=409, content={"error": str(e)}),
     )
+    app.add_exception_handler(
+        KAnonymityError,
+        lambda r, e: JSONResponse(status_code=400, content={"error": str(e)}),
+    )
     app.add_exception_handler(
         NoCandidateError,
         lambda r, e: JSONResponse(status_code=409, content={"error": str(e)}),

vero/src/vero/harbor/build/compiler.py

@@ -236,12 +236,15 @@ def _serve_config(
                 "split": t.split,
                 "reward_key": t.reward_key,
                 "sample_ids": t.sample_ids,
+                "model": t.model,
             }
             for t in config.targets
         ],
         "base_commit": base_commit,
         "submit_enabled": config.submit_enabled,
         "score_baseline": config.score_baseline,
+        "k_anonymity_floor": config.k_anonymity_floor,
+        "instruct_exhaust_budget": config.instruct_exhaust_budget,
         "agent_volume": AGENT_VOLUME,
         "admin_volume": ADMIN_VOLUME,
         "admin_token_path": TOKEN_PATH,
@@ -404,6 +407,7 @@ def compile_task(
         and {"sample_ids", "num_samples"}
         <= {f.name for f in dataclasses.fields(EvalRequest)}
         and any(s.access == "viewable" for s in config.splits),
+        exhaust_budget=config.instruct_exhaust_budget,
     )
     _render(jenv, "task.toml.j2", out / "task.toml", **ctx)
     _render(jenv, "instruction.md.j2", out / "instruction.md", **ctx)

vero/src/vero/harbor/build/config.py

@@ -36,6 +36,12 @@ class TargetSpec(BaseModel):
     split: str
     reward_key: str = "reward"
     sample_ids: list[int] | None = None
+    # Executor-model override for this target (transfer probe; Mode B only):
+    # score the selected commit AND the baseline under a model it was not
+    # optimized on, so model-specific couplings (measured live: hardcoded
+    # temperature=0 crashing 72/72 on an executor that rejects it) surface at
+    # finalize instead of one substrate away.
+    model: str | None = None
 
 
 class _BuildConfigBase(BaseModel):
@@ -67,6 +73,18 @@ class _BuildConfigBase(BaseModel):
     # WORSE than the untouched repo is visible as a regression.
     score_baseline: bool = False
 
+    # Minimum sample count for agent-chosen subset evals of non_viewable splits
+    # (full-split evals always pass; <=1 disables). Aggregate responses carry
+    # mean_score, so singleton subsets would hand back per-sample labels.
+    k_anonymity_floor: int = 5
+
+    # Instruction lever: render the "unspent budget is wasted" persistence
+    # bullet that tells the optimizer to keep spending (re-measure the champion,
+    # try one more variant) instead of stopping early. On by default (current
+    # behavior); off makes stopping-early a choice the agent arrives at itself,
+    # which is the ablation arm for measuring what the exhortation contributes.
+    instruct_exhaust_budget: bool = True
+
     # write-access: paths in the target repo the optimizer may NOT edit
     # (the scorer, by default). Applied as unix perms in main before the agent runs.
     read_only_paths: list[str] = Field(default_factory=list)

vero/src/vero/harbor/build/templates/instruction.md.j2

@@ -51,9 +51,11 @@ in rough proportion to its size, so it is the cheap way to triage ideas:
   from a regression. Repeat baseline evals are metered. `vero harbor status`
   shows the baseline sha and whether the free eval is still available.
 {% endif %}
-- Scores are noisy. Unspent budget is wasted: if you finish with evals left, spend
+- Scores are noisy.
+{% if exhaust_budget %}- Unspent budget is wasted: if you finish with evals left, spend
   them re-measuring your best candidate to confirm its score, or trying one more
   variant, rather than stopping early.
+{% endif %}
 - The test split is hidden: you cannot evaluate it, and its labels never reach this
   container. Trying to read it will fail.
 - The scorer is locked. Only the eval sidecar scores.

vero/src/vero/harbor/config.py

@@ -30,6 +30,42 @@ class HarborConfig:
     #           estimates pass probability instead of pass@k (which "best"
     #           inflates toward).
     aggregate_attempts: str = "best"
+    # Trusted source for the nested `harbor` CLI, as a uv requirement spec
+    # (e.g. "harbor==0.1.17" or a pinned git URL). When set, the runner layers
+    # it over the candidate env with `uv run --with`, whose ephemeral overlay
+    # takes precedence for both the console script and sys.path — so the
+    # orchestrator that scores the candidate resolves from THIS spec, not from
+    # whatever the candidate's own pyproject/uv.lock pin (which the agent
+    # controls, and could point at a fork that fabricates trial results
+    # without running anything). None keeps the current behavior: the
+    # candidate env supplies harbor, and is trusted to.
+    harbor_requirement: str | None = None
+    # Bounded within-eval retry for infra-destroyed samples. A sample whose
+    # EVERY attempt died of a transient infrastructure cause (connection,
+    # timeout, rate limit, 5xx) was never measured at all: re-run it after a
+    # backoff instead of booking the outage as a permanent error. Measured
+    # live: a 65-second host DNS blip killed 44 of 72 attempts of one eval
+    # with ConnectionError, and nothing in the record distinguished the blip
+    # from a bad candidate.
+    #
+    # OFF BY DEFAULT, and it must stay off when the candidate is an
+    # adversarial optimizer. The qualifying predicate is built from exception
+    # types raised inside candidate code, and agents are stochastic: a
+    # candidate that raises an allowlisted exception whenever an attempt is
+    # going badly loses nothing on partially-good samples (its fakes
+    # zero-fill like honest failures) but converts every all-bad sample from
+    # a booked 0.0 into a fresh re-roll. That is one-sided selection over
+    # attempt sets, exactly what the zero-fill invariant exists to prevent.
+    # Enable only for trusted-candidate evaluations (frozen agents,
+    # operator-run matrices), where re-measuring an outage is pure signal
+    # recovery. Candidate crashes and exhausted key budgets never retry
+    # regardless (a crash is a result; a spent key cannot recover by
+    # waiting), and recovered samples carry an ``infra_retry`` audit marker.
+    infra_retry_rounds: int = 0
+    # Backoff before retry round N is N times this many seconds. Transient
+    # infra needs time, not immediacy: instant retries burned 6 of 8 run
+    # attempts inside one live DNS blip.
+    infra_retry_delay_s: float = 30.0
     extra_args: list[str] = field(default_factory=list)  # passthrough harbor run flags
 
     def __post_init__(self) -> None:
@@ -40,6 +76,18 @@ def __post_init__(self) -> None:
                 f"aggregate_attempts must be 'best' or 'mean', got "
                 f"{self.aggregate_attempts!r}"
             )
+        if self.infra_retry_rounds < 0:
+            raise ValueError(
+                f"infra_retry_rounds must be >= 0, got {self.infra_retry_rounds}"
+            )
+        # A zero (or negative) delay silently nullifies the backoff, and an
+        # instant retry re-enters the same outage: 6 of 8 run attempts once
+        # burned inside a single live DNS blip for exactly this reason.
+        if self.infra_retry_rounds > 0 and self.infra_retry_delay_s <= 0:
+            raise ValueError(
+                f"infra_retry_delay_s must be > 0 when infra retries are "
+                f"enabled, got {self.infra_retry_delay_s}"
+            )
 
     @property
     def is_registry(self) -> bool:

vero/src/vero/harbor/protocol.py

@@ -104,6 +104,10 @@ def build_status(
     split_accesses: list[SplitAccess],
     base_commit: str | None = None,
     free_baseline_available: bool = False,
+    # Default matches EvaluationSidecar's enforcement default: a caller that
+    # forgets to pass the floor must not advertise a laxer one than the
+    # sidecar enforces (agents would send sub-floor requests that 400).
+    k_anonymity_floor: int = 5,
 ) -> StatusSummary:
     """Build the agent-facing status from the budget ledger + split tiers.
 
@@ -119,6 +123,15 @@ def build_status(
                 "dataset_id": dataset_id,
                 "tier": str(tier),
                 "agent_evaluable": tier != SplitAccessLevel.no_access,
+                # Subset evals below this sample count are rejected on
+                # non_viewable splits (full-split evals always pass). Advertised
+                # so an agent plans subset probes instead of burning budget on
+                # requests the sidecar will refuse.
+                "min_subset_samples": (
+                    k_anonymity_floor
+                    if tier == SplitAccessLevel.non_viewable
+                    else 1
+                ),
                 "remaining_sample_budget": b.remaining_sample_budget,
                 "remaining_run_budget": b.remaining_run_budget,
             }