diff --git a/vero/.gitignore b/vero/.gitignore index d8d3a3c..14d7b2c 100644 --- a/vero/.gitignore +++ b/vero/.gitignore @@ -11,6 +11,8 @@ __pycache__/ *.egg-info/ dist/ build/ +# ...but the harbor compiler package is source, not a packaging artifact: +!src/vero/harbor/build/ # Testing .pytest_cache/ diff --git a/vero/README.md b/vero/README.md index cddac72..e8d8d28 100644 --- a/vero/README.md +++ b/vero/README.md @@ -525,6 +525,22 @@ agent = VeroAgent( ) ``` +## Harbor integration + +vero can compile an optimization run into a [Harbor](https://www.harborframework.com) task, so the *optimizer* itself becomes a Harbor agent-under-test: any Harbor agent (Claude Code, an oracle script, …) edits a target repo and spends an evaluation budget, and the reward is the best candidate's score on a hidden split. This makes optimization runs reproducible and leaderboard-gradeable — the optimizer can't read hidden labels, modify the scorer, or bypass its budget. + +```bash +uv pip install 'scale-vero[harbor]' +vero harbor build -c build.yaml -o /tmp/opt-task # compile a Harbor task +vero harbor run -c build.yaml -a claude-code -m claude-haiku-4-5 -e docker # build + run +``` + +Two evaluation modes: **Mode A** (vero runs inference + scoring against vero-side labels) and **Mode B** (evaluation is delegated to a *nested* `harbor run`, e.g. on Modal). See: + +- [`docs/harbor/architecture.md`](docs/harbor/architecture.md) — what it is, the topology, and the leaderboard-integrity model. +- [`docs/harbor/tutorial.md`](docs/harbor/tutorial.md) — build and run a task end to end. +- [`examples/gaia-optimization`](examples/gaia-optimization) (Mode B), the complete runnable example (ships a `build.yaml`). [`examples/gsm8k-agent`](examples/gsm8k-agent) (Mode A) ships the agent + vero task but not yet a `build.yaml`; pair it with the Mode A `build.yaml` snippet in [`docs/harbor/tutorial.md`](docs/harbor/tutorial.md). + ## Examples See [`examples/matmul-kernel/`](examples/matmul-kernel/) for a complete runnable example that optimizes a matrix multiply kernel for speed. It demonstrates eval-only mode, full optimization with VeroAgent or Claude Code, filesystem artifacts, and resource-based editing. diff --git a/vero/docs/harbor/architecture.md b/vero/docs/harbor/architecture.md new file mode 100644 index 0000000..1a86359 --- /dev/null +++ b/vero/docs/harbor/architecture.md @@ -0,0 +1,140 @@ +# Harbor integration — architecture + +The Harbor integration turns a **vero optimization run into a [Harbor](https://www.harborframework.com) +task**. The agent-under-test of that Harbor task is an *optimizer*: any Harbor agent +(Claude Code, an oracle script, …) edits a target repository and spends an evaluation +budget; the reward is the best candidate's score on a hidden test split. + +This lets anyone optimize a coding agent with plain `harbor run`, and aims to make the +result leaderboard-gradeable. On a best-effort, OS/process-level basis the integration +withholds per-sample labels from the agent volume, meters every agent evaluation against +a budget, applies read-only paths, and gates final hidden-split scoring behind a +`root:600` token the de-privileged optimizer cannot read (a container escape is out of +scope). See [Leaderboard integrity](#leaderboard-integrity-the-trust-boundary) for the +exact mechanisms and their limits. + +``` +harbor run -p -a -m -e + │ + ▼ one optimization trial (a Docker Compose environment): + ┌────────────────────────┐ ┌────────────────────────────────────┐ + │ main (optimizer bench) │ HTTP │ eval-sidecar (the evaluation engine) │ + │ • target repo (rw*) │ ─────► │ • dataset + scorer + baseline repo │ + │ • `vero harbor` client │ │ • budget ledger + creds │ + │ • runs the -a optimizer│ │ • `vero harbor serve` (FastAPI) │ + └────────────────────────┘ └────────────────────────────────────┘ + │ (trial end, shared verifier) ▲ + └── `vero harbor finalize` (admin token) ──┘ → /logs/verifier/reward.json +``` + +## The optimization loop + +1. **`vero harbor build`** compiles a `build.yaml` into a Harbor task directory + (`environment/` compose + Dockerfiles, `instruction.md`, `tests/test.sh`), baking + the dataset, scorer, baseline repo, and a `ServeConfig`. +2. At trial start, **`main`** seeds the target repo onto a shared volume and applies + write-access rules; the **eval-sidecar** starts `vero harbor serve` and writes a + per-trial admin token. +3. The **optimizer** (the `-a` agent) edits the repo, commits, and calls + `vero harbor eval --split ` to measure a commit. The sidecar + fetches that commit, evaluates it (metered against the budget), and returns an + **aggregate** score (never per-sample labels). +4. At trial end, Harbor runs `tests/test.sh` in `main` (shared verifier mode). It + reads the admin token and calls the sidecar's **`finalize`**: the sidecar selects + the winning commit and scores it on the **hidden** test split, producing the reward. + +## Two evaluation modes + +The seam is a single injection point on the `Evaluator` (`eval_strategy`): + +- **Mode A — vero scores** (`task_project`/`task` + dataset). vero runs the agent's + inference and a vero scoring function against vero-side labels. Example agent: + [`examples/gsm8k-agent`](../../examples/gsm8k-agent) (note: it ships the agent and + vero task but not yet a `build.yaml`; see the Mode A `build.yaml` snippet in the + [tutorial](./tutorial.md) for a runnable config). +- **Mode B — Harbor scores** (`HarborConfig`). Inference is delegated: for each + candidate, `HarborRunner` runs a *nested* `harbor run` of the agent on a set of + Harbor tasks (e.g. on Modal) and collates the verifier rewards. One Harbor task = + one sample. Example: [`examples/gaia-optimization`](../../examples/gaia-optimization). + +Both modes share the same topology, trust boundary, budget, and verifier — only the +"produce sample results" step differs. + +## Leaderboard integrity (the trust boundary) + +The optimizer is untrusted. Integrity rests on a few mechanisms, all best-effort at +the OS/process level (a container escape is out of scope): + +- **3-tier split visibility** (`SplitAccessLevel`): `visible` (aggregate + per-sample + results), `non_viewable` (aggregate score only, no labels), `no_access` (hidden; + never evaluable by the agent, never written to its volume). **A split not listed in + `splits:` currently defaults to viewable (fail-open).** List *every* split explicitly + and give held-out splits `no_access`; do not rely on omission to hide a split. (Once + the protocol fix lands this default becomes fail-closed: an unlisted split defaults to + `no_access`.) +- **Write-routing by tier**: the sidecar writes only the agent-permitted projection of + each result to the *agent-results* volume (read-only in `main`). Full results, the + dataset, the ledger, and creds live on the *admin* volume, **never** mounted to `main`. +- **Token-gated finalize**: `finalize` (selection + hidden-split scoring) requires an + admin token written `root:600` on a volume `main` mounts read-only. The optimizer + runs as a de-privileged user and cannot read it, so it cannot trigger scoring or + probe the test split; the verifier (root, shared mode) can. +- **Metered budget**: a durable `BudgetLedger` caps how much the agent can evaluate per + split. Admin (verifier) evaluations bypass the meter. +- **Commit transfer**: the sidecar `git fetch`es the agent's commit from the mounted + repo into its *own* repo with hooks disabled and `file://` (object copy, no + alternates), so the evaluated tree is fully owned by the sidecar and tamper-evident. +- **Protected scorer / write-access** (mode-dependent): in **Mode B** the scorer, + dataset, and creds are sidecar-only and never share a filesystem with the optimizer. + In **Mode A** the scoring function is vero task code that lives inside the agent's + *editable* `agent_repo` (e.g. `src/gsm8k_agent/vero_tasks`); it is protected only by + `read_only_paths` in `build.yaml`, applied as unix perms in `main` before the + optimizer runs (best-effort, not container isolation). Full Mode A scorer isolation + requires baking the task project into the sidecar instead of the agent repo, which + lands with the `serve.py` fix. + +### Why a sidecar + shared verifier + +The evaluation engine, dataset, and creds live in a separate container so the optimizer +never shares a filesystem or process space with them (in **Mode B** the scorer too; in +**Mode A** the scorer currently lives in the agent's editable repo and is guarded only by +`read_only_paths`, until the `serve.py` fix bakes a Mode A task project into the sidecar). We use Harbor's +**shared verifier** (the env, including the sidecar, stays up during `tests/test.sh`) +so the verifier can reach the live engine over HTTP and stay the single source of +truth — avoiding shipping the repo/dataset/ledger into a fresh verifier container. The +agent/admin split is enforced by the `root:600` token rather than separate services. + +## Component map + +``` +vero/harbor/ +├── build/ `vero harbor build`: BuildConfig → Harbor task dir +│ ├── config.py BuildConfig (the build.yaml schema) +│ ├── compiler.py renders the task dir; bakes dataset/scorer/repo/ServeConfig +│ └── templates/ compose, two Dockerfiles, instruction.md, test.sh, seed.sh, solve.sh +├── serve.py `vero harbor serve`: assemble engine+sidecar+verifier from ServeConfig +├── app.py FastAPI surface: /eval /submit /status (agent), /finalize (admin) +├── server.py EvaluationSidecar: commit transfer + tier write-routing (transport-agnostic) +├── verifier.py Verifier: commit selection (submit | auto_best) + hidden-split scoring +├── auth.py per-trial admin token (generate / root:600 write / verify) +├── cli.py `vero harbor` group: build | run | serve | eval | submit | status | finalize +├── config.py HarborConfig (Mode B) +├── runner.py HarborRunner (Mode-B EvalStrategy): nested `harbor run` → collate +├── dataset.py Mode-B {split: [task_names]} partition → DatasetDict +└── protocol.py aggregate-safe wire types + the redaction of an Experiment + (note: an unlisted split currently defaults to viewable / fail-open) + +vero/evaluation/ +├── engine.py EvaluationEngine: budget metering + the single evaluate() entry point +├── evaluator.py Evaluator: checkout + run; the eval_strategy seam (Mode A vs B) +└── strategy.py EvalStrategy protocol +``` + +The compiler↔sidecar contract is `ServeConfig` (baked as `environment/sidecar/serve.json`); +the optimizer↔sidecar contract is the HTTP API in `app.py` (+ the `vero harbor` CLI clients). + +## See also + +- [Tutorial](./tutorial.md) — build and run an optimization task end to end. +- [`examples/gsm8k-agent`](../../examples/gsm8k-agent) (Mode A agent; no `build.yaml` yet, use the tutorial's Mode A snippet). +- [`examples/gaia-optimization`](../../examples/gaia-optimization) — Mode B (nested Harbor on Modal). diff --git a/vero/docs/harbor/tutorial.md b/vero/docs/harbor/tutorial.md new file mode 100644 index 0000000..37a0319 --- /dev/null +++ b/vero/docs/harbor/tutorial.md @@ -0,0 +1,134 @@ +# Harbor integration — tutorial + +This walks through compiling a vero optimization run into a Harbor task and running it +with an optimizer agent. Read the [architecture](./architecture.md) first for the +concepts (modes, the trust boundary, the optimization loop). + +## Install + +```bash +uv pip install 'scale-vero[harbor]' # adds the `vero harbor` CLI +# the Harbor CLI itself is invoked via uvx; for Modal-backed inner runs use the extra: +uvx --from 'harbor[modal]' harbor --help +``` + +## 1. Write a `build.yaml` + +A build config describes the optimization task: the repo to optimize, how candidates +are scored, the split tiers, the budget, and the reward. + +### Mode A — vero runs inference + scoring + +```yaml +name: myorg/gsm8k-opt +agent_repo: /path/to/gsm8k-agent # the repo the optimizer edits +mode: A +task: gsm8k # vero task name +task_module: gsm8k_agent.vero_tasks # module that registers it +dataset: /path/to/gsm8k-dataset # a saved DatasetDict (inputs + labels) + +splits: + - { split: validation, access: non_viewable } # optimizer sees aggregate score only + - { split: test, access: no_access } # hidden; scored at the end +budgets: + - { split: validation, total_run_budget: 5 } +reward_mode: auto_best # best validation commit auto-selected +selection_split: validation +targets: + - { split: test, reward_key: reward } +read_only_paths: + - src/gsm8k_agent/vero_tasks # the scorer — optimizer may not edit it +secrets: [OPENAI_API_KEY, OPENAI_BASE_URL] # injected into the eval sidecar only +``` + +### Mode B — a nested `harbor run` scores (e.g. on Modal) + +```yaml +name: myorg/gaia-opt +agent_repo: /path/to/gaia-agent +mode: B +harbor: + agent_import_path: "gaia_agent:GaiaAgent" # the agent inside agent_repo + task_source: gaia/gaia # Harbor registry benchmark (or a local dir) + environment: modal + model: openai/gpt-4o-mini # the inner agent's model +partition: # {split: [harbor task names]} — one task = one sample + train: [gaia/, gaia/, ...] + validation: [gaia/, gaia/, ...] +splits: + - { split: train, access: non_viewable } + - { split: validation, access: no_access } +budgets: + - { split: train, total_run_budget: 3 } +reward_mode: auto_best +selection_split: train +targets: + - { split: validation, reward_key: accuracy } +secrets: [OPENAI_API_KEY, OPENAI_BASE_URL, MODAL_TOKEN_ID, MODAL_TOKEN_SECRET] +``` + +`secrets` are variable **names**: their values are read from your shell at run time and +injected into the eval sidecar only — never into the optimizer's container. The full +field list is in `vero/harbor/build/config.py` (`BuildConfig`). + +## 2. Build the task + +```bash +vero harbor build -c build.yaml -o /tmp/opt-task +``` + +This emits a Harbor task directory: `environment/` (a Docker Compose env = the optimizer +workbench `main` + the `eval-sidecar`, plus volumes), `instruction.md` (the protocol the +optimizer reads), and `tests/test.sh` (the verifier). The dataset/scorer/baseline repo +and the sidecar's `ServeConfig` are baked in. + +## 3. Run it with an optimizer + +Any Harbor agent can be the optimizer. Provide its creds in your shell (Harbor forwards +them into `main`); e.g. for `claude-code` set `ANTHROPIC_API_KEY` (+ `ANTHROPIC_BASE_URL` +if routing through a gateway). + +```bash +# build + run in one step: +vero harbor run -c build.yaml -a claude-code -m claude-haiku-4-5 -e docker + +# or run a pre-built task dir: +uvx harbor run -p /tmp/opt-task -a claude-code -m claude-haiku-4-5 -e docker + +# the `oracle` agent runs solution/solve.sh (a scripted optimizer) — handy for a smoke test: +uvx harbor run -p /tmp/opt-task -a oracle -e docker +``` + +The reward lands in the job's `verifier/reward.json` (e.g. `{"reward": 0.42}`), and Harbor +reports it as the trial reward. + +## What the optimizer does (the agent-side protocol) + +Inside `main`, the optimizer follows `instruction.md`. The `vero harbor` CLI talks to the +eval sidecar over `VERO_EVAL_URL` (set automatically): + +```bash +vero harbor status # remaining budget, evaluable splits +# edit the repo, commit, then measure the current HEAD: +vero harbor eval --dataset-id --split validation +vero harbor submit # (if reward_mode: submit) nominate the final commit +``` + +- `eval` returns an aggregate score + remaining budget; for `no_access` splits it is + rejected, and labels are never returned. +- With `reward_mode: auto_best`, the best commit on `selection_split` is chosen + automatically; with `submit`, the agent nominates one. +- The verifier scores the chosen commit on the hidden `targets` split at the end. + +## Inspecting a run + +```bash +uvx harbor view # browse trials +cat /*/*/verifier/reward.json +``` + +## Examples + +- [`examples/gsm8k-agent`](../../examples/gsm8k-agent) (Mode A agent, vero scores gsm8k). It ships the agent + vero task but not a `build.yaml` yet; use the Mode A `build.yaml` snippet above to drive it. +- [`examples/gaia-optimization`](../../examples/gaia-optimization) — Mode B (terminus on + GAIA via nested Harbor on Modal), with an editable-prompt optimization surface. diff --git a/vero/examples/gaia-optimization/README.md b/vero/examples/gaia-optimization/README.md new file mode 100644 index 0000000..2d2450f --- /dev/null +++ b/vero/examples/gaia-optimization/README.md @@ -0,0 +1,79 @@ +# GAIA optimization example (Harbor Mode B) + +This example shows the **vero ⇄ Harbor** integration optimizing a coding agent on a +real benchmark. An optimizer (e.g. Claude Code) edits a GAIA agent's prompt; each +candidate is scored by a **nested `harbor run`** of the agent on real +[GAIA](https://huggingface.co/datasets/gaia-benchmark/GAIA) tasks (on Modal). The +reward is accuracy on a hidden split. + +This is "Mode B": vero does **no** inference itself — evaluation is delegated to a +nested Harbor run, and the reward comes from Harbor's verifier. (Contrast "Mode A", +e.g. [`../gsm8k-agent`](../gsm8k-agent), where vero runs inference and scoring directly.) + +## What's here + +``` +gaia-optimization/ +├── build.yaml # the optimization task definition (vero harbor build -c) +├── pyproject.toml # deps: harbor[modal] +└── src/gaia_agent/ + ├── agent.py # GaiaAgent(Terminus2): the editable agent + └── prompts/ # the OPTIMIZATION SURFACE — the optimizer edits these + ├── terminus-json-plain.txt + └── terminus-xml-plain.txt +``` + +`GaiaAgent` subclasses Harbor's `Terminus2` and overrides only its prompt-template +path so the prompt is read from this package's editable `prompts/` directory. The +optimizer improves `prompts/terminus-json-plain.txt`; the terminal loop, tmux +session, and response parsing are reused from `Terminus2` unchanged. + +## Prerequisites + +- The `harbor` CLI (`uvx --from 'harbor[modal]' harbor ...`) and Docker (outer trial). +- A [Modal](https://modal.com) account for the inner GAIA runs: + `MODAL_TOKEN_ID` / `MODAL_TOKEN_SECRET` in your shell env. +- An OpenAI-compatible LLM endpoint for the **inner** GAIA agent: + `OPENAI_API_KEY` (+ optional `OPENAI_BASE_URL` to point at a gateway). The model is + set in `build.yaml` (`harbor.model`, default `openai/gpt-4o-mini`). +- Creds for the **outer** optimizer agent, per that agent (e.g. `ANTHROPIC_API_KEY` + for `-a claude-code`). Harbor forwards these from your shell into the optimizer's + container; they are **not** shared with the eval sidecar. + +Secrets are resolved from your shell at run time and injected into the eval sidecar +**only** (see `build.yaml`'s `secrets:` — those are variable *names*, not values). + +## Run it + +```bash +# install vero with the harbor extra +uv pip install 'scale-vero[harbor]' + +# build the task, then run it with an optimizer of your choice +vero harbor build -c build.yaml -o /tmp/gaia-task +uvx harbor run -p /tmp/gaia-task -a claude-code -m claude-haiku-4-5 -e docker + +# ...or build + run in one step: +vero harbor run -c build.yaml -a claude-code -m claude-haiku-4-5 -e docker +``` + +The optimizer reads the task instruction, edits `src/gaia_agent/prompts/...`, commits, +and calls `vero harbor eval --split train` to measure candidates within its budget. +At the end, the best train commit is scored on the hidden `validation` split and the +accuracy is written to Harbor's `reward.json`. + +## Notes + +- **GAIA is hard.** A terminal agent solves only some tasks; expect low scores and + weak optimization signal on a 5-task subset. Increase the subset, pick easier tasks, + or use a stronger model for a more meaningful run. +- **Cost/time.** Each GAIA task is a full agent rollout on a Modal sandbox (minutes + + LLM tokens). The default budget keeps a run to a handful of nested evals. +- Pick your own task ids by enumerating the benchmark: + `python -c "import asyncio; from harbor.models.job.config import DatasetConfig as D; print(asyncio.run(D(name='gaia/gaia').get_task_configs()))"` + +## Attribution + +`src/gaia_agent/prompts/*.txt` are copied from Harbor's `terminus_2` agent +(© Harbor authors, Apache-2.0) so the prompt stays compatible with the parser +`GaiaAgent` inherits. They are included here as the editable optimization surface. diff --git a/vero/examples/gaia-optimization/build.yaml b/vero/examples/gaia-optimization/build.yaml new file mode 100644 index 0000000..1eaec11 --- /dev/null +++ b/vero/examples/gaia-optimization/build.yaml @@ -0,0 +1,72 @@ +# `vero harbor build -c build.yaml -o ` compiles this into a Harbor task. +# Then: `harbor run -p -a -m -e docker` +# (or use `vero harbor run -c build.yaml -a ...` to build + run in one step). +# +# Mode B: the optimizer edits the GaiaAgent prompt in this repo; the eval sidecar +# scores candidates by a *nested* `harbor run` of the agent on real GAIA tasks +# (here on Modal). Reward = accuracy on the hidden `validation` split. + +name: examples/gaia-optimization +description: Optimize a terminus-2 GAIA agent's prompt; reward = accuracy on hidden GAIA tasks. + +agent_repo: . # this directory (the GaiaAgent the optimizer edits) +mode: B + +harbor: + agent_import_path: "gaia_agent:GaiaAgent" + task_source: gaia/gaia # Harbor registry benchmark (enumerated below) + environment: modal # inner provider; needs MODAL_TOKEN_ID/SECRET (see README) + # The inner GAIA agent's model. `openai/` routes via OPENAI_BASE_URL, so this + # works against OpenAI directly or any OpenAI-compatible gateway you point it at. + model: openai/gpt-4o-mini + max_retries: 1 + +# A small subset of gaia/gaia: 5 tasks the optimizer may measure on (train) and +# 5 held-out tasks scored once at the end (validation). Swap in your own ids; +# list them with: DatasetConfig(name="gaia/gaia").get_task_configs() +partition: + train: + - gaia/00d579ea-0889-4fd9-a771-2c8d79835c8d + - gaia/023e9d44-96ae-4eed-b912-244ee8c3b994 + - gaia/0383a3ee-47a7-41a4-b493-519bdefe0488 + - gaia/04a04a9b-226c-43fd-b319-d5e89743676f + - gaia/0512426f-4d28-49f0-be77-06d05daec096 + validation: + - gaia/05407167-39ec-4d3a-a234-73a9120c325d + - gaia/076c8171-9b3b-49b9-a477-244d2a532826 + - gaia/08c0b6e9-1b43-4c2e-ae55-4e3fce2c2715 + - gaia/08cae58d-4084-4616-b6dd-dd6534e4825b + - gaia/08f3a05f-5947-4089-a4c4-d4bcfaa6b7a0 + +splits: + - { split: train, access: non_viewable } # optimizer sees aggregate scores only + - { split: validation, access: no_access } # scores are never returned to the optimizer +# CAVEAT: `access: no_access` withholds the validation *scores*, not the validation +# *identity*. Because `agent_repo: .` and this build.yaml is git-tracked, the validation +# task ids above are seeded into the optimizer's repo (via `git archive HEAD`) and ARE +# readable by it. That is acceptable here because gaia/gaia is a public benchmark, so the +# held-out *ids* are not secret; only the per-sample results are. If you adapt this for a +# benchmark whose held-out *identity* must stay secret, do NOT keep the held-out partition +# in the git-tracked agent_repo: move build.yaml (or just the held-out `partition`/`splits` +# entries) outside the agent_repo subtree, add it to .gitignore, or inject the held-out +# split sidecar-side at build time so it is never archived into /work/agent. + +budgets: + - { split: train, total_run_budget: 3 } # up to 3 measured evals of the train split + +reward_mode: auto_best # best train commit is auto-selected +selection_split: train +targets: + - { split: validation, reward_key: accuracy } + +# Secrets are resolved from your shell env and injected into the eval sidecar only, +# never into the optimizer's container. These are variable NAMES, not values. +secrets: + - OPENAI_API_KEY + - OPENAI_BASE_URL + - MODAL_TOKEN_ID + - MODAL_TOKEN_SECRET + +timeout: 3600 +sample_timeout: 900 +max_concurrency: 5 diff --git a/vero/examples/gaia-optimization/pyproject.toml b/vero/examples/gaia-optimization/pyproject.toml new file mode 100644 index 0000000..ad04d9a --- /dev/null +++ b/vero/examples/gaia-optimization/pyproject.toml @@ -0,0 +1,18 @@ +[project] +name = "gaia-agent" +version = "0.1.0" +description = "A GAIA agent (terminus-2 + editable prompt) optimized via the vero Harbor integration." +requires-python = ">=3.12" # harbor[modal] requires Python 3.12+ +dependencies = [ + "harbor[modal]>=0.13", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/gaia_agent"] + +[tool.hatch.build.targets.wheel.force-include] +"src/gaia_agent/prompts" = "gaia_agent/prompts" diff --git a/vero/examples/gaia-optimization/src/gaia_agent/__init__.py b/vero/examples/gaia-optimization/src/gaia_agent/__init__.py new file mode 100644 index 0000000..1da8a40 --- /dev/null +++ b/vero/examples/gaia-optimization/src/gaia_agent/__init__.py @@ -0,0 +1,3 @@ +from gaia_agent.agent import GaiaAgent + +__all__ = ["GaiaAgent"] diff --git a/vero/examples/gaia-optimization/src/gaia_agent/agent.py b/vero/examples/gaia-optimization/src/gaia_agent/agent.py new file mode 100644 index 0000000..205191c --- /dev/null +++ b/vero/examples/gaia-optimization/src/gaia_agent/agent.py @@ -0,0 +1,38 @@ +"""A GAIA optimization target: Harbor's terminus-2 with an editable prompt. + +``GaiaAgent`` subclasses Harbor's ``Terminus2`` and points its prompt template at +this package's ``prompts/`` directory instead of the copy baked into the harbor +package. That makes the prompt the *optimization surface*: an optimizer (e.g. +Claude Code, driving ``vero harbor eval``) edits ``prompts/terminus-json-plain.txt`` +to improve the agent's GAIA score, while the terminal loop, tmux session, and +response parsing are reused unchanged from ``Terminus2``. + +The agent runs in the Harbor orchestrator process (where the LLM creds live) and +drives the task sandbox via ``environment.exec``; see the example README. +""" + +from __future__ import annotations + +from pathlib import Path + +from harbor.agents.terminus_2.terminus_2 import Terminus2 + +_PROMPTS = Path(__file__).parent / "prompts" + + +class GaiaAgent(Terminus2): + """Terminus-2 with its prompt sourced from this package's editable ``prompts/``.""" + + @staticmethod + def name() -> str: + return "gaia-agent" + + def version(self) -> str: + return "0.1.0" + + def _get_prompt_template_path(self) -> Path: + if self._parser_name == "json": + return _PROMPTS / "terminus-json-plain.txt" + if self._parser_name == "xml": + return _PROMPTS / "terminus-xml-plain.txt" + return super()._get_prompt_template_path() diff --git a/vero/examples/gaia-optimization/src/gaia_agent/prompts/terminus-json-plain.txt b/vero/examples/gaia-optimization/src/gaia_agent/prompts/terminus-json-plain.txt new file mode 100644 index 0000000..6481d56 --- /dev/null +++ b/vero/examples/gaia-optimization/src/gaia_agent/prompts/terminus-json-plain.txt @@ -0,0 +1,54 @@ +You are an AI assistant tasked with solving command-line tasks in a Linux environment. You will be given a task description and the output from previously executed commands. Your goal is to solve the task by providing batches of shell commands. + +Format your response as JSON with the following structure: + +{{ + "analysis": "Analyze the current state based on the terminal output provided. What do you see? What has been accomplished? What still needs to be done?", + "plan": "Describe your plan for the next steps. What commands will you run and why? Be specific about what you expect each command to accomplish.", + "commands": [ + {{ + "keystrokes": "ls -la\n", + "duration": 0.1 + }}, + {{ + "keystrokes": "cd project\n", + "duration": 0.1 + }} + ], + "task_complete": true +}} + +Required fields: +- "analysis": Your analysis of the current situation +- "plan": Your plan for the next steps +- "commands": Array of command objects to execute + +Optional fields: +- "task_complete": Boolean indicating if the task is complete (defaults to false if not present) + +Command object structure: +- "keystrokes": String containing the exact keystrokes to send to the terminal (required) +- "duration": Number of seconds to wait for the command to complete before the next command will be executed (defaults to 1.0 if not present) + +IMPORTANT: The text inside "keystrokes" will be used completely verbatim as keystrokes. Write commands exactly as you want them sent to the terminal: +- You must end every command with a newline (\n) or it will not execute. +- For special key sequences, use tmux-style escape sequences: + - C-c for Ctrl+C + - C-d for Ctrl+D + +The "duration" attribute specifies the number of seconds to wait for the command to complete (default: 1.0) before the next command will be executed. On immediate tasks (e.g., cd, ls, echo, cat) set a duration of 0.1 seconds. On commands (e.g., gcc, find, rustc) set a duration of 1.0 seconds. On slow commands (e.g., make, python3 [long running script], wget [file]) set an appropriate duration as you determine necessary. + +It is better to set a smaller duration than a longer duration. It is always possible to wait again if the prior output has not finished, by running {{"keystrokes": "", "duration": 10.0}} on subsequent requests to wait longer. Never wait longer than 60 seconds; prefer to poll to see intermediate result status. + +Important notes: +- Each command's keystrokes are sent exactly as written to the terminal +- Do not include extra whitespace before or after the keystrokes unless it's part of the intended command +- Extra text before or after the JSON will generate warnings but be tolerated +- The JSON must be valid - use proper escaping for quotes and special characters within strings +- Commands array can be empty if you want to wait without taking action + +Task Description: +{instruction} + +Current terminal state: +{terminal_state} diff --git a/vero/examples/gaia-optimization/src/gaia_agent/prompts/terminus-xml-plain.txt b/vero/examples/gaia-optimization/src/gaia_agent/prompts/terminus-xml-plain.txt new file mode 100644 index 0000000..6386356 --- /dev/null +++ b/vero/examples/gaia-optimization/src/gaia_agent/prompts/terminus-xml-plain.txt @@ -0,0 +1,60 @@ +You are an AI assistant tasked with solving command-line tasks in a Linux environment. You will be given a task description and the output from previously executed commands. Your goal is to solve the task by providing batches of shell commands. + +Format your response as XML with the following structure: + + + +Analyze the current state based on the terminal output provided. What do you see? What has been accomplished? What still needs to be done? + + +Describe your plan for the next steps. What commands will you run and why? Be specific about what you expect each command to accomplish. + + +ls -la + +cd project + + +true + + +Required sections: +- : Your analysis of the current situation +- : Your plan for the next steps +- : XML structure containing commands to execute + +The `duration` attribute of specifies the number of seconds to wait for the command to complete (default: 1.0) before the next command will be executed. On immediate tasks (e.g., cd, ls, echo, cat) set a duration of 0.1 seconds. On commands (e.g., gcc, find, rustc) set a duration of 1.0 seconds. On slow commands (e.g., make, python3 [long running script], wget [file]) set an apprpriate duration as you determine necessary. + +It is better to set a smaller duration than a longer duration. In is always possible to wait again if the prior output has not finished, by running on subsequent requests to wait longer. Never wait longer than 60 seconds; prefer to poll to see intermediate result status. + +Optional sections: +- : Include this tag if the task is complete. Can be: + - true (task complete) + - false (task not complete) + - (self-closing, equivalent to false) + - (empty, equivalent to false) + - If not present, task is assumed not complete + +IMPORTANT: The text inside each tag will be used completely verbatim as keystrokes. DO NOT XML-encode special characters - write them directly: +- Use < and > directly, NOT < and > +- Use & directly, NOT & +- Use quotes directly, NOT " +Even though this is XML, the content inside keystrokes tags is treated as raw text and sent exactly as written. Ensure there is no extra leading or trailing whitespace unless intended. You must end every command with a newline (\n) or it will not execute. + + +Special key sequences (use tmux-style escape sequences): +- C-c for Ctrl+C. MUST be sent as a keystroke by itself, e.g., C-c +- C-d for Ctrl+D. MUST be sent as a keystroke by itself, e.g., C-d +- For Enter/newline: simply add a newline (line break) in the XML, everything inside the command tag will be sent byte-for-byte + +Important notes: +- Each command's text content is sent exactly as keystrokes to the terminal +- Do not include extra whitespace before or after the command text unless it's part of the intended command +- Avoid extra text before or after the tags +- Avoid additional XML tags outside of analysis/plan/commands/task_complete + +Task Description: +{instruction} + +Current terminal state: +{terminal_state} diff --git a/vero/src/vero/harbor/build/__init__.py b/vero/src/vero/harbor/build/__init__.py new file mode 100644 index 0000000..17711fd --- /dev/null +++ b/vero/src/vero/harbor/build/__init__.py @@ -0,0 +1,6 @@ +"""The `vero harbor build` compiler: BuildConfig -> a runnable Harbor task dir.""" + +from vero.harbor.build.compiler import compile_task +from vero.harbor.build.config import BuildConfig + +__all__ = ["BuildConfig", "compile_task"] diff --git a/vero/src/vero/harbor/build/compiler.py b/vero/src/vero/harbor/build/compiler.py new file mode 100644 index 0000000..b667113 --- /dev/null +++ b/vero/src/vero/harbor/build/compiler.py @@ -0,0 +1,296 @@ +"""The `vero harbor build` compiler: BuildConfig -> a runnable Harbor task dir. + +Emits the environment (optimizer workbench `main` + eval `eval-sidecar`), the +protocol (instruction.md), the verifier (tests/test.sh -> `vero harbor finalize`), +and bakes the ServeConfig + dataset + baseline repo + vero source. The result runs +with `harbor run -p -a -m -e docker`. +""" + +from __future__ import annotations + +import logging +import re +import shutil +import subprocess +from pathlib import Path + +from jinja2 import Environment, FileSystemLoader + +from vero.harbor.build.config import BuildConfig + +logger = logging.getLogger(__name__) + +_TEMPLATES = Path(__file__).parent / "templates" + +# Container paths (must match the templates). +VERO_DIR = "/opt/vero" +AGENT_BASELINE = "/opt/agent-baseline" # sidecar engine workspace +WORK_AGENT = "/work/agent" # shared agent repo (main rw, sidecar ro) +VERO_HOME = "/opt/vero_home" +INNER_TASK = "/opt/inner-task" # Mode B: baked inner Harbor task (the protected benchmark) +SERVE_JSON = "/opt/serve.json" +ADMIN_VOLUME = "/state/admin" +AGENT_VOLUME = "/state/agent-results" +TOKEN_PATH = "/state/token/admin.token" +SESSION_ID = "trial" + +# vero source items copied into the build context (enough to `uv pip install`). +_VERO_COPY = ["pyproject.toml", "README.md", "uv.lock", "src"] + + +def _render(env: Environment, template_name: str, dest: Path, **ctx) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text(env.get_template(template_name).render(**ctx)) + + +def _copy_vero_source(vero_root: Path, dest: Path) -> None: + dest.mkdir(parents=True, exist_ok=True) + for item in _VERO_COPY: + src = vero_root / item + if not src.exists(): + continue + if src.is_dir(): + shutil.copytree(src, dest / item, dirs_exist_ok=True) + else: + shutil.copy2(src, dest / item) + + +def _rewrite_vero_source_path(pyproject: Path) -> None: + """Point a relative `scale-vero` path dependency at the baked /opt/vero so it + resolves regardless of where the repo (or a temp worktree of it) lives.""" + if not pyproject.exists(): + return + text = pyproject.read_text() + new = re.sub( + r'(scale-vero\s*=\s*\{[^}]*?path\s*=\s*")[^"]*(")', + rf"\g<1>{VERO_DIR}\g<2>", + text, + ) + if new != text: + pyproject.write_text(new) + logger.info("Rewrote scale-vero source path -> %s", VERO_DIR) + + +def _prepare_baseline_repo(agent_repo: Path, dest: Path) -> str: + """Materialize the target repo at HEAD into a clean standalone git repo + (vero path rewritten) and return its commit sha. Copied verbatim (incl. .git) + into both the sidecar (engine workspace) and main (seed), so they share a sha.""" + dest.mkdir(parents=True, exist_ok=True) + toplevel = subprocess.run( + ["git", "-C", str(agent_repo), "rev-parse", "--show-toplevel"], + capture_output=True, text=True, + ) + if toplevel.returncode == 0: + # Extract only the target subtree at HEAD (the repo may be a monorepo and + # agent_repo a subdirectory of it), stripping the leading path components. + repo_root = Path(toplevel.stdout.strip()) + rel = agent_repo.relative_to(repo_root) + strip = len(rel.parts) + archive = subprocess.Popen( + ["git", "-C", str(repo_root), "archive", "HEAD", str(rel)] + if strip else ["git", "-C", str(repo_root), "archive", "HEAD"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + try: + subprocess.run( + ["tar", "xf", "-", "--strip-components", str(strip)], + cwd=dest, stdin=archive.stdout, check=True, + ) + finally: + # Let git see SIGPIPE if tar died, then reap it (no zombie). + if archive.stdout is not None: + archive.stdout.close() + archive_err = (archive.communicate()[1] or b"").decode(errors="replace") + # A failed `git archive` can emit a truncated stream that `tar` still + # accepts with exit 0, baking a near-empty baseline. Fail loudly instead. + if archive.returncode != 0: + raise RuntimeError( + f"git archive failed (exit {archive.returncode}) for {repo_root}: " + f"{archive_err.strip()}" + ) + else: + shutil.copytree(agent_repo, dest, dirs_exist_ok=True) + + _rewrite_vero_source_path(dest / "pyproject.toml") + + def git(*args: str) -> str: + return subprocess.run( + ["git", "-c", "user.name=vero", "-c", "user.email=vero@localhost", + "-C", str(dest), *args], + capture_output=True, text=True, check=True, + ).stdout.strip() + + git("init", "-q") + git("add", "-A") + git("commit", "-q", "-m", "baseline") + return git("rev-parse", "HEAD") + + +def _register(dataset, vero_home: Path, tmp: Path) -> str: + """Register a dataset (path/DatasetDict) into a baked VERO_HOME; return dataset_id.""" + from vero.core.dataset.store import resolve_and_save_dataset + + sessions = vero_home / "sessions" + datasets = vero_home / "datasets" + (sessions / SESSION_ID).mkdir(parents=True, exist_ok=True) + datasets.mkdir(parents=True, exist_ok=True) + if not isinstance(dataset, str): # a DatasetDict -> save_to_disk first + path = tmp / "ds" + dataset.save_to_disk(str(path)) + dataset = str(path) + return resolve_and_save_dataset(dataset, sessions, datasets, SESSION_ID) + + +def _serve_config(config: BuildConfig, dataset_id: str | None, base_commit: str) -> dict: + harbor = None + if config.harbor is not None: + # Local inner task -> baked sidecar-only path; registry ref -> pass through. + harbor = {**config.harbor} + if config.inner_task: + harbor["task_source"] = INNER_TASK + targets = [ + { + "task": config.task, + "dataset_id": dataset_id, + "split": t.split, + "reward_key": t.reward_key, + "sample_ids": t.sample_ids, + } + for t in config.targets + ] + return { + "repo_path": AGENT_BASELINE, + "agent_repo_path": WORK_AGENT, + "session_id": SESSION_ID, + "dataset_id": dataset_id, + "split_accesses": [s.model_dump() for s in config.splits], + "budgets": [ + {"split": b.split, "dataset_id": dataset_id, **b.model_dump(exclude={"split"}, exclude_none=True)} + for b in config.budgets + ], + "task": config.task, + "task_project": config.task_project, + "task_module": config.task_module, + "harbor": harbor, + "reward_mode": config.reward_mode, + "selection_split": config.selection_split, + "targets": targets, + "base_commit": base_commit, + "submit_enabled": config.submit_enabled, + "agent_volume": AGENT_VOLUME, + "admin_volume": ADMIN_VOLUME, + "admin_token_path": TOKEN_PATH, + "timeout": config.timeout, + "sample_timeout": config.sample_timeout, + "max_concurrency": config.max_concurrency, + "host": "0.0.0.0", + "port": 8000, + } + + +def compile_task( + config: BuildConfig, out_dir: Path | str, *, vero_root: Path | None = None +) -> Path: + """Compile ``config`` into a Harbor task directory at ``out_dir``.""" + import json + + from vero.core.constants import PACKAGE_DIR + + vero_root = vero_root or PACKAGE_DIR + out = Path(out_dir) + if out.exists(): + shutil.rmtree(out) + env_dir = out / "environment" + env_dir.mkdir(parents=True) + + agent_repo = Path(config.agent_repo).resolve() + + # 1. vero source (both images install from here) + _copy_vero_source(vero_root, env_dir / "vero") + + # 2. baseline repo -> sidecar engine workspace + main seed (shared sha) + base_commit = _prepare_baseline_repo(agent_repo, env_dir / "agent-baseline") + shutil.copytree(env_dir / "agent-baseline", env_dir / "agent-seed") + + # 3. dataset -> baked VERO_HOME. Mode A: input+label rows. Mode B: the + # {split: [task_names]} partition + the inner Harbor task baked sidecar-only. + import tempfile + + vh = env_dir / "sidecar" / "vero_home" + # Stage the dataset in a scratch dir that is always cleaned up (datasets can be + # gigabytes; a leaked mkdtemp would accumulate across builds). _register copies + # the dataset into vh before the dir is torn down, so cleanup is safe. + with tempfile.TemporaryDirectory() as tmp_str: + tmp = Path(tmp_str) + if config.mode == "A": + if not config.dataset: + raise ValueError("Mode A requires a dataset.") + dataset_id = _register(config.dataset, vh, tmp) + else: + if not (config.partition and config.harbor): + raise ValueError("Mode B requires partition + harbor.") + if not (config.inner_task or config.harbor.get("task_source")): + raise ValueError("Mode B requires inner_task (local) or harbor.task_source (registry).") + from vero.harbor.dataset import build_harbor_dataset + + dataset_id = _register(build_harbor_dataset(config.partition), vh, tmp) + if config.inner_task: # local benchmark -> bake sidecar-only + shutil.copytree(Path(config.inner_task).resolve(), env_dir / "sidecar" / "inner-task") + + # 4. ServeConfig (compiler <-> serve contract) + (env_dir / "sidecar" / "serve.json").write_text( + json.dumps(_serve_config(config, dataset_id, base_commit), indent=2) + ) + + # 4b. Fail early if a declared secret is missing from the host env, so the + # operator finds out at build time rather than via a credential-less + # sidecar. The compose ${VAR:?} guard is the run-time backstop. + import os + + if not os.environ.get("VERO_SKIP_SECRET_CHECK"): + missing = [s for s in config.secrets if not os.environ.get(s)] + if missing: + raise ValueError( + "Declared secrets missing from the host environment: " + f"{', '.join(missing)}. Set them, or set VERO_SKIP_SECRET_CHECK=1 " + "to defer to the run-time compose check." + ) + + # 5. render templates + jenv = Environment( + loader=FileSystemLoader(str(_TEMPLATES)), + keep_trailing_newline=True, + trim_blocks=True, + lstrip_blocks=True, + ) + ctx = dict( + name=config.name, + description=config.description, + mode=config.mode, + timeout=config.timeout, + secrets=config.secrets, + read_only_paths=config.read_only_paths, + base_image_main=config.base_image_main, + base_image_sidecar=config.base_image_sidecar, + dataset_id=dataset_id, + selection_split=config.selection_split, + submit_enabled=config.submit_enabled, + eval_num_samples=None, + bake_inner_task=bool(config.inner_task), + ) + _render(jenv, "task.toml.j2", out / "task.toml", **ctx) + _render(jenv, "instruction.md.j2", out / "instruction.md", **ctx) + _render(jenv, "docker-compose.yaml.j2", env_dir / "docker-compose.yaml", **ctx) + _render(jenv, "Dockerfile.main.j2", env_dir / "Dockerfile", **ctx) + _render(jenv, "Dockerfile.sidecar.j2", env_dir / "sidecar" / "Dockerfile", **ctx) + _render(jenv, "seed.sh.j2", env_dir / "main" / "seed.sh", **ctx) + _render(jenv, "test.sh.j2", out / "tests" / "test.sh", **ctx) + _render(jenv, "solve.sh.j2", out / "solution" / "solve.sh", **ctx) + + for script in [out / "tests" / "test.sh", out / "solution" / "solve.sh", + env_dir / "main" / "seed.sh"]: + script.chmod(0o755) + + logger.info("Compiled Harbor task -> %s (baseline %s)", out, base_commit[:12]) + return out diff --git a/vero/src/vero/harbor/build/config.py b/vero/src/vero/harbor/build/config.py new file mode 100644 index 0000000..7be37b7 --- /dev/null +++ b/vero/src/vero/harbor/build/config.py @@ -0,0 +1,97 @@ +"""`BuildConfig` — the `vero harbor build -c build.yaml` schema. + +Everything the compiler needs to emit a Harbor optimization task. Mode A (vero +runs inference + scoring) and Mode B (nested `harbor run`) share one topology; +the differences are which extras the sidecar bakes and which secrets it needs. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Literal + +import yaml +from pydantic import BaseModel, Field + + +class SplitAccessSpec(BaseModel): + split: str + access: Literal["viewable", "non_viewable", "no_access"] + + +class BudgetSpec(BaseModel): + split: str + total_run_budget: int | None = None + total_sample_budget: int | None = None + + +class TargetSpec(BaseModel): + """A scoring target the verifier evaluates the selected commit on.""" + + split: str + reward_key: str = "reward" + sample_ids: list[int] | None = None + + +class BuildConfig(BaseModel): + """Inputs to `vero harbor build`.""" + + # identity + name: str = Field(description="Harbor task name, 'org/name' format.") + description: str = "" + + # the target repo the optimizer edits (baseline in main + sidecar) + agent_repo: str + + # mode A (scoring in vero): task name + dataset (+ optional separate task project) + mode: Literal["A", "B"] = "A" + task: str | None = None + task_project: str | None = None + task_module: str | None = None + dataset: str | None = Field( + default=None, description="Path to a saved DatasetDict (Mode A)." + ) + + # mode B (scoring in nested harbor): HarborConfig kwargs (task_source filled by the + # compiler from inner_task), the {split: [task_names]} partition, and the inner + # Harbor task dir baked sidecar-only (the protected benchmark, mirrors Mode A's dataset). + harbor: dict | None = None + partition: dict[str, list[str]] | None = None + inner_task: str | None = None + + # tiers / budget / reward + splits: list[SplitAccessSpec] + budgets: list[BudgetSpec] = Field(default_factory=list) + reward_mode: Literal["submit", "auto_best"] = "auto_best" + selection_split: str = "validation" + targets: list[TargetSpec] = Field(default_factory=list) + submit_enabled: bool = False + + # write-access: paths in the target repo the optimizer may NOT edit + # (the scorer, by default). Applied as unix perms in main before the agent runs. + read_only_paths: list[str] = Field(default_factory=list) + + # secrets resolved from the host and injected into the SIDECAR only + secrets: list[str] = Field(default_factory=lambda: ["OPENAI_API_KEY"]) + + # image bases + base_image_main: str = "ghcr.io/astral-sh/uv:python3.12-bookworm" + base_image_sidecar: str = "ghcr.io/astral-sh/uv:python3.12-bookworm" + + # eval params baked into the ServeConfig + timeout: int = 1800 + sample_timeout: int = 300 + max_concurrency: int = 8 + + @classmethod + def from_file(cls, path: Path | str) -> BuildConfig: + path = Path(path).resolve() + data = yaml.safe_load(path.read_text()) + # Resolve relative local-path fields against the build.yaml's directory, so a + # config is portable regardless of the working directory it's built from. + base = path.parent + for field in ("agent_repo", "dataset", "inner_task"): + val = data.get(field) + if isinstance(val, str) and not Path(val).is_absolute(): + data[field] = str((base / val).resolve()) + return cls.model_validate(data) diff --git a/vero/src/vero/harbor/build/templates/Dockerfile.main.j2 b/vero/src/vero/harbor/build/templates/Dockerfile.main.j2 new file mode 100644 index 0000000..0861553 --- /dev/null +++ b/vero/src/vero/harbor/build/templates/Dockerfile.main.j2 @@ -0,0 +1,20 @@ +# main: the optimizer's workbench. Harbor installs the `-a` optimizer agent here +# and runs it against instruction.md. Holds the target repo (rw, minus locked +# paths) + the `vero` CLI client. Runs the container as root (for seed + verifier); +# the optimizer is exec'd as the de-privileged `agent` user. +FROM {{ base_image_main }} + +RUN apt-get update \ + && apt-get install -y --no-install-recommends git ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* + +# vero + CLI client (eval / submit / status / finalize over VERO_EVAL_URL) +COPY vero /opt/vero +RUN uv pip install --system "/opt/vero[harbor]" + +# baseline target repo (seeded onto the shared volume at start) + the seed script +COPY agent-seed /opt/agent-seed +COPY main/seed.sh /opt/seed.sh +RUN chmod +x /opt/seed.sh && useradd -m -u 1001 agent + +WORKDIR /work/agent diff --git a/vero/src/vero/harbor/build/templates/Dockerfile.sidecar.j2 b/vero/src/vero/harbor/build/templates/Dockerfile.sidecar.j2 new file mode 100644 index 0000000..7eea688 --- /dev/null +++ b/vero/src/vero/harbor/build/templates/Dockerfile.sidecar.j2 @@ -0,0 +1,29 @@ +# eval-sidecar: the evaluation engine. Holds the dataset + scoring + baseline repo +# + ledger + creds. Runs `vero harbor serve` (HTTP). Secrets reach this container +# only (compose); the admin volume is never mounted to main. +FROM {{ base_image_sidecar }} + +RUN apt-get update \ + && apt-get install -y --no-install-recommends git ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +COPY vero /opt/vero +RUN uv pip install --system "/opt/vero[harbor]" + +# baseline repo = the engine's GitWorkspace (fetches the optimizer's commits from +# the ro-mounted /work/agent); baked vero_home (registered dataset{% if mode == 'A' %} + scoring{% endif %}). +COPY agent-baseline /opt/agent-baseline +COPY sidecar/vero_home /opt/vero_home +COPY sidecar/serve.json /opt/serve.json +{% if bake_inner_task %} +# inner Harbor task (the protected benchmark the candidate agent is run against) +COPY sidecar/inner-task /opt/inner-task +{% endif %} + +# warm the uv cache so eval-time `uv run --project ` resolves offline-fast +RUN cd /opt/agent-baseline && uv sync 2>/dev/null || true + +# allow the engine to fetch from the ro-mounted agent repo (different owner) +RUN git config --system --add safe.directory '*' + +WORKDIR /opt diff --git a/vero/src/vero/harbor/build/templates/docker-compose.yaml.j2 b/vero/src/vero/harbor/build/templates/docker-compose.yaml.j2 new file mode 100644 index 0000000..9ec2b40 --- /dev/null +++ b/vero/src/vero/harbor/build/templates/docker-compose.yaml.j2 @@ -0,0 +1,47 @@ +# Merged LAST by Harbor over its build template (which auto-configures `main` +# from environment/Dockerfile). We add the eval-sidecar + volumes and wire main. +services: + main: + # Run as root so the seed step can chown the repo and the verifier (shared + # mode) can read the root:600 admin token. Harbor execs the optimizer as the + # [agent].user ("agent") declared in task.toml. + command: ["/opt/seed.sh"] + environment: + VERO_EVAL_URL: "http://eval-sidecar:8000" + volumes: + - agent_repo:/work/agent + - agent_results:/state/agent-results:ro + - token_state:/state/token:ro + depends_on: + eval-sidecar: + condition: service_healthy + + eval-sidecar: + build: + context: . + dockerfile: sidecar/Dockerfile + command: ["vero", "harbor", "serve", "--config", "/opt/serve.json"] + environment: + VERO_HOME_DIR: "/opt/vero_home" +{% for secret in secrets %} + {# Fail-fast: docker compose aborts if this host var is unset/empty, rather + than silently baking an empty credential into the sidecar. #} + {{ secret }}: "${{ '{' }}{{ secret }}:?{{ secret }} must be set in the host environment for the eval sidecar{{ '}' }}" +{% endfor %} + volumes: + - agent_repo:/work/agent:ro + - agent_results:/state/agent-results + - admin_state:/state/admin + - token_state:/state/token + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://localhost:8000/health').status==200 else 1)"] + interval: 5s + timeout: 10s + retries: 30 + start_period: 10s + +volumes: + agent_repo: + agent_results: + admin_state: + token_state: diff --git a/vero/src/vero/harbor/build/templates/instruction.md.j2 b/vero/src/vero/harbor/build/templates/instruction.md.j2 new file mode 100644 index 0000000..1e11430 --- /dev/null +++ b/vero/src/vero/harbor/build/templates/instruction.md.j2 @@ -0,0 +1,28 @@ +# Optimization task + +You are optimizing the code in `/work/agent`. Improve it so it scores as high as +possible on a **hidden test split** — but you never see the test split. You measure +progress on the splits you *are* allowed to evaluate, within a fixed budget. + +## Workflow + +1. Edit the repo at `/work/agent`. Some paths are read-only (the scorer) — leave them. +2. Commit your changes (`git commit`). +3. Measure a commit on an allowed split: + + ``` + vero harbor eval --dataset-id {{ dataset_id }} --split {{ selection_split }} + ``` + + (defaults to your current `HEAD`). Returns an aggregate score and remaining budget. +4. Check budget / which splits are evaluable anytime: `vero harbor status`. +{% if submit_enabled %}5. When done, nominate your best commit: `vero harbor submit`.{% else %} +The best commit you evaluate on `{{ selection_split }}` is selected automatically and +scored on the hidden test split at the end.{% endif %} + +## Rules + +- Budget is finite and metered per split — spend it wisely. +- The test split is hidden: you cannot evaluate it, and its labels never reach this + container. Trying to read it will fail. +- The scorer is locked. Only the eval sidecar scores. diff --git a/vero/src/vero/harbor/build/templates/seed.sh.j2 b/vero/src/vero/harbor/build/templates/seed.sh.j2 new file mode 100644 index 0000000..6e16ab2 --- /dev/null +++ b/vero/src/vero/harbor/build/templates/seed.sh.j2 @@ -0,0 +1,28 @@ +#!/bin/sh +# Seed the optimizer's working repo onto the shared volume and apply write-access +# rules, then keep `main` alive. Runs as root at container start. +set -e + +if [ ! -d /work/agent/.git ]; then + cp -a /opt/agent-seed/. /work/agent/ +fi + +# Whole repo is the optimizer's to edit... +chown -R agent:agent /work/agent +git config --system --add safe.directory /work/agent +# NOTE: read_only_paths is ADVISORY ONLY. The chmod/chown below locks paths in +# the live working tree (/work/agent), but the verifier checks out git blobs from +# the agent's commit, not this working tree, so an optimizer can still stage edits +# to a "locked" scorer path. Authoritative scorer-provenance is enforced +# sidecar-side: in Mode A the verifier scores with the sidecar-baked task project +# (task_project), never the scorer in the agent's commit. Treat this purely as a +# guardrail / footgun-reducer, not security. +{% for p in read_only_paths %} +# ...except locked paths (e.g. the scorer): root-owned + unwritable (advisory). +if [ -e "/work/agent/{{ p }}" ]; then + chown -R root:root "/work/agent/{{ p }}" + chmod -R a-w "/work/agent/{{ p }}" +fi +{% endfor %} + +exec sleep infinity diff --git a/vero/src/vero/harbor/build/templates/solve.sh.j2 b/vero/src/vero/harbor/build/templates/solve.sh.j2 new file mode 100644 index 0000000..bc97e5e --- /dev/null +++ b/vero/src/vero/harbor/build/templates/solve.sh.j2 @@ -0,0 +1,17 @@ +#!/bin/bash +# Oracle "optimizer" used for the e2e smoke test: make one trivial edit, commit, +# and measure it on the selection split. The auto-best verifier then scores the +# selected commit on the hidden test split. A real optimizer agent replaces this. +set -ex +cd /work/agent +git config user.email optimizer@example.com +git config user.name optimizer + +# A no-op-ish "improvement" so there is a non-baseline commit to select. +echo "# optimizer touch" >> README.md 2>/dev/null || echo "# optimizer touch" > NOTES.md +git add -A +git commit -m "optimizer candidate" + +vero harbor eval --dataset-id {{ dataset_id }} --split {{ selection_split }}{% if eval_num_samples %} --num-samples {{ eval_num_samples }}{% endif %} + +vero harbor status diff --git a/vero/src/vero/harbor/build/templates/task.toml.j2 b/vero/src/vero/harbor/build/templates/task.toml.j2 new file mode 100644 index 0000000..c037e22 --- /dev/null +++ b/vero/src/vero/harbor/build/templates/task.toml.j2 @@ -0,0 +1,23 @@ +schema_version = "1.3" + +[task] +name = "{{ name }}" +description = "{{ description }}" + +[agent] +# The optimizer runs as a de-privileged user so it cannot read the admin token +# (root:600) or the admin volume. It edits the target repo + calls `vero harbor eval`. +user = "agent" + +[verifier] +# Shared mode: Harbor runs tests/test.sh in `main` with the whole env (incl. the +# eval-sidecar) still up. The verifier runs as root, reads the admin token, and +# calls the sidecar's `finalize` endpoint to score the selected commit. +environment_mode = "shared" +timeout_sec = {{ timeout }} + +[environment] +# Compose-based environment: environment/docker-compose.yaml adds the eval-sidecar +# service + volumes and wires `main`. Secrets are injected into the sidecar only +# (see the compose file), never declared here (this section's env reaches `main`). +build_timeout_sec = 1800 diff --git a/vero/src/vero/harbor/build/templates/test.sh.j2 b/vero/src/vero/harbor/build/templates/test.sh.j2 new file mode 100644 index 0000000..f65e477 --- /dev/null +++ b/vero/src/vero/harbor/build/templates/test.sh.j2 @@ -0,0 +1,10 @@ +#!/bin/bash +# Verifier (shared mode, root). Reads the admin token (root:600, unreadable by the +# optimizer) and asks the eval sidecar to select + score the commit on the hidden +# test split, writing the reward. +set -e +mkdir -p /logs/verifier +vero harbor finalize \ + --token-file /state/token/admin.token \ + --output /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/vero/src/vero/harbor/runner.py b/vero/src/vero/harbor/runner.py new file mode 100644 index 0000000..8b63038 --- /dev/null +++ b/vero/src/vero/harbor/runner.py @@ -0,0 +1,252 @@ +"""HarborRunner — the Mode-B evaluation strategy. + +Implements ``EvalStrategy``: for a checked-out candidate, runs a nested ``harbor run`` +(in the candidate's own uv env) over the Harbor tasks selected by the split/sample_ids, +then collates the jobs dir into vero ``SampleResult``s. One Harbor task = one sample. + +Shells out to the ``harbor`` CLI (no harbor import here) and reads trial ``result.json`` +as plain dicts, so ``vero`` itself needs no ``harbor`` dependency at runtime. +""" + +from __future__ import annotations + +import json +import logging +from pathlib import Path +from typing import TYPE_CHECKING + +from vero.core.db.dataset import DatasetSample +from vero.core.db.result import SampleResult +from vero.core.sessions import ( + get_vero_home_dir, + load_sample_result, + save_sample_result, +) +from vero.harbor.config import HarborConfig +from vero.utils import run_subprocess_with_tee + +if TYPE_CHECKING: + from vero.core.evaluation import EvaluationParameters + from vero.workspace import Workspace + +logger = logging.getLogger(__name__) + + +class HarborRunner: + """Mode-B EvalStrategy: nested `harbor run` + collate -> SampleResults.""" + + def __init__(self, config: HarborConfig): + self.config = config + + async def produce_sample_results( + self, + *, + workspace: Workspace, + params: EvaluationParameters, + result_dir: Path, + ) -> None: + pairs = self._task_names_for(params) # [(sample_id, task_name), ...] + if not pairs: + return + jobs_dir = Path(result_dir) / "jobs" + + # Resume: only run tasks not already completed successfully. A persisted + # *error* sample (transient harbor/verifier failure) is NOT done, so it is + # re-run rather than permanently skipped. + pending = [(sid, t) for sid, t in pairs if not self._is_done(params, sid)] + if pending: + await self._run_harbor( + str(workspace.project_path), params, [t for _, t in pending], jobs_dir + ) + self._collate(jobs_dir, pairs, params) + + # ------------------------------------------------------------------ + # Task selection (host-side; just task names) + # ------------------------------------------------------------------ + + def _task_names_for(self, params: EvaluationParameters) -> list[tuple[int, str]]: + from vero.core.dataset.store import load_dataset + + vero_home = get_vero_home_dir() + dataset = load_dataset( + vero_home / "sessions", + vero_home / "datasets", + params.session_id, + params.run.dataset_subset.dataset_id, + ) + split = dataset[params.run.dataset_subset.split] + ids = params.run.dataset_subset.sample_ids + if ids is None: + ids = list(range(len(split))) + return [(i, split[i]["task_name"]) for i in ids] + + # ------------------------------------------------------------------ + # Execute + # ------------------------------------------------------------------ + + def _build_command( + self, + project_path: str, + params: EvaluationParameters, + task_names: list[str], + jobs_dir: Path, + ) -> list[str]: + c = self.config + cmd = [ + "uv", "run", "--project", project_path, + "harbor", "run", + *c.source_args(), + "--agent-import-path", c.agent_import_path, + "-e", c.environment, + "-n", str(params.max_concurrency), + "--n-attempts", str(c.n_attempts), + "--max-retries", str(c.max_retries), + ] + if c.model: + cmd += ["-m", c.model] + for task_name in task_names: + cmd += ["-i", task_name] + cmd += ["--jobs-dir", str(jobs_dir), *c.extra_args] + return cmd + + async def _run_harbor( + self, + project_path: str, + params: EvaluationParameters, + task_names: list[str], + jobs_dir: Path, + ) -> None: + cmd = self._build_command(project_path, params, task_names, jobs_dir) + logger.info(f"Mode B: {' '.join(cmd)}") + result = await run_subprocess_with_tee( + cmd, timeout=params.timeout, cwd=project_path + ) + # Non-zero is not fatal: partial trials may still exist; collation fills gaps. + if result.returncode != 0: + logger.warning( + f"`harbor run` exited {result.returncode}: " + f"{(result.stderr or '')[:500]}" + ) + + # ------------------------------------------------------------------ + # Collate + # ------------------------------------------------------------------ + + def _collate( + self, + jobs_dir: Path, + pairs: list[tuple[int, str]], + params: EvaluationParameters, + ) -> None: + trials = self._load_trials(jobs_dir) # {task_name: result_dict} + for sample_id, task_name in pairs: + if self._is_done(params, sample_id): + continue # already collated successfully (resume); errors are redone + sample_result = self._sample_result( + trials.get(task_name), sample_id, task_name, params + ) + save_sample_result( + get_vero_home_dir() / "sessions", + params.session_id, + params.result_id, + sample_id=sample_id, + result=sample_result, + ) + + def _load_trials(self, jobs_dir: Path) -> dict[str, dict]: + trials: dict[str, dict] = {} + if not jobs_dir.exists(): + return trials + # Trial result.json files live at ///result.json; the + # job-level //result.json carries no task_name, so recurse and + # key on task_name (skipping the job summary). A task may have several trials + # (retries / multiple attempts); rglob order is undefined, so keep the BEST + # trial per task deterministically rather than last-write-wins (a failing + # retry must never clobber a passing trial). + best_rank: dict[str, tuple] = {} + for result_json in jobs_dir.rglob("result.json"): + try: + data = json.loads(result_json.read_text()) + except (json.JSONDecodeError, OSError): + continue + task_name = data.get("task_name") + if not task_name: + continue + rank = self._trial_rank(data, result_json) + if task_name not in best_rank or rank > best_rank[task_name]: + best_rank[task_name] = rank + trials[task_name] = data + return trials + + @staticmethod + def _trial_rank(data: dict, result_json: Path) -> tuple: + """Sort key for picking the best of several trials of one task. Higher wins: + prefer a clean trial with rewards, then any trial with rewards, then the most + recent attempt (finished_at, falling back to file mtime).""" + has_rewards = bool((data.get("verifier_result") or {}).get("rewards")) + clean = has_rewards and not data.get("exception_info") + finished_at = data.get("finished_at") or "" + try: + mtime = result_json.stat().st_mtime + except OSError: + mtime = 0.0 + return (clean, has_rewards, finished_at, mtime) + + def _sample_result( + self, + trial: dict | None, + sample_id: int, + task_name: str, + params: EvaluationParameters, + ) -> SampleResult: + common = { + "dataset_sample": DatasetSample( + sample_id=sample_id, + split=params.run.dataset_subset.split, + dataset_id=params.run.dataset_subset.dataset_id, + ), + "commit": params.run.candidate.commit, + "result_id": params.result_id, + } + if trial is None: + return SampleResult( + error=f"No Harbor trial result for task '{task_name}'.", **common + ) + rewards = (trial.get("verifier_result") or {}).get("rewards") or {} + if not rewards: + return SampleResult( + error=f"No verifier rewards for task '{task_name}'.", + output={"task_name": task_name, "trial_name": trial.get("trial_name")}, + **common, + ) + return SampleResult( + score=self._extract_reward(rewards), + metrics={k: float(v) for k, v in rewards.items()}, + output={ + "task_name": task_name, + "trial_name": trial.get("trial_name"), + "rewards": rewards, + }, + **common, + ) + + def _extract_reward(self, rewards: dict) -> float: + for key in (self.config.reward_key, "pass", "reward"): + if key and key in rewards: + return float(rewards[key]) + values = [float(v) for v in rewards.values()] + return sum(values) / len(values) if values else 0.0 + + def _existing(self, params: EvaluationParameters, sample_id: int) -> SampleResult | None: + return load_sample_result( + get_vero_home_dir() / "sessions", + params.session_id, + params.result_id, + sample_id, + ) + + def _is_done(self, params: EvaluationParameters, sample_id: int) -> bool: + """A sample is done only if a persisted result exists AND is not an error; + a transiently-failed sample must be re-run on resume, not skipped.""" + existing = self._existing(params, sample_id) + return existing is not None and not existing.is_error() diff --git a/vero/tests/test_harbor_build.py b/vero/tests/test_harbor_build.py new file mode 100644 index 0000000..27416af --- /dev/null +++ b/vero/tests/test_harbor_build.py @@ -0,0 +1,169 @@ +"""Unit test for the `vero harbor build` compiler: a BuildConfig compiles to a +well-formed Harbor task directory whose ServeConfig validates and whose rendered +task.toml / compose / scripts parse. No Docker (that's the e2e).""" + +from __future__ import annotations + +import json +import subprocess +import tomllib +from pathlib import Path + +import pytest +import yaml + +from vero.harbor.build import BuildConfig, compile_task +from vero.harbor.serve import ServeConfig + + +def _stub_vero(root: Path) -> Path: + """A minimal stand-in for the vero source tree (compiler just copies it).""" + d = root / "vero-src" + (d / "src" / "vero").mkdir(parents=True) + (d / "pyproject.toml").write_text("[project]\nname='scale-vero'\nversion='0'\n") + (d / "README.md").write_text("vero\n") + (d / "src" / "vero" / "__init__.py").write_text("") + return d + + +def _agent_repo(root: Path) -> Path: + d = root / "agent" + (d / "src" / "gsm8k_agent").mkdir(parents=True) + (d / "pyproject.toml").write_text( + "[project]\nname='gsm8k-agent'\nversion='0'\n\n" + '[tool.uv.sources]\nscale-vero = { path = "../../", editable = true }\n' + ) + (d / "src" / "gsm8k_agent" / "agent.py").write_text("X = 1\n") + subprocess.run(["git", "init", "-q"], cwd=d, check=True) + subprocess.run(["git", "add", "-A"], cwd=d, check=True) + subprocess.run( + ["git", "-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", "i"], + cwd=d, check=True, + ) + return d + + +def _dataset(root: Path) -> Path: + from datasets import Dataset, DatasetDict + + ds = DatasetDict({ + "validation": Dataset.from_dict({"question": ["1+1?"], "answer": ["#### 2"]}), + "test": Dataset.from_dict({"question": ["2+2?"], "answer": ["#### 4"]}), + }) + p = root / "ds" + ds.save_to_disk(str(p)) + return p + + +@pytest.fixture +def built(tmp_path, monkeypatch): + monkeypatch.setenv("VERO_SKIP_SECRET_CHECK", "1") + config = BuildConfig( + name="vero/gsm8k-opt", + description="optimize gsm8k", + agent_repo=str(_agent_repo(tmp_path)), + mode="A", + task="gsm8k", + task_module="gsm8k_agent.vero_tasks", + dataset=str(_dataset(tmp_path)), + splits=[ + {"split": "validation", "access": "non_viewable"}, + {"split": "test", "access": "no_access"}, + ], + budgets=[{"split": "validation", "total_run_budget": 5}], + reward_mode="auto_best", + selection_split="validation", + targets=[{"split": "test", "reward_key": "reward"}], + read_only_paths=["src/gsm8k_agent/vero_tasks"], + secrets=["OPENAI_API_KEY"], + ) + out = compile_task(config, tmp_path / "task", vero_root=_stub_vero(tmp_path)) + return out + + +def test_structure(built): + for rel in [ + "task.toml", "instruction.md", + "environment/docker-compose.yaml", "environment/Dockerfile", + "environment/sidecar/Dockerfile", "environment/sidecar/serve.json", + "environment/main/seed.sh", "environment/vero/pyproject.toml", + "environment/agent-baseline/.git", "environment/agent-seed/.git", + "environment/sidecar/vero_home", "tests/test.sh", "solution/solve.sh", + ]: + assert (built / rel).exists(), f"missing {rel}" + + +def test_serve_config_validates(built): + cfg = ServeConfig.from_file(built / "environment" / "sidecar" / "serve.json") + assert cfg.repo_path == "/opt/agent-baseline" + assert cfg.agent_repo_path == "/work/agent" + assert cfg.task == "gsm8k" + assert cfg.dataset_id # registered + assert cfg.base_commit # baseline sha recorded for auto_best exclusion + assert cfg.targets and cfg.targets[0].split == "test" + assert cfg.budgets[0]["dataset_id"] == cfg.dataset_id + + +def test_rendered_files_parse(built): + tomllib.loads((built / "task.toml").read_text()) # valid TOML + compose = yaml.safe_load((built / "environment/docker-compose.yaml").read_text()) + assert "eval-sidecar" in compose["services"] + assert compose["services"]["main"]["depends_on"]["eval-sidecar"]["condition"] == "service_healthy" + # secret reaches the sidecar only, via host-resolved compose interpolation + # with a fail-fast guard (${VAR:?msg}) so an unset host var aborts the run. + sidecar_secret = compose["services"]["eval-sidecar"]["environment"]["OPENAI_API_KEY"] + assert sidecar_secret.startswith("${OPENAI_API_KEY:?") + assert "OPENAI_API_KEY" not in compose["services"]["main"].get("environment", {}) + + +def test_vero_source_path_rewritten(built): + pyproject = (built / "environment/agent-baseline/pyproject.toml").read_text() + assert 'path = "/opt/vero"' in pyproject + assert "../../" not in pyproject + + +def test_baseline_sha_shared(built): + def head(p): + return subprocess.run( + ["git", "-C", str(built / p), "rev-parse", "HEAD"], + capture_output=True, text=True, check=True, + ).stdout.strip() + + assert head("environment/agent-baseline") == head("environment/agent-seed") + cfg = json.loads((built / "environment/sidecar/serve.json").read_text()) + assert cfg["base_commit"] == head("environment/agent-baseline") + + +def test_baseline_archive_failure_raises(tmp_path, monkeypatch): + monkeypatch.setenv("VERO_SKIP_SECRET_CHECK", "1") + # An empty git repo with no commits: `git archive HEAD` exits nonzero. + bad = tmp_path / "emptyrepo" + (bad / "src").mkdir(parents=True) + (bad / "pyproject.toml").write_text("[project]\nname='x'\nversion='0'\n") + subprocess.run(["git", "init", "-q"], cwd=bad, check=True) + config = BuildConfig( + name="vero/x", agent_repo=str(bad), mode="A", task="gsm8k", + dataset=str(_dataset(tmp_path)), + splits=[{"split": "validation", "access": "non_viewable"}], + ) + with pytest.raises(RuntimeError, match="git archive failed"): + compile_task(config, tmp_path / "task", vero_root=_stub_vero(tmp_path)) + + +def test_missing_secret_fails_build(tmp_path, monkeypatch): + monkeypatch.delenv("VERO_SKIP_SECRET_CHECK", raising=False) + monkeypatch.delenv("DEFINITELY_MISSING_SECRET", raising=False) + config = BuildConfig( + name="vero/x", agent_repo=str(_agent_repo(tmp_path)), mode="A", task="gsm8k", + dataset=str(_dataset(tmp_path)), + splits=[{"split": "validation", "access": "non_viewable"}], + secrets=["DEFINITELY_MISSING_SECRET"], + ) + with pytest.raises(ValueError, match="missing from the host environment"): + compile_task(config, tmp_path / "task", vero_root=_stub_vero(tmp_path)) + + +def test_seed_documents_advisory_read_only(built): + seed = (built / "environment/main/seed.sh").read_text() + assert "ADVISORY ONLY" in seed + assert "sidecar-side" in seed diff --git a/vero/tests/test_harbor_runner.py b/vero/tests/test_harbor_runner.py new file mode 100644 index 0000000..b81a80a --- /dev/null +++ b/vero/tests/test_harbor_runner.py @@ -0,0 +1,214 @@ +"""Tests for vero.harbor.runner.HarborRunner — command build, collation, resume.""" + +import json +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from vero.core.db.candidate import Candidate +from vero.core.db.dataset import DatasetSample, DatasetSubset +from vero.core.db.result import SampleResult +from vero.core.db.run import ExperimentRun +from vero.core.evaluation import EvaluationParameters +from vero.core.sessions import ( + get_vero_home_dir, + load_all_sample_results, + save_sample_result, +) +from vero.harbor.config import HarborConfig +from vero.harbor.runner import HarborRunner + + +def _runner(reward_key=None, task_source="org/ds@1"): + return HarborRunner( + HarborConfig( + task_source=task_source, + agent_import_path="pkg.mod:Agent", + model="anthropic/x", + environment="modal", + reward_key=reward_key, + ) + ) + + +def _params(): + return EvaluationParameters( + run=ExperimentRun( + candidate=Candidate(commit="c1", repo_name="r"), + dataset_subset=DatasetSubset(split="test", dataset_id="ds", sample_ids=[0, 1]), + ), + session_id="s", + ) + + +def _write_trial(jobs_dir: Path, trial: str, task_name: str, rewards: dict): + # Real harbor layout: ///result.json, plus a job-level + # //result.json summary (no task_name) that collation must skip. + run = jobs_dir / "2026-01-01__00-00-00" + d = run / trial + d.mkdir(parents=True, exist_ok=True) + (run / "result.json").write_text(json.dumps({"job": "summary"})) # job-level, no task_name + (d / "result.json").write_text( + json.dumps({"task_name": task_name, "trial_name": trial, "verifier_result": {"rewards": rewards}}) + ) + + +class TestBuildCommand: + def test_registry_source_and_flags(self): + cmd = _runner()._build_command("/wt", _params(), ["t0", "t1"], Path("/jobs")) + assert cmd[:5] == ["uv", "run", "--project", "/wt", "harbor"] + assert "-d" in cmd and "org/ds@1" in cmd + assert "--agent-import-path" in cmd and "pkg.mod:Agent" in cmd + assert cmd.count("-i") == 2 and "t0" in cmd and "t1" in cmd + assert "-m" in cmd and "-e" in cmd and "--jobs-dir" in cmd + + def test_local_source(self, tmp_path): + cmd = _runner(task_source=str(tmp_path))._build_command("/wt", _params(), ["t0"], Path("/jobs")) + assert "-p" in cmd and str(tmp_path) in cmd + assert "-d" not in cmd + + +class TestExtractReward: + def test_priority_pass_then_reward_then_mean(self): + r = _runner() + assert r._extract_reward({"pass": 1.0, "reward": 0.0}) == 1.0 + assert r._extract_reward({"reward": 0.7}) == 0.7 + assert r._extract_reward({"a": 0.2, "b": 0.4}) == pytest.approx(0.3) + + def test_reward_key_override(self): + assert _runner(reward_key="acc")._extract_reward({"acc": 0.9, "pass": 0.0}) == 0.9 + + +class TestCollate: + @pytest.mark.asyncio + async def test_produces_results_and_marks_missing(self, tmp_path, monkeypatch): + monkeypatch.setenv("VERO_HOME_DIR", str(tmp_path / "vh")) + runner = _runner() + params = _params() + result_dir = tmp_path / "result" + jobs = result_dir / "jobs" + _write_trial(jobs, "trial0", "t0", {"pass": 1.0, "extra": 0.5}) + # no trial for t1 + + monkeypatch.setattr(runner, "_task_names_for", lambda p: [(0, "t0"), (1, "t1")]) + runner._run_harbor = AsyncMock() # fixtures already present; don't shell out + + ws = MagicMock(project_path="/wt") + await runner.produce_sample_results(workspace=ws, params=params, result_dir=result_dir) + + results = load_all_sample_results(get_vero_home_dir() / "sessions", "s", params.result_id) + assert results[0].score == 1.0 + assert results[0].metrics["extra"] == 0.5 + assert results[1].error is not None # missing trial -> error sample + + @pytest.mark.asyncio + async def test_resume_only_runs_pending(self, tmp_path, monkeypatch): + monkeypatch.setenv("VERO_HOME_DIR", str(tmp_path / "vh")) + runner = _runner() + params = _params() + result_dir = tmp_path / "result" + + # sample 0 already done + save_sample_result( + get_vero_home_dir() / "sessions", "s", params.result_id, sample_id=0, + result=SampleResult( + dataset_sample=DatasetSample(sample_id=0, split="test", dataset_id="ds"), + score=1.0, commit="c1", result_id=params.result_id, + ), + ) + _write_trial(result_dir / "jobs", "trial1", "t1", {"pass": 0.0}) + monkeypatch.setattr(runner, "_task_names_for", lambda p: [(0, "t0"), (1, "t1")]) + runner._run_harbor = AsyncMock() + + ws = MagicMock(project_path="/wt") + await runner.produce_sample_results(workspace=ws, params=params, result_dir=result_dir) + + # only the pending task name was passed to harbor + assert runner._run_harbor.await_args.args[2] == ["t1"] + + +class TestReviewFixes: + def test_emits_attempts_and_retries(self): + runner = HarborRunner( + HarborConfig( + task_source="org/ds@1", + agent_import_path="pkg.mod:Agent", + model="anthropic/x", + environment="modal", + n_attempts=3, + max_retries=5, + ) + ) + cmd = runner._build_command("/wt", _params(), ["t0"], Path("/jobs")) + assert "--n-attempts" in cmd and cmd[cmd.index("--n-attempts") + 1] == "3" + assert "--max-retries" in cmd and cmd[cmd.index("--max-retries") + 1] == "5" + + def test_passing_trial_wins_over_later_failing_retry(self, tmp_path): + runner = _runner() + jobs = tmp_path / "jobs" + run = jobs / "2026-01-01__00-00-00" + # passing trial, earlier finished_at + good = run / "trial0" + good.mkdir(parents=True) + (good / "result.json").write_text(json.dumps({ + "task_name": "t0", "trial_name": "trial0", "finished_at": "2026-01-01T00:01:00", + "verifier_result": {"rewards": {"pass": 1.0}}, + })) + # failing retry, later finished_at + exception_info; written second (newer mtime) + bad = run / "trial1" + bad.mkdir(parents=True) + (bad / "result.json").write_text(json.dumps({ + "task_name": "t0", "trial_name": "trial1", "finished_at": "2026-01-01T00:09:00", + "exception_info": {"exception_type": "RuntimeError", "exception_message": "boom", + "exception_traceback": ""}, + "verifier_result": None, + })) + trials = runner._load_trials(jobs) + assert trials["t0"]["trial_name"] == "trial0" + assert (trials["t0"]["verifier_result"] or {}).get("rewards") == {"pass": 1.0} + + def test_latest_attempt_wins_when_both_clean(self, tmp_path): + runner = _runner() + jobs = tmp_path / "jobs" + run = jobs / "2026-01-01__00-00-00" + early = run / "a" + early.mkdir(parents=True) + (early / "result.json").write_text(json.dumps({ + "task_name": "t0", "trial_name": "early", "finished_at": "2026-01-01T00:01:00", + "verifier_result": {"rewards": {"pass": 0.0}}, + })) + late = run / "b" + late.mkdir(parents=True) + (late / "result.json").write_text(json.dumps({ + "task_name": "t0", "trial_name": "late", "finished_at": "2026-01-01T00:05:00", + "verifier_result": {"rewards": {"pass": 1.0}}, + })) + trials = runner._load_trials(jobs) + assert trials["t0"]["trial_name"] == "late" + + @pytest.mark.asyncio + async def test_resume_reruns_persisted_error_sample(self, tmp_path, monkeypatch): + monkeypatch.setenv("VERO_HOME_DIR", str(tmp_path / "vh")) + runner = _runner() + params = _params() + result_dir = tmp_path / "result" + # sample 0 previously errored (transient failure) + save_sample_result( + get_vero_home_dir() / "sessions", "s", params.result_id, sample_id=0, + result=SampleResult( + dataset_sample=DatasetSample(sample_id=0, split="test", dataset_id="ds"), + error="transient harbor failure", commit="c1", result_id=params.result_id, + ), + ) + # a good trial for t0 now exists + _write_trial(result_dir / "jobs", "trial0", "t0", {"pass": 1.0}) + monkeypatch.setattr(runner, "_task_names_for", lambda p: [(0, "t0")]) + runner._run_harbor = AsyncMock() + ws = MagicMock(project_path="/wt") + await runner.produce_sample_results(workspace=ws, params=params, result_dir=result_dir) + # error sample was treated as pending (re-run) and re-collated to a score + assert runner._run_harbor.await_args.args[2] == ["t0"] + results = load_all_sample_results(get_vero_home_dir() / "sessions", "s", params.result_id) + assert results[0].error is None + assert results[0].score == 1.0