diff --git a/EVALUATIONS.md b/EVALUATIONS.md index 5fbe291..3afc8c6 100644 --- a/EVALUATIONS.md +++ b/EVALUATIONS.md @@ -846,19 +846,19 @@ These four metrics also appear per-model (as `average_bleu_score`/`average_rouge ### CI gate (self-host) -A finalized run can pass/fail a CI job. `report.gate(...)` checks the run's average rating, prints per-check verdicts into the CI log, and returns a `GateResult` - the caller decides the exit code: +A finalized run can pass/fail a CI job. `run.gate(...)` (on the run context `.execute()` returns - not on the `Report` model from `get_report()`) checks the run's average rating, prints per-check verdicts into the CI log, and returns a `GateResult` - the caller decides the exit code: ```python import sys -report = ( +run = ( client.evaluations .run(dataset_id="evds_…", subject={"kind": "custom_agent", "framework": "raw_python"}) .execute(my_agent) # in CI, this is the PR's version of your agent .finalize() ) -gate = report.gate(fail_under=7, no_regression=True, caller="github-actions") +gate = run.gate(fail_under=7, no_regression=True, caller="github-actions") sys.exit(gate.exit_code) # 0 = merge, 1 = block ``` diff --git a/README.md b/README.md index abb38f0..7df75bb 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ client.evaluations.run( See [Prompt registry](EVALUATIONS.md#prompt-registry) in the full guide, or [self-host's docs](https://docs.agentx.so/improve/prompt-management) for the "Suggest improvement" dashboard flow (self-host only - no hosted-SaaS equivalent yet). -On self-host, a finalized run can also **gate a CI job**: `report.gate(fail_under=7, no_regression=True)` checks the run's average rating against an absolute floor and/or the dataset's previous run, prints per-check verdicts into the CI log, and returns an exit code - `sys.exit(gate.exit_code)` blocks the merge on regression. Recorded gates appear in the dashboard's CI Gates tab. See [self-host's CI docs](https://docs.agentx.so/integrations/self-host-ci) for the GitHub Actions recipe. +On self-host, a finalized run can also **gate a CI job**: `run.gate(fail_under=7, no_regression=True)` (on the run context `.execute()` returns) checks the run's average rating against an absolute floor and/or the dataset's previous run, prints per-check verdicts into the CI log, and returns an exit code - `sys.exit(gate.exit_code)` blocks the merge on regression. Recorded gates appear in the dashboard's CI Gates tab. See [self-host's CI docs](https://docs.agentx.so/integrations/self-host-ci) for the GitHub Actions recipe. See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide - dataset builder, framework adapters, similarity metrics, smoke testing, judge configuration, prompt registry, and the complete API reference. diff --git a/agentx/evaluations/client.py b/agentx/evaluations/client.py index 0aa993b..5dcef56 100644 --- a/agentx/evaluations/client.py +++ b/agentx/evaluations/client.py @@ -37,6 +37,9 @@ # wait out the whole job on one connection. Matches EvaluationRunContext.analyze()'s own # default timeout. _SELF_HOST_ANALYZE_TIMEOUT = 1800 +# Batch result submission scores each result synchronously inside the request (one judge call +# per result on the sync path) - a big batch on a slow judge legitimately takes minutes. +_SELF_HOST_SCORING_TIMEOUT = 900 class AgentXEvaluationsError(Exception): @@ -344,7 +347,15 @@ def append_results( "batchId": batch_id, "results": [_result_to_payload(r) for r in results], } - data = self._request("POST", f"/runs/{run_id}/results", json=payload) + # Scoring is synchronous inside this request (a judge call per result - the runner's + # spinner says "~60s+" for a reason), so the default 30s timeout + silent backoff loop + # re-POSTed the batch WHILE the first submission was still scoring, and the two raced + # into the engine's (run_id, idempotency_key) unique constraint. Long timeout, no + # transport retry - the runner's own logged retry-once is the retry layer here. + data = self._request( + "POST", f"/runs/{run_id}/results", json=payload, + timeout=_SELF_HOST_SCORING_TIMEOUT, retry=False, + ) return BatchAppendResponse(**data) def finalize_run(self, run_id: str) -> Dict[str, Any]: