diff --git a/benchmark/core/model_runner.py b/benchmark/core/model_runner.py index 1bf54a9..96f8d30 100644 --- a/benchmark/core/model_runner.py +++ b/benchmark/core/model_runner.py @@ -931,7 +931,9 @@ def _retry_validation_prompt_suffix(output_contract: str) -> str: " Before returning, check every leaf: full credit requires " "deduction_trace=null; every non-full score requires a non-empty " "deduction_trace whose points_deducted values sum exactly to " - "max_score minus score." + "max_score minus score. For an execution-contract rubric, use only " + "declared criterion IDs and copy their atomic point values exactly; " + "do not split or rescale points to force a total." ) return suffix @@ -966,7 +968,11 @@ def _execution_contract_prompt_note(rubric: dict[str, Any] | None) -> str: "\nExecution-contract rule: score only the declared criteria. Ignore " "unrelated extra content unless it directly contradicts a declared " "criterion. For every deduction_trace, rubric_criterion must be the " - "exact applicable criterion ID from this rubric." + "exact applicable criterion ID from this rubric. Criterion points are " + "atomic: copy each selected criterion's declared points exactly; never " + "split, rescale, or invent a deduction amount just to make a trace add " + "up. Select a score only when max_score minus score equals the exact " + "sum of the withheld declared criteria." ) diff --git a/tests/benchmark/core/test_model_runner.py b/tests/benchmark/core/test_model_runner.py index d19bd82..ea0f5ac 100644 --- a/tests/benchmark/core/test_model_runner.py +++ b/tests/benchmark/core/test_model_runner.py @@ -179,8 +179,28 @@ def test_deduction_trace_retry_feedback_repeats_exact_deduction_invariant(self): self.assertIn("deduction_trace=null", suffix) self.assertIn("sum exactly to max_score minus score", suffix) + self.assertIn("copy their atomic point values exactly", suffix) self.assertNotIn("student_id", suffix) + def test_execution_contract_prompt_requires_atomic_trace_bookkeeping(self): + course = CourseSpec.from_dict( + json.loads((FIXTURES / "course_dsaa3073_hw1.json").read_text(encoding="utf-8")) + ) + + prompt = _compose_student_prompt( + "Return JSON.", + "S001", + course, + rubric={"rubric_format": "execution_contract_v1"}, + inputs=[], + task="grade", + output_contract=GRADING_OUTPUT_CONTRACT_DEDUCTION_TRACE_V1, + ) + + self.assertIn("Criterion points are atomic", prompt) + self.assertIn("never split, rescale, or invent a deduction amount", prompt) + self.assertIn("exact sum of the withheld declared criteria", prompt) + def test_multimodal_prompt_repeats_page_locator_rule(self): course = CourseSpec.from_dict( json.loads((FIXTURES / "course_dsaa3073_hw1.json").read_text(encoding="utf-8"))