Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions benchmark/core/model_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -931,7 +931,9 @@ def _retry_validation_prompt_suffix(output_contract: str) -> str:
" Before returning, check every leaf: full credit requires "
"deduction_trace=null; every non-full score requires a non-empty "
"deduction_trace whose points_deducted values sum exactly to "
"max_score minus score."
"max_score minus score. For an execution-contract rubric, use only "
"declared criterion IDs and copy their atomic point values exactly; "
"do not split or rescale points to force a total."
)
return suffix

Expand Down Expand Up @@ -966,7 +968,11 @@ def _execution_contract_prompt_note(rubric: dict[str, Any] | None) -> str:
"\nExecution-contract rule: score only the declared criteria. Ignore "
"unrelated extra content unless it directly contradicts a declared "
"criterion. For every deduction_trace, rubric_criterion must be the "
"exact applicable criterion ID from this rubric."
"exact applicable criterion ID from this rubric. Criterion points are "
"atomic: copy each selected criterion's declared points exactly; never "
"split, rescale, or invent a deduction amount just to make a trace add "
"up. Select a score only when max_score minus score equals the exact "
"sum of the withheld declared criteria."
)


Expand Down
20 changes: 20 additions & 0 deletions tests/benchmark/core/test_model_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -179,8 +179,28 @@ def test_deduction_trace_retry_feedback_repeats_exact_deduction_invariant(self):

self.assertIn("deduction_trace=null", suffix)
self.assertIn("sum exactly to max_score minus score", suffix)
self.assertIn("copy their atomic point values exactly", suffix)
self.assertNotIn("student_id", suffix)

def test_execution_contract_prompt_requires_atomic_trace_bookkeeping(self):
course = CourseSpec.from_dict(
json.loads((FIXTURES / "course_dsaa3073_hw1.json").read_text(encoding="utf-8"))
)

prompt = _compose_student_prompt(
"Return JSON.",
"S001",
course,
rubric={"rubric_format": "execution_contract_v1"},
inputs=[],
task="grade",
output_contract=GRADING_OUTPUT_CONTRACT_DEDUCTION_TRACE_V1,
)

self.assertIn("Criterion points are atomic", prompt)
self.assertIn("never split, rescale, or invent a deduction amount", prompt)
self.assertIn("exact sum of the withheld declared criteria", prompt)

def test_multimodal_prompt_repeats_page_locator_rule(self):
course = CourseSpec.from_dict(
json.loads((FIXTURES / "course_dsaa3073_hw1.json").read_text(encoding="utf-8"))
Expand Down
Loading