diff --git a/benchmark/core/model_runner.py b/benchmark/core/model_runner.py index 4b3bd7f..b458ab2 100644 --- a/benchmark/core/model_runner.py +++ b/benchmark/core/model_runner.py @@ -770,12 +770,13 @@ def _structured_output_contract( trace_instruction = "" if task == "grade" and output_contract == GRADING_OUTPUT_CONTRACT_DEDUCTION_TRACE_V1: trace_instruction = ( - " Every non-full-credit leaf must include deduction_trace with one or " - "more four-field entries; their points_deducted must sum exactly to " - "that leaf's max_score minus score. Omit deduction_trace for full " - "credit. Any row with flags or low confidence must include a concise " - "attention_note. Deduction traces are audit statements, not hidden " - "reasoning or chain-of-thought." + " Every leaf row must include deduction_trace and attention_note. " + "Use null for either field when it is not applicable. Every non-full-" + "credit leaf must instead provide deduction_trace with one or more " + "four-field entries; their points_deducted must sum exactly to that " + "leaf's max_score minus score. Any row with flags or low confidence " + "must instead provide a concise non-null attention_note. Deduction " + "traces are audit statements, not hidden reasoning or chain-of-thought." ) return ( "Return exactly one JSON object and no Markdown. Follow this structural " @@ -956,6 +957,9 @@ def _validate_grade_payload( scores = payload.get("scores") if not isinstance(scores, list): raise ValueError("scores must be a list") + _normalize_full_credit_empty_deduction_traces( + scores, course, output_contract=output_contract + ) records = [] required_row_fields = { "question_id", @@ -1096,6 +1100,37 @@ def _validate_grade_payload( raise ValueError(f"invalid confidence: {row['confidence']}") +def _normalize_full_credit_empty_deduction_traces( + scores: list[Any], + course: CourseSpec, + *, + output_contract: str, +) -> None: + """Normalize a provider's empty-list spelling of a nullable full-credit trace. + + The strict v5.3 schema represents an inapplicable trace as ``null``. Some + JSON-object providers still emit ``[]`` for a full-credit leaf. Converting + that one syntactic variant before validation preserves the score and cannot + hide a missing deduction: non-full leaves keep ``[]`` and are rejected. + """ + + if output_contract != GRADING_OUTPUT_CONTRACT_DEDUCTION_TRACE_V1: + return + question_map = course.question_map + for row in scores: + if not isinstance(row, dict) or row.get("deduction_trace") != []: + continue + question = question_map.get(row.get("question_id")) + score = row.get("score") + if ( + question is not None + and isinstance(score, (int, float)) + and not isinstance(score, bool) + and abs(float(score) - float(question.max_score)) <= 1e-9 + ): + row["deduction_trace"] = None + + def _validate_transcript_payload( payload: dict[str, Any], student_id: str, course: CourseSpec ) -> None: diff --git a/benchmark/core/packets.py b/benchmark/core/packets.py index 49a2f2e..bece09d 100644 --- a/benchmark/core/packets.py +++ b/benchmark/core/packets.py @@ -368,11 +368,19 @@ def grading_output_schema( "items": {"type": "string"}, }, } + score_required = [ + "question_id", + "extracted_evidence", + "score", + "evidence", + "confidence", + "flags", + ] if grading_output_contract == GRADING_OUTPUT_CONTRACT_DEDUCTION_TRACE_V1: score_properties.update( { "deduction_trace": { - "type": "array", + "type": ["array", "null"], "minItems": 1, "items": { "type": "object", @@ -407,12 +415,17 @@ def grading_output_schema( }, }, "attention_note": { - "type": "string", + "type": ["string", "null"], "minLength": 1, "maxLength": 500, }, } ) + # Codex strict structured output requires every declared property to + # appear in `required`. Nullable values retain this contract's + # semantic optionality: a validator still requires a non-empty trace + # for non-full credit and a non-empty note for flags/low confidence. + score_required.extend(["deduction_trace", "attention_note"]) return { "type": "object", "properties": { @@ -427,14 +440,7 @@ def grading_output_schema( "items": { "type": "object", "properties": score_properties, - "required": [ - "question_id", - "extracted_evidence", - "score", - "evidence", - "confidence", - "flags", - ], + "required": score_required, "additionalProperties": False, }, }, diff --git a/tests/benchmark/core/test_deduction_trace_contract.py b/tests/benchmark/core/test_deduction_trace_contract.py index a14f592..7868ab3 100644 --- a/tests/benchmark/core/test_deduction_trace_contract.py +++ b/tests/benchmark/core/test_deduction_trace_contract.py @@ -114,6 +114,26 @@ def test_full_credit_leaf_may_omit_trace_and_leaf_deductions_are_independent(sel self.assertNotIn("deduction_trace", payload["scores"][0]) self.assertEqual(payload["total"], 19) + def test_full_credit_leaf_accepts_nullable_strict_schema_fields(self): + payload = self._payload() + payload["scores"][0].update( + {"deduction_trace": None, "attention_note": None} + ) + + self._validate(payload) + + def test_full_credit_empty_trace_is_normalized_but_nonfull_trace_is_not(self): + payload = self._payload() + payload["scores"][0]["deduction_trace"] = [] + + self._validate(payload) + self.assertIsNone(payload["scores"][0]["deduction_trace"]) + + payload = self._payload() + payload["scores"][1]["deduction_trace"] = [] + with self.assertRaisesRegex(ValueError, "deduction_trace"): + self._validate(payload) + def test_rejects_nonfull_leaf_without_trace(self): payload = self._payload() del payload["scores"][1]["deduction_trace"] @@ -170,6 +190,21 @@ def test_schema_and_packet_bind_deduction_trace_contract(self): "points_deducted", ], ) + self.assertEqual( + schema["properties"]["scores"]["items"]["required"], + [ + "question_id", + "extracted_evidence", + "score", + "evidence", + "confidence", + "flags", + "deduction_trace", + "attention_note", + ], + ) + self.assertEqual(score_properties["deduction_trace"]["type"], ["array", "null"]) + self.assertEqual(score_properties["attention_note"]["type"], ["string", "null"]) with tempfile.TemporaryDirectory() as tmp: root = Path(tmp)