diff --git a/src/reviewer/cli.py b/src/reviewer/cli.py index c5828d7..5399490 100644 --- a/src/reviewer/cli.py +++ b/src/reviewer/cli.py @@ -194,6 +194,7 @@ def _build_paper_json( "explanation": c.explanation, "comment_type": c.comment_type, "paragraph_index": c.paragraph_index, + "severity": c.severity, }) model_short = _model_short_name(result.model) if result.model else "" diff --git a/src/reviewer/models.py b/src/reviewer/models.py index 671bbc0..b00e9bb 100644 --- a/src/reviewer/models.py +++ b/src/reviewer/models.py @@ -2,6 +2,8 @@ from dataclasses import dataclass, field +SEVERITY_VALUES = ("minor", "moderate", "major") + @dataclass class Comment: @@ -11,6 +13,7 @@ class Comment: explanation: str # reviewer's explanation comment_type: str # "technical" or "logical" paragraph_index: int | None = None # 0-based index in split paragraphs + severity: str | None = None # one of SEVERITY_VALUES, or None when unknown def to_dict(self) -> dict: d = { @@ -21,6 +24,8 @@ def to_dict(self) -> dict: } if self.paragraph_index is not None: d["paragraph_index"] = self.paragraph_index + if self.severity: + d["severity"] = self.severity return d diff --git a/src/reviewer/prompts.py b/src/reviewer/prompts.py index 6ca28f3..34b7968 100644 --- a/src/reviewer/prompts.py +++ b/src/reviewer/prompts.py @@ -41,12 +41,19 @@ DO_NOT_FLAG_CHUNKED = DO_NOT_FLAG_BASE.rstrip() + """ - Incomplete text at passage boundaries""" +SEVERITY_RUBRIC = """\ +For each issue, assign a "severity" tier: +- "major": undermines a key claim, methodology, or comparison; affects conclusions +- "moderate": real error or gap, but localized and fixable +- "minor": framing concern, mild overclaim, or ambiguity resolvable from context""" + JSON_ARRAY_OUTPUT = """\ Return ONLY a JSON array (can be []). Each item: - "title": concise title of the issue - "quote": the exact verbatim text (preserving LaTeX) - "explanation": deep reasoning — what you initially thought, whether context resolves it, and what specifically remains problematic - "type": "technical" or "logical" +- "severity": "minor", "moderate", or "major" """ # ── Deep-check prompt (used by local and progressive methods) ─────────────── @@ -73,6 +80,8 @@ {DO_NOT_FLAG_CHUNKED} +{SEVERITY_RUBRIC} + {JSON_ARRAY_OUTPUT}""" @@ -98,6 +107,8 @@ {DO_NOT_FLAG_BASE} +{SEVERITY_RUBRIC} + Return a JSON object with this structure: {{{{ "overall_feedback": "one paragraph high-level assessment of the paper's quality and main issues", @@ -106,7 +117,8 @@ "title": "concise title of the issue", "quote": "exact verbatim text from the paper (preserving LaTeX)", "explanation": "precise explanation of what is wrong and why", - "type": "technical" or "logical" + "type": "technical" or "logical", + "severity": "minor" or "moderate" or "major" }}}} ] }}}} @@ -133,6 +145,8 @@ {DO_NOT_FLAG_CHUNKED} +{SEVERITY_RUBRIC} + Return a JSON object with this structure: {{{{ "overall_feedback": "brief assessment of this section", @@ -141,7 +155,8 @@ "title": "concise title of the issue", "quote": "exact verbatim text from the paper (preserving LaTeX)", "explanation": "precise explanation of what is wrong and why", - "type": "technical" or "logical" + "type": "technical" or "logical", + "severity": "minor" or "moderate" or "major" }}}} ] }}}} diff --git a/src/reviewer/utils.py b/src/reviewer/utils.py index ec62cce..347e926 100644 --- a/src/reviewer/utils.py +++ b/src/reviewer/utils.py @@ -6,7 +6,7 @@ import tiktoken -from .models import Comment +from .models import Comment, SEVERITY_VALUES def _get_encoding(): @@ -205,12 +205,18 @@ def parse_comments_from_list(items: list[dict]) -> list[Comment]: paragraph_index = item.get("paragraph_index", None) if paragraph_index is not None: paragraph_index = int(paragraph_index) + severity = item.get("severity") + if isinstance(severity, str): + severity = severity.strip().lower() + if severity not in SEVERITY_VALUES: + severity = "moderate" comments.append(Comment( title=title, quote=quote, explanation=explanation, comment_type=comment_type, paragraph_index=paragraph_index, + severity=severity, )) return comments