diff --git a/services/verifier/app/engine/matcher.py b/services/verifier/app/engine/matcher.py index 1791db8..0301abf 100644 --- a/services/verifier/app/engine/matcher.py +++ b/services/verifier/app/engine/matcher.py @@ -5,11 +5,20 @@ NUMERIC_MATCH_TOL = 1e-8 def expressions_match(left, right) -> bool: - reduced = simplify(left - right) + # SymPy can raise internally on exotic inputs (e.g. unevaluated integrals + # hitting bugs in risch); never let that escape as a 500 — report no + # match and let the numeric fallback have a try. + try: + reduced = simplify(left - right) + except Exception: + return False if reduced == 0: return True - equals = reduced.equals(0) + try: + equals = reduced.equals(0) + except Exception: + return False return equals is True def numeric_match(left, right, tol: float = NUMERIC_MATCH_TOL) -> bool: diff --git a/services/verifier/app/engine/parser.py b/services/verifier/app/engine/parser.py index 6ddd44b..8211ec5 100644 --- a/services/verifier/app/engine/parser.py +++ b/services/verifier/app/engine/parser.py @@ -1,5 +1,5 @@ import re -from sympy import Symbol, cos, exp, log, oo, pi, sin, sqrt, tan +from sympy import E, Expr, Symbol, cos, cosh, exp, log, oo, pi, sin, sinh, sqrt, tan, tanh from sympy.parsing.sympy_parser import ( convert_xor, implicit_multiplication_application, @@ -27,16 +27,20 @@ "sin": sin, "cos": cos, "tan": tan, + "sinh": sinh, + "cosh": cosh, + "tanh": tanh, "log": log, "ln": log, "sqrt": sqrt, "exp": exp, + "e": E, "pi": pi, "inf": oo, "infinity": oo, "oo": oo, } -FUNCTION_NAMES = ("sin", "cos", "tan", "log", "ln", "sqrt", "exp") +FUNCTION_NAMES = ("sinh", "cosh", "tanh", "sin", "cos", "tan", "log", "ln", "sqrt", "exp") def parse_math_expression(value: str): if not value.strip(): @@ -45,7 +49,7 @@ def parse_math_expression(value: str): text = clean_latex_expression(value) try: - return parse_expr( + result = parse_expr( text, local_dict=LOCAL_SYMBOLS, transformations=TRANSFORMATIONS, @@ -54,6 +58,11 @@ def parse_math_expression(value: str): except Exception: return None + # Guard against non-expression parses (tuples, relationals, ...): downstream + # code calls simplify()/diff() on the result, which raises TypeError on + # those and would turn into a 500 instead of unverifiable/mismatch. + return result if isinstance(result, Expr) else None + def clean_latex_expression(value: str) -> str: text = english_math_words(value) text = text.strip().strip("$") @@ -61,7 +70,7 @@ def clean_latex_expression(value: str) -> str: text = text.replace("\\,", "").replace("\\;", "") text = text.replace("\\mathrm{d}x", "dx").replace("\\mathrm{d}", "d") text = text.replace("\\cdot", "*").replace("\\times", "*") - text = text.replace("\\pi", "pi") + text = text.replace("\\pi", "pi").replace("\\infty", "oo") text = replace_command_groups(text, "\\frac", lambda a, b: f"(({a})/({b}))") text = replace_command_groups(text, "\\sqrt", lambda a, _b=None: f"sqrt({a})") @@ -70,6 +79,9 @@ def clean_latex_expression(value: str) -> str: "\\sin": "sin", "\\cos": "cos", "\\tan": "tan", + "\\sinh": "sinh", + "\\cosh": "cosh", + "\\tanh": "tanh", "\\ln": "log", "\\log": "log", "\\exp": "exp", @@ -88,6 +100,9 @@ def clean_latex_expression(value: str) -> str: return text def normalize_function_calls(text: str) -> str: + # Hyperbolic names are listed first in FUNCTION_NAMES, and the (?!h\b) + # guard keeps "sin"/"cos"/"tan" from matching the prefix of + # "sinh"/"cosh"/"tanh" (e.g. "sinh x" must not become "sin(h) x"). for function_name in FUNCTION_NAMES: text = re.sub( rf"\b{function_name}\s+([A-Za-z](?:\s*\^\s*\(?[-A-Za-z0-9+*/]+\)?)?)", @@ -95,7 +110,7 @@ def normalize_function_calls(text: str) -> str: text, ) text = re.sub( - rf"\b{function_name}([A-Za-z])\b", + rf"\b{function_name}(?!h\b)([A-Za-z])\b", rf"{function_name}(\1)", text, ) diff --git a/services/verifier/app/rules/definite_integral.py b/services/verifier/app/rules/definite_integral.py index 2951803..cfa1b3a 100644 --- a/services/verifier/app/rules/definite_integral.py +++ b/services/verifier/app/rules/definite_integral.py @@ -60,15 +60,19 @@ def extract_definite_integral_info(payload: VerifyRequest) -> dict | None: cleaned = cleaned.replace("\\mathrm{d}x", "dx") # LaTeX: \int_{a}^{b} expr dx or \int_a^b expr dx + # Each bound is either a braced group (any non-brace content, so + # "\infty" is captured whole) or a bare non-space token. m = re.search( - r"\\int\s*_\s*\{?\s*(.+?)\s*\}?\s*\^\s*\{?\s*(.+?)\s*\}?\s*(.+?)(?:d\s*x|dx)(?:\b|$)", + r"\\int\s*_(?:\{\s*([^{}]+?)\s*\}|([^{}\s]+))" + r"\s*\^(?:\{\s*([^{}]+?)\s*\}|([^{}\s]+))" + r"\s*(.+?)(?:d\s*x|dx)(?:\b|$)", cleaned, ) if m: return { - "lower": m.group(1).strip(), - "upper": m.group(2).strip(), - "integrand": m.group(3).strip(), + "lower": (m.group(1) or m.group(2)).strip(), + "upper": (m.group(3) or m.group(4)).strip(), + "integrand": m.group(5).strip(), } # English: "integrate expr from a to b" diff --git a/services/verifier/app/rules/limit.py b/services/verifier/app/rules/limit.py index 90d901d..4a7b810 100644 --- a/services/verifier/app/rules/limit.py +++ b/services/verifier/app/rules/limit.py @@ -1,6 +1,6 @@ import re -from sympy import limit -from app.engine.parser import X, LOCAL_SYMBOLS, parse_math_expression +from sympy import Symbol, limit +from app.engine.parser import LOCAL_SYMBOLS, parse_math_expression from app.engine.matcher import expressions_match_or_numeric, verify_equivalent_step from app.engine.responses import VerifyRequest, VerifyResponse, unsupported_response @@ -22,7 +22,11 @@ def verify_limit(payload: VerifyRequest) -> VerifyResponse: "The limit expression or final answer could not be parsed.", ) - var = LOCAL_SYMBOLS.get(target.get("var", "x"), X) + var_name = target.get("var", "x") + # Fall back to a fresh Symbol for variables outside the canonical set + # (e.g. "n"); defaulting to X here silently computed the limit in the + # wrong variable and mismatched correct answers. + var = LOCAL_SYMBOLS.get(var_name, Symbol(var_name)) try: expected = limit(expr, var, point) @@ -54,10 +58,18 @@ def extract_limit_info(payload: VerifyRequest) -> dict | None: source = payload.problem_text # LaTeX: \lim_{x \to a} expr or \lim_{x\to a} expr + # Braced subscript first: the point is a non-brace run so "\to\infty" + # captures "\infty" whole (a lazy (.+?) stopped after the backslash). m = re.search( - r"\\lim\s*_\s*\{?\s*([a-z])\s*(?:\\to|\\rightarrow|->)\s*(.+?)\s*\}?\s*(.+)", + r"\\lim\s*_\{\s*([a-z])\s*(?:\\to|\\rightarrow|->)\s*([^{}]+?)\s*\}\s*(.+)", source, ) + if not m: + # Unbraced subscript: \lim_x \to a expr + m = re.search( + r"\\lim\s*_([a-z])\s*(?:\\to|\\rightarrow|->)\s*(\S+)\s+(.+)", + source, + ) if m: return {"var": m.group(1), "point": m.group(2).strip(), "expr": m.group(3).strip()} diff --git a/services/verifier/app/rules/linear_system.py b/services/verifier/app/rules/linear_system.py index cf62a03..4ad735a 100644 --- a/services/verifier/app/rules/linear_system.py +++ b/services/verifier/app/rules/linear_system.py @@ -145,8 +145,9 @@ def check_value_in_answer(answer_text: str, var_name: str, expected_value) -> bo if answer_expr is not None and expressions_match_or_numeric(answer_expr, expected_value): return True - # Look for patterns like "x = 3" or "x=3" in the text. - pattern = rf"{re.escape(var_name)}\s*=\s*([^,;\s]+)" + # Look for patterns like "x = 3" or "x=3" in the text. The leading \b + # keeps "x" from matching inside longer words (e.g. "max = 5"). + pattern = rf"\b{re.escape(var_name)}\s*=\s*([^,;\s]+)" for m in re.finditer(pattern, answer_text): val = parse_math_expression(m.group(1)) if val is not None and expressions_match_or_numeric(val, expected_value): diff --git a/services/verifier/tests/test_verifier.py b/services/verifier/tests/test_verifier.py index a1a4725..6da1751 100644 --- a/services/verifier/tests/test_verifier.py +++ b/services/verifier/tests/test_verifier.py @@ -188,6 +188,156 @@ def test_linear_system_mismatches(self): self.assertEqual(response.verification_status, "mismatch") self.assertFalse(response.verified) + def test_hyperbolic_derivative_verifies(self): + # Regression: "sinh x" was mangled to "sin(h) x" by function-call + # normalization, so a correct "cosh x" answer mismatched. + response = verify_payload( + VerifyRequest( + problem_text="Differentiate sinh x with respect to x.", + subject="math", + final_answer="cosh x", + steps=[SolveStep(step_num=1, latex="\\cosh x")], + ), + ) + + self.assertEqual(response.verification_status, "verified") + self.assertTrue(response.verified) + self.assertEqual(response.computed_value, "cosh(x)") + + def test_hyperbolic_derivative_mismatches(self): + response = verify_payload( + VerifyRequest( + problem_text="Differentiate cosh x with respect to x.", + subject="math", + final_answer="cosh x", + steps=[SolveStep(step_num=1, latex="\\cosh x")], + ), + ) + + self.assertEqual(response.verification_status, "mismatch") + self.assertFalse(response.verified) + self.assertEqual(response.computed_value, "sinh(x)") + + def test_limit_to_infinity_verifies(self): + # Regression: the point group captured just "\" for "\to\infty", + # leaving the problem unverifiable instead of verified. + response = verify_payload( + VerifyRequest( + problem_text="\\lim_{x\\to\\infty} \\frac{1}{x}", + subject="math", + final_answer="0", + steps=[SolveStep(step_num=1, latex="0")], + ), + ) + + self.assertEqual(response.verification_status, "verified") + self.assertTrue(response.verified) + self.assertEqual(response.computed_value, "0") + + def test_limit_other_variable_verifies(self): + # Regression: variables outside x/y/z/t fell back to X, so the limit + # was computed in the wrong variable and correct answers mismatched. + response = verify_payload( + VerifyRequest( + problem_text="\\lim_{n \\to 2} n^2", + subject="math", + final_answer="4", + steps=[SolveStep(step_num=1, latex="4")], + ), + ) + + self.assertEqual(response.verification_status, "verified") + self.assertTrue(response.verified) + self.assertEqual(response.computed_value, "4") + + def test_tuple_final_answer_does_not_crash(self): + # Regression: a tuple-shaped parse result (e.g. "2, 3") raised + # TypeError inside simplify() and would surface as a 500. + response = verify_payload( + VerifyRequest( + problem_text="Solve the system of equations: x + y = 5 and 2x - y = 1", + subject="math", + final_answer="2, 3", + steps=[SolveStep(step_num=1, latex="x=2, y=3")], + ), + ) + + self.assertEqual(response.verification_status, "mismatch") + self.assertFalse(response.verified) + + def test_tuple_step_does_not_crash(self): + response = verify_payload( + VerifyRequest( + problem_text="Integrate x^2 dx.", + subject="math", + final_answer="x^3/3+C", + steps=[SolveStep(step_num=1, latex="x^2, x^3")], + ), + ) + + self.assertEqual(response.verification_status, "verified") + self.assertTrue(response.verified) + self.assertEqual(response.steps[0].verification_status, "unverifiable") + + def test_linear_system_ignores_longer_words(self): + # Regression: "max = 5" matched the "x = " pattern inside "max", + # falsely verifying an answer that never states x = 5. + response = verify_payload( + VerifyRequest( + problem_text="Solve the system of equations: x + y = 6 and x - y = 4", + subject="math", + final_answer="the max = 5, y = 1", + steps=[SolveStep(step_num=1, latex="x=5, y=1")], + ), + ) + + self.assertEqual(response.verification_status, "mismatch") + self.assertFalse(response.verified) + + def test_definite_integral_unbraced_bounds_verify(self): + # Regression: bound extraction must handle \int_0^1 as well as + # \int_{0}^{1}. + response = verify_payload( + VerifyRequest( + problem_text="\\int_0^1 x^2 dx", + subject="math", + final_answer="1/3", + steps=[SolveStep(step_num=1, latex="1/3")], + ), + ) + + self.assertEqual(response.verification_status, "verified") + self.assertTrue(response.verified) + + def test_definite_integral_infinite_bound_verifies(self): + # Regression: "\infty" bounds were split into garbage by a lazy + # capture group, and bare "e" parsed as a symbol instead of Euler. + response = verify_payload( + VerifyRequest( + problem_text="\\int_{0}^{\\infty} e^{-x} dx", + subject="math", + final_answer="1", + steps=[SolveStep(step_num=1, latex="1")], + ), + ) + + self.assertEqual(response.verification_status, "verified") + self.assertTrue(response.verified) + self.assertEqual(response.computed_value, "1") + + def test_euler_number_parses(self): + response = verify_payload( + VerifyRequest( + problem_text="Differentiate e^x with respect to x.", + subject="math", + final_answer="e^x", + steps=[SolveStep(step_num=1, latex="e^x")], + ), + ) + + self.assertEqual(response.verification_status, "verified") + self.assertTrue(response.verified) + if __name__ == "__main__": unittest.main()