From 43a3bd2b4707844c3e6140156c33f8f6dd61080e Mon Sep 17 00:00:00 2001 From: ishtar Date: Wed, 14 Oct 2026 01:29:22 +0900 Subject: [PATCH] Recognise single-quoted __main__ guards in LiveCodeBench stdin tests run_test_std only treated a completion as a script when it contained the exact string __name__ == "__main__". A program using the equally valid single-quoted form, if __name__ == '__main__':, was exec'd with __name__ bound to builtins, so its entry point never ran, it printed nothing and every stdin test failed even for a correct solution. Match the guard with either quote style and either operand order. Completions without a guard, and double-quoted guards, run exactly as before. --- llm_evaluation/livecodebench_util.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/llm_evaluation/livecodebench_util.py b/llm_evaluation/livecodebench_util.py index 304db062..f2bc59b9 100644 --- a/llm_evaluation/livecodebench_util.py +++ b/llm_evaluation/livecodebench_util.py @@ -302,13 +302,20 @@ def run_test_func(completion, is_extracted, test_input, test_output): return True, result_output +# `if __name__ == "__main__":` and `if __name__ == '__main__':` are the same +# guard; either quote style (and either operand order) must run the program. +MAIN_GUARD = re.compile( + r"""__name__\s*==\s*(['"])__main__\1|(['"])__main__\2\s*==\s*__name__""" +) + + def run_test_std(completion, test_input, test_output): sys.stdin = StringIO(test_input) output = StringIO() sys.stdout = output - if '__name__ == "__main__"' in completion: + if MAIN_GUARD.search(completion): # Simulate that the code is being run as the main script completion = '__name__ = "__main__"\n' + completion