From 59142675d7c05a9bff59208cf190fbba5699c137 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 4 Nov 2026 13:40:12 -0800 Subject: [PATCH 1/7] fix(harness): command arms send the task instruction as written, no dokimi wrapper The 'Solve this coding task...' line and '# Task: ' header that _task_prompt put around {prompt} are text a user's agent never sees. Pass the bare instruction, as the real harnesses do. Signed-off-by: Nikolay Petrov --- .../20261104_120000_napetrov_raw-task-prompt.md | 8 ++++++++ dokimi/dokimi/harnesses/command.py | 12 +++++++----- dokimi/tests/test_task_harnesses.py | 10 ++++++++++ 3 files changed, 25 insertions(+), 5 deletions(-) create mode 100644 dokimi/changelog.d/20261104_120000_napetrov_raw-task-prompt.md diff --git a/dokimi/changelog.d/20261104_120000_napetrov_raw-task-prompt.md b/dokimi/changelog.d/20261104_120000_napetrov_raw-task-prompt.md new file mode 100644 index 000000000..36f17e4cb --- /dev/null +++ b/dokimi/changelog.d/20261104_120000_napetrov_raw-task-prompt.md @@ -0,0 +1,8 @@ + + +### Changed + +- **Command and Harbor-style harnesses send the task instruction as written** — the `Solve this coding task. Work only inside the task environment…` and `# Task: ` lines that dokimi put around `{prompt}` are gone, as a user's agent never sees them. `{prompt}`, `{prompt_quoted}` and `{prompt_escaped}` now expand to the bare `instruction.md`. Command-arm runs measured before this change are not comparable in their prompt text. diff --git a/dokimi/dokimi/harnesses/command.py b/dokimi/dokimi/harnesses/command.py index 7ab2ffa01..4d038d15f 100644 --- a/dokimi/dokimi/harnesses/command.py +++ b/dokimi/dokimi/harnesses/command.py @@ -386,11 +386,13 @@ def _render_template(template: str, context: dict[str, Any]) -> str: def _task_prompt(task: TaskSpec) -> str: - return ( - "Solve this coding task. Work only inside the task environment. " - "When finished, leave the files ready for the verifier.\n\n" - f"# Task: {task.name}\n\n{task.instruction}" - ) + """The task instruction as the user wrote it: no wrapper, the way a real harness sends it. + + A wrapper ("Solve this coding task…", "# Task: ") is text dokimi adds that a + user's agent never sees, so it can move the score of a command arm for no reason + the task measures. + """ + return task.instruction # Windows reports a crash or Ctrl-C as an NTSTATUS error code (0xC0000005, 0xC000013A, 0xC0000409). diff --git a/dokimi/tests/test_task_harnesses.py b/dokimi/tests/test_task_harnesses.py index d474b457b..bbed9d643 100644 --- a/dokimi/tests/test_task_harnesses.py +++ b/dokimi/tests/test_task_harnesses.py @@ -105,6 +105,16 @@ def test_command_harness_id_includes_template_to_avoid_collisions(): assert second.harness_id == "command:cat {task_path}" +def test_command_prompt_is_the_raw_instruction(tmp_path): + # No dokimi wrapper ("Solve this coding task…", "# Task: ") around the instruction. + task_path = _fixture_task(tmp_path) + (task_path / "instruction.md").write_text("Fix the off-by-one in sum().", encoding="utf-8") + task = load_task(task_path) + harness = CommandHarness("fake", "echo {prompt_quoted}") + argv = harness.build_command(task, model=None, output_dir=tmp_path / "out") + assert argv == ["echo", "Fix the off-by-one in sum()."] + + def test_command_template_allows_code_braces_in_template_and_prompt(tmp_path): task_path = _fixture_task(tmp_path) (task_path / "instruction.md").write_text( From 87de7dafb33ac40784c2fccd7bbe00b1c84acd21 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 4 Nov 2026 14:17:54 -0800 Subject: [PATCH 2/7] test: move the raw-prompt check to its own file (file-length ratchet) test_task_harnesses.py went past its 1000-line cap; the check is a test of command.py, so it gets its own file. Refs #401 Signed-off-by: Nikolay Petrov --- dokimi/tests/test_command_prompt.py | 16 ++++++++++++++++ dokimi/tests/test_task_harnesses.py | 10 ---------- 2 files changed, 16 insertions(+), 10 deletions(-) create mode 100644 dokimi/tests/test_command_prompt.py diff --git a/dokimi/tests/test_command_prompt.py b/dokimi/tests/test_command_prompt.py new file mode 100644 index 000000000..919fc6a9c --- /dev/null +++ b/dokimi/tests/test_command_prompt.py @@ -0,0 +1,16 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""A command arm's prompt is the task instruction as written, with no dokimi wrapper.""" + +from __future__ import annotations + +from dokimi.harnesses.command import CommandHarness +from dokimi.harnesses.tasks import TaskSpec + + +def test_command_prompt_is_the_raw_instruction(tmp_path): + task = TaskSpec(name="toy", path=tmp_path / "toy", instruction="Fix the off-by-one in sum().") + harness = CommandHarness("fake", "echo {prompt_quoted}") + argv = harness.build_command(task, model=None, output_dir=tmp_path / "out") + assert argv == ["echo", "Fix the off-by-one in sum()."] diff --git a/dokimi/tests/test_task_harnesses.py b/dokimi/tests/test_task_harnesses.py index bbed9d643..d474b457b 100644 --- a/dokimi/tests/test_task_harnesses.py +++ b/dokimi/tests/test_task_harnesses.py @@ -105,16 +105,6 @@ def test_command_harness_id_includes_template_to_avoid_collisions(): assert second.harness_id == "command:cat {task_path}" -def test_command_prompt_is_the_raw_instruction(tmp_path): - # No dokimi wrapper ("Solve this coding task…", "# Task: ") around the instruction. - task_path = _fixture_task(tmp_path) - (task_path / "instruction.md").write_text("Fix the off-by-one in sum().", encoding="utf-8") - task = load_task(task_path) - harness = CommandHarness("fake", "echo {prompt_quoted}") - argv = harness.build_command(task, model=None, output_dir=tmp_path / "out") - assert argv == ["echo", "Fix the off-by-one in sum()."] - - def test_command_template_allows_code_braces_in_template_and_prompt(tmp_path): task_path = _fixture_task(tmp_path) (task_path / "instruction.md").write_text( From dba386cefccc96fedf668a88ce58baa4ac5be735 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Thu, 5 Nov 2026 13:25:29 +0000 Subject: [PATCH 3/7] Scope the raw-prompt changelog entry to command arms Signed-off-by: Anton Volkov --- dokimi/changelog.d/20261104_120000_napetrov_raw-task-prompt.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/changelog.d/20261104_120000_napetrov_raw-task-prompt.md b/dokimi/changelog.d/20261104_120000_napetrov_raw-task-prompt.md index 36f17e4cb..993cc1a71 100644 --- a/dokimi/changelog.d/20261104_120000_napetrov_raw-task-prompt.md +++ b/dokimi/changelog.d/20261104_120000_napetrov_raw-task-prompt.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Changed -- **Command and Harbor-style harnesses send the task instruction as written** — the `Solve this coding task. Work only inside the task environment…` and `# Task: ` lines that dokimi put around `{prompt}` are gone, as a user's agent never sees them. `{prompt}`, `{prompt_quoted}` and `{prompt_escaped}` now expand to the bare `instruction.md`. Command-arm runs measured before this change are not comparable in their prompt text. +- **Command arms (`command:` and `--command-template`) send the task instruction as written** — the `Solve this coding task. Work only inside the task environment…` and `# Task: ` lines that dokimi put around `{prompt}` are gone, as a user's agent never sees them. `{prompt}` and `{prompt_quoted}` now expand to the bare `instruction.md`, and `{prompt_escaped}` to the same text with `{` and `}` doubled. Command-arm runs measured before this change are not comparable in their prompt text. From cdcb9c8e88a93f72016410def066c83bd477467f Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Thu, 5 Nov 2026 13:25:48 +0000 Subject: [PATCH 4/7] Test all three prompt placeholders on a multi-line instruction Signed-off-by: Anton Volkov --- dokimi/tests/test_command_prompt.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/dokimi/tests/test_command_prompt.py b/dokimi/tests/test_command_prompt.py index 919fc6a9c..fb26779c1 100644 --- a/dokimi/tests/test_command_prompt.py +++ b/dokimi/tests/test_command_prompt.py @@ -14,3 +14,16 @@ def test_command_prompt_is_the_raw_instruction(tmp_path): harness = CommandHarness("fake", "echo {prompt_quoted}") argv = harness.build_command(task, model=None, output_dir=tmp_path / "out") assert argv == ["echo", "Fix the off-by-one in sum()."] + + +def test_every_prompt_placeholder_keeps_a_multiline_instruction_as_written(tmp_path): + instruction = "Fix {x}.\nSecond line.\n" + task = TaskSpec(name="toy", path=tmp_path / "toy", instruction=instruction) + + def argv(template): + harness = CommandHarness("fake", template) + return harness.build_command(task, model=None, output_dir=tmp_path / "out") + + assert argv("echo '{prompt}'") == ["echo", instruction] + assert argv("echo {prompt_quoted}") == ["echo", instruction] + assert argv("echo '{prompt_escaped}'") == ["echo", "Fix {{x}}.\nSecond line.\n"] From bdf2df33e062e01f975112b196a86d25395bfa08 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 5 Nov 2026 07:40:52 -0800 Subject: [PATCH 5/7] Address review: changelog wording, already fixed in dba386c Thread: PRRT_kwDOVBg7g860FaDg: already fixed in dba386c Signed-off-by: Nikolay Petrov From 06b315662d76775763b871b82fd1875f0db66ed2 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 5 Nov 2026 07:40:52 -0800 Subject: [PATCH 6/7] Address review: prompt placeholder test, already fixed in cdcb9c8 Thread: PRRT_kwDOVBg7g860FaDn: already fixed in cdcb9c8 Signed-off-by: Nikolay Petrov From f36cef841c85b4d095c541b86971f5fde45285d5 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 5 Nov 2026 07:40:52 -0800 Subject: [PATCH 7/7] Address review: nits of the LGTM review, already fixed in dba386c and cdcb9c8 Review: PRR_kwDOVBg7g88AAAABWNlvRA: already fixed in dba386c and cdcb9c8 Signed-off-by: Nikolay Petrov