diff --git a/dokimi/changelog.d/20261104_120000_napetrov_raw-task-prompt.md b/dokimi/changelog.d/20261104_120000_napetrov_raw-task-prompt.md new file mode 100644 index 000000000..993cc1a71 --- /dev/null +++ b/dokimi/changelog.d/20261104_120000_napetrov_raw-task-prompt.md @@ -0,0 +1,8 @@ + + +### Changed + +- **Command arms (`command:` and `--command-template`) send the task instruction as written** — the `Solve this coding task. Work only inside the task environment…` and `# Task: ` lines that dokimi put around `{prompt}` are gone, as a user's agent never sees them. `{prompt}` and `{prompt_quoted}` now expand to the bare `instruction.md`, and `{prompt_escaped}` to the same text with `{` and `}` doubled. Command-arm runs measured before this change are not comparable in their prompt text. diff --git a/dokimi/dokimi/harnesses/command.py b/dokimi/dokimi/harnesses/command.py index 7ab2ffa01..4d038d15f 100644 --- a/dokimi/dokimi/harnesses/command.py +++ b/dokimi/dokimi/harnesses/command.py @@ -386,11 +386,13 @@ def _render_template(template: str, context: dict[str, Any]) -> str: def _task_prompt(task: TaskSpec) -> str: - return ( - "Solve this coding task. Work only inside the task environment. " - "When finished, leave the files ready for the verifier.\n\n" - f"# Task: {task.name}\n\n{task.instruction}" - ) + """The task instruction as the user wrote it: no wrapper, the way a real harness sends it. + + A wrapper ("Solve this coding task…", "# Task: ") is text dokimi adds that a + user's agent never sees, so it can move the score of a command arm for no reason + the task measures. + """ + return task.instruction # Windows reports a crash or Ctrl-C as an NTSTATUS error code (0xC0000005, 0xC000013A, 0xC0000409). diff --git a/dokimi/tests/test_command_prompt.py b/dokimi/tests/test_command_prompt.py new file mode 100644 index 000000000..fb26779c1 --- /dev/null +++ b/dokimi/tests/test_command_prompt.py @@ -0,0 +1,29 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""A command arm's prompt is the task instruction as written, with no dokimi wrapper.""" + +from __future__ import annotations + +from dokimi.harnesses.command import CommandHarness +from dokimi.harnesses.tasks import TaskSpec + + +def test_command_prompt_is_the_raw_instruction(tmp_path): + task = TaskSpec(name="toy", path=tmp_path / "toy", instruction="Fix the off-by-one in sum().") + harness = CommandHarness("fake", "echo {prompt_quoted}") + argv = harness.build_command(task, model=None, output_dir=tmp_path / "out") + assert argv == ["echo", "Fix the off-by-one in sum()."] + + +def test_every_prompt_placeholder_keeps_a_multiline_instruction_as_written(tmp_path): + instruction = "Fix {x}.\nSecond line.\n" + task = TaskSpec(name="toy", path=tmp_path / "toy", instruction=instruction) + + def argv(template): + harness = CommandHarness("fake", template) + return harness.build_command(task, model=None, output_dir=tmp_path / "out") + + assert argv("echo '{prompt}'") == ["echo", instruction] + assert argv("echo {prompt_quoted}") == ["echo", instruction] + assert argv("echo '{prompt_escaped}'") == ["echo", "Fix {{x}}.\nSecond line.\n"]