diff --git a/server/workflows/errors/triage.ts b/server/workflows/errors/triage.ts index 11f7cc1..181a9dc 100644 --- a/server/workflows/errors/triage.ts +++ b/server/workflows/errors/triage.ts @@ -136,22 +136,49 @@ export function rulesTriage(event: NormalizedEvent): TriageResult { return result(['needs-human'], `${what}. Thrown from ${culprit?.text ?? 'your code'}. It is not an obvious small fix.`, 'Report it, and have a person look first. It is not a clear small fix for the resolver.'); } +/** + * The prompt is written for small local models: a fixed order of questions, one short example per outcome, and the + * free-text "message" asked for before the verdict so the model has said why before it decides. The checks in + * `modelTriager` still overrule it on facts, so this prompt only has to be right most of the time, not every time. + */ const SYSTEM_PROMPT = `You triage production errors for a code-fixing bot. -Reply with JSON only: {"small_fix": boolean, "message": string, "suggestion": string, "tags": string[]} +Reply with JSON only, keys in this order: {"message": string, "suggestion": string, "small_fix": boolean, "tags": string[]} -small_fix: true only if the error is a thrown exception that can be fixed with a change of about 2-5 lines in one -place in the app's own code (a missing null check, a typo, a missing import, a wrong variable) or is a lint/formatting finding. -false for log messages, permission or configuration problems, anything that depends on infrastructure or network -conditions, anything that needs a design change or touches several files, or when you cannot tell. When unsure, false. +First decide what the error is. Go through these questions in order and stop at the first "yes": +1. A test message (sent on purpose, at level debug, info or warning, saying test / hello world / ignore me)? Then small_fix is false and tags is ["test-event"]. +2. A permission, a setting, a missing file or service, a network or timeout problem? Then small_fix is false and tags has "infrastructure" and "needs-human". +3. A log message, not a thrown exception? Then small_fix is false and tags has "needs-human". +4. A stack that never enters the app's own code (every frame is a library)? Then small_fix is false and tags has "third-party" and "needs-human". +5. No stack trace at all? Then small_fix is false and tags has "no-stack-trace" and "needs-human". +6. A thrown exception in the app's own code that a change of about 2-5 lines in one place fixes (a missing null check, a typo, a missing import, a wrong variable), or a lint/formatting finding? Then small_fix is true. +7. Anything else, or you cannot tell: small_fix is false and tags has "needs-human". When unsure, false. message: 1-2 plain sentences. What this is, why it most likely happened, and who or what is the culprit (a function and file in the app, a library, the environment or infrastructure, or nobody: it is a test message). -Use the locals and the request when they are given. +Use the locals and the request when they are given. Only name files and functions that appear in the error. suggestion: one sentence on what to do. Say "Don't report it" ONLY for a test message or pure noise. A real failure is never "don't report it": if the fix is outside the code (a permission, a setting, a service), say what to fix instead. -tags: any of ${TRIAGE_TAGS.filter((t) => t !== 'bot-safe').join(', ')}. Do not include bot-safe; it follows small_fix.`; +tags: any of ${TRIAGE_TAGS.filter((t) => t !== 'bot-safe').join(', ')}. Do not include bot-safe; it follows small_fix. Use only the tags that apply. + +Examples (the errors are shortened; answer in your own words, not these): + +Error: TypeError: Cannot read properties of undefined (reading 'id') +Stack (newest first): + at getUserId (static/main.js:42) [app] + locals: user=undefined +{"message": "getUserId in main.js reads user.id while user is undefined, so a null check is missing.", "suggestion": "Report it: add a check before reading user.id.", "small_fix": true, "tags": []} + +Error: HTTPException: osascript is not allowed assistive access. (-1719) +Stack (newest first): + at press_key (control.py:31) [app] +{"message": "The app is not allowed to control the keyboard on this Mac, so the call in press_key was refused. No code is wrong.", "suggestion": "Grant Accessibility access to the app in System Settings, Privacy & Security.", "small_fix": false, "tags": ["infrastructure", "needs-human"]} + +Error: Message: Sentry test please ignore +Level: info +No stack trace: this was sent as a plain message. +{"message": "This is a test message sent on purpose. Nothing failed and there is no culprit.", "suggestion": "Don't report it: it is a test.", "small_fix": false, "tags": ["test-event", "no-stack-trace"]}`; const text = (value: unknown): string | null => (typeof value === 'string' && value.trim() !== '' ? clip(value.trim()) : null); const DISMISSES = /^don'?t report it/i; diff --git a/server/workflows/resolver/fixer.ts b/server/workflows/resolver/fixer.ts index b49f4b6..5f76c55 100644 --- a/server/workflows/resolver/fixer.ts +++ b/server/workflows/resolver/fixer.ts @@ -20,7 +20,21 @@ const SYSTEM_PROMPT = 'succeed, you have not made a fix, no matter how confident the reasoning sounds.\n\n' + 'You are running unattended — no person is reading this conversation or able to answer a question ' + 'from you. If you want to see a file, call read_file yourself; never ask for information instead of ' + - 'calling the tool that gets it.'; + 'calling the tool that gets it.\n\n' + + 'How to work, in this order:\n' + + "1. Find where it broke. In a stack trace the newest frame in the repository's own code is where the " + + 'error was thrown. Read that file around that line (the issue may already show it).\n' + + '2. Decide if a code change can fix it. If the error is a permission, a setting, a missing service, a ' + + "network or timeout problem, a failure inside a library with none of this repository's code in the " + + 'stack, or a log message with no failing code, answer SKIP and change nothing.\n' + + '3. Name the cause before you edit: the exact line and why it fails. Fix that cause with the smallest ' + + 'change, in the style of the surrounding code. Do not hide the error (an empty catch, a blanket ' + + 'try/except) unless the surrounding code already handles errors that way.\n' + + '4. edit_file needs old_string copied exactly from the file, character for character and with its ' + + 'indentation, without any line-number prefix, and it must appear only once: add a neighbouring line if ' + + 'it does not. One tool call makes one change. If edit_file fails, read the file again and retry with ' + + 'the exact text; never repeat the same call.\n' + + '5. After edit_file succeeds, read the changed lines once to check them, then give your verdict.'; const NUDGE = 'No one is here to answer that. Either call a tool yourself to get the information you need, or, ' + diff --git a/test/workflows/errors/triage.test.ts b/test/workflows/errors/triage.test.ts index dec8ba4..eb6c4ae 100644 --- a/test/workflows/errors/triage.test.ts +++ b/test/workflows/errors/triage.test.ts @@ -39,6 +39,17 @@ describe('modelTriager over Ollama', () => { assert.deepEqual((await triage(normalized(pyEvent()))).tags, ['bot-safe']); // the rules alone would say no }); + it('asks for the reason before the verdict, and shows the model one example per outcome', async () => { + const triage = triagerAt(await ready); + reply = () => content('{"message": "m", "suggestion": "s", "small_fix": false, "tags": []}'); + await triage(normalized(jsEvent())); + const system = JSON.parse(seen).messages[0].content as string; + assert.ok(system.indexOf('"message": string') < system.indexOf('"small_fix": boolean')); + assert.match(system, /"small_fix": true/); + assert.match(system, /"test-event"/); + assert.match(system, /"infrastructure", "needs-human"/); + }); + it("never lets the rules' wording contradict the model when it leaves its own out", async () => { const triage = triagerAt(await ready); reply = () => content('{"small_fix": false}');