Agent mode phase 3: verification loop / completion criteria

The agent loop just stopped whenever the model ran out of tool calls, or
hit the step limit — nothing ever checked that the work it did actually
builds or passes tests. "Done" was whatever the model said it was.

- detectProjectScripts() (already backing the Test/Lint/Format quick-action
  buttons) gained a fourth field, build, detected the same way as the
  others from package.json's scripts.build.

- New opt-in settings (off by default — this runs commands automatically,
  which shouldn't surprise anyone who didn't turn it on): verificationEnabled,
  verificationCommands (explicit override, one per line in the new Settings
  section; falls back to the workspace's detected build+test scripts when
  unset), verificationMaxRetries (default 3, deliberately separate from
  agentMaxSteps so a persistently failing check can't loop forever even
  while the step budget still has headroom).

- Hooked into the exact point where runCompletion's post-stream handler
  already distinguishes "model called more tools" from "model believes the
  turn is over" (previously only used to trigger TTS auto-read). On the
  "turn is over" branch, when verification is enabled, runVerification()
  runs the configured command(s) for real through the same run_command tool
  path everything else uses — deterministically, not left to the model to
  decide whether to check its own work — and only lets the turn actually
  end once they pass. A failure gets fed back as a tool-result message so
  the model can see and fix it, then loops through the existing
  continueAfterTools plumbing for another attempt.

- New isVerification flag on ChatMessage (mirrors the existing pinned
  field: a UI-only affordance, never sent to a provider) lets the verification
  result render as a distinct pass/fail card instead of a generic tool-output
  box, plus a toolbar attempt counter next to the existing agent-step one.

2 existing detectProjectScripts test cases updated for the new build field,
plus a new one covering all four scripts detected together.

Author: voidstackloop

app/src/agent-tools.test.ts

@@ -208,18 +208,24 @@ describe("agent-tools", () => {
                 path.join(workspace, "package.json"),
                 JSON.stringify({ scripts: { test: "vitest", build: "tsc" } })
             );
-            expect(detectProjectScripts(workspace)).toEqual({ test: "npm test", lint: undefined, format: undefined });
+            expect(detectProjectScripts(workspace)).toEqual({
+                test: "npm test",
+                lint: undefined,
+                format: undefined,
+                build: "npm run build",
+            });
         });
 
-        it("reports test, lint, and format scripts together when all are present", () => {
+        it("reports test, lint, format, and build scripts together when all are present", () => {
             fs.writeFileSync(
                 path.join(workspace, "package.json"),
-                JSON.stringify({ scripts: { test: "vitest", lint: "eslint .", format: "prettier --write ." } })
+                JSON.stringify({ scripts: { test: "vitest", lint: "eslint .", format: "prettier --write .", build: "tsc" } })
             );
             expect(detectProjectScripts(workspace)).toEqual({
                 test: "npm test",
                 lint: "npm run lint",
                 format: "npm run format",
+                build: "npm run build",
             });
         });
     });

app/src/agent-tools.ts

@@ -1315,11 +1315,14 @@ export interface ProjectScripts {
     test?: string;
     lint?: string;
     format?: string;
+    build?: string;
 }
 
-// Backs the Test/Lint/Format quick-action buttons — only npm-style
-// package.json scripts are recognized, which covers the JS/TS projects this
-// app's Agent mode is primarily used against.
+// Backs the Test/Lint/Format quick-action buttons, and (build + test) the
+// default command list for the verification loop when the user hasn't set
+// an explicit override — only npm-style package.json scripts are
+// recognized, which covers the JS/TS projects this app's Agent mode is
+// primarily used against.
 export function detectProjectScripts(workspaceRoot: string): ProjectScripts {
     const pkgPath = resolveSafePath(workspaceRoot, "package.json");
     let scripts: Record<string, string> = {};
@@ -1333,6 +1336,7 @@ export function detectProjectScripts(workspaceRoot: string): ProjectScripts {
         test: scripts.test ? "npm test" : undefined,
         lint: scripts.lint ? "npm run lint" : undefined,
         format: scripts.format ? "npm run format" : undefined,
+        build: scripts.build ? "npm run build" : undefined,
     };
 }
 

app/src/providers/types.ts

@@ -41,6 +41,9 @@ export interface ChatMessage {
     toolName?: string;
     // User-set bookmark, purely a UI affordance — never sent to a provider.
     pinned?: boolean;
+    // Set on the synthetic message the verification loop appends — a UI
+    // affordance like `pinned`, never sent to a provider.
+    isVerification?: boolean;
 }
 
 export interface ChatChunk {

app/src/settings-store.ts

@@ -110,6 +110,18 @@ export interface AppSettings {
     // a runaway process rather than act as a real resource quota system.
     sandboxMaxMemoryMB?: number;
     sandboxMaxCpuPercent?: number;
+    // Verification loop (Agent mode): once a turn ends with the model
+    // calling no more tools, optionally run these command(s) for real and
+    // feed the result back before treating the turn as actually finished.
+    // Off by default — running commands automatically after every turn
+    // would surprise anyone who didn't explicitly turn it on.
+    // verificationCommands unset falls back to the workspace's detected
+    // build/test scripts (see detectProjectScripts) at the point of use.
+    verificationEnabled?: boolean;
+    verificationCommands?: string[];
+    // Distinct from agentMaxSteps — bounds verify-fail-retry cycles
+    // specifically, so a persistently failing check can't loop forever.
+    verificationMaxRetries?: number;
 }
 
 const DEFAULTS: AppSettings = {
@@ -131,6 +143,8 @@ const DEFAULTS: AppSettings = {
     llamaCppMaxCachedModels: 2,
     networkToolsEnabled: true,
     sandboxMaxMemoryMB: 2048,
+    verificationEnabled: false,
+    verificationMaxRetries: 3,
 };
 
 function filePath(): string {

frontend/src/lib/translations.ts

@@ -118,6 +118,8 @@ export interface Dictionary {
     undoLastEdit: string;
     nothingToUndo: string;
     terminalPanelToggle: string;
+    verificationCard: string;
+    verificationAttemptCounter: (attempt: number, max: number) => string;
     restoredFile: string;
     deletedNewFile: string;
     runTests: string;
@@ -364,6 +366,13 @@ export interface Dictionary {
     sandboxStatusBubblewrap: string;
     sandboxStatusSandboxExec: string;
     sandboxStatusNone: string;
+    verificationSectionTitle: string;
+    verificationSectionHint: string;
+    verificationEnabledLabel: string;
+    verificationCommandsLabel: string;
+    verificationCommandsHint: string;
+    verificationMaxRetriesLabel: string;
+    verificationMaxRetriesHint: string;
     connectedAccountsTitle: string;
     connectedAccountsHint: string;
     accountGithubHint: string;
@@ -497,6 +506,8 @@ export const en: Dictionary = {
     undoLastEdit: "Undo last edit",
     nothingToUndo: "Nothing to undo.",
     terminalPanelToggle: "Terminal",
+    verificationCard: "Verification",
+    verificationAttemptCounter: (attempt, max) => `attempt ${attempt}/${max}`,
     restoredFile: "Restored previous content of",
     deletedNewFile: "Removed newly-created file",
     runTests: "Run tests",
@@ -751,6 +762,13 @@ export const en: Dictionary = {
     sandboxStatusBubblewrap: "Linux, via bubblewrap — commands run confined to the workspace, with network access denied unless a command explicitly requests it.",
     sandboxStatusSandboxExec: "macOS, via sandbox-exec — commands run confined to the workspace, with network access denied unless a command explicitly requests it.",
     sandboxStatusNone: "Not available on this platform — only the command blocklist and the resource limits below apply. On Linux, installing bubblewrap (bwrap) enables real filesystem and network containment.",
+    verificationSectionTitle: "Verification",
+    verificationSectionHint: "After the agent stops calling tools, optionally run real commands and feed the result back before treating the turn as done — off by default, since this runs commands automatically.",
+    verificationEnabledLabel: "Verify before finishing a turn",
+    verificationCommandsLabel: "Commands to run",
+    verificationCommandsHint: "One per line. Leave blank to use the workspace's detected build/test scripts automatically.",
+    verificationMaxRetriesLabel: "Max retries",
+    verificationMaxRetriesHint: "How many times the agent can try to fix a failing check before it stops and asks you.",
     connectedAccountsTitle: "Connected accounts",
     connectedAccountsHint: "Link developer services for repository analysis and access to private or gated models. Tokens stay encrypted locally when your OS credential store is available.",
     accountGithubHint: "Connect repositories for AI analysis and developer workflows.",
@@ -885,6 +903,8 @@ export const tr: Dictionary = {
     undoLastEdit: "Son düzenlemeyi geri al",
     nothingToUndo: "Geri alınacak bir şey yok.",
     terminalPanelToggle: "Terminal",
+    verificationCard: "Doğrulama",
+    verificationAttemptCounter: (attempt, max) => `${attempt}/${max}. deneme`,
     restoredFile: "Önceki içerik geri yüklendi:",
     deletedNewFile: "Yeni oluşturulan dosya kaldırıldı:",
     runTests: "Testleri çalıştır",
@@ -1139,6 +1159,13 @@ export const tr: Dictionary = {
     sandboxStatusBubblewrap: "Linux, bubblewrap aracılığıyla — komutlar çalışma alanıyla sınırlı çalışır, bir komut açıkça talep etmedikçe ağ erişimi reddedilir.",
     sandboxStatusSandboxExec: "macOS, sandbox-exec aracılığıyla — komutlar çalışma alanıyla sınırlı çalışır, bir komut açıkça talep etmedikçe ağ erişimi reddedilir.",
     sandboxStatusNone: "Bu platformda mevcut değil — yalnızca komut engelleme listesi ve aşağıdaki kaynak sınırları uygulanır. Linux'ta bubblewrap (bwrap) kurmak gerçek dosya sistemi ve ağ sınırlaması sağlar.",
+    verificationSectionTitle: "Doğrulama",
+    verificationSectionHint: "Ajan araç çağırmayı bıraktıktan sonra, isteğe bağlı olarak gerçek komutlar çalıştırıp sonucu geri bildirerek turu bitmiş saymadan önce kontrol edin — komutları otomatik çalıştırdığı için varsayılan olarak kapalıdır.",
+    verificationEnabledLabel: "Turu bitirmeden önce doğrula",
+    verificationCommandsLabel: "Çalıştırılacak komutlar",
+    verificationCommandsHint: "Her satıra bir tane. Çalışma alanının algılanan build/test betiklerini otomatik kullanmak için boş bırakın.",
+    verificationMaxRetriesLabel: "Maksimum yeniden deneme",
+    verificationMaxRetriesHint: "Ajanın başarısız bir kontrolü durup sizden yardım istemeden önce kaç kez düzeltmeyi deneyebileceği.",
     connectedAccountsTitle: "Bağlı hesaplar",
     connectedAccountsHint: "Depo analizi ve özel veya kısıtlı modellere erişim için geliştirici hizmetlerini bağlayın. İşletim sistemi kimlik bilgisi deposu kullanılabildiğinde jetonlar yerel olarak şifreli kalır.",
     accountGithubHint: "Yapay zeka analizi ve geliştirici iş akışları için depoları bağlayın.",

frontend/src/pages/Chat.tsx

@@ -250,6 +250,24 @@ const MessageBubble = memo(function MessageBubble({
     const { t } = useI18n();
 
     if (m.role === "tool") {
+        if (m.isVerification) {
+            const passed = m.content.startsWith("Verification passed");
+            return (
+                <div className="flex flex-col items-start">
+                    <div
+                        className={cn(
+                            "max-w-[85%] rounded-lg border px-3 py-2 text-sm",
+                            passed ? "border-primary/30 bg-primary/5" : "border-destructive/40 bg-destructive/5"
+                        )}
+                    >
+                        <div className={cn("mb-1 flex items-center gap-1.5 font-medium", passed ? "text-primary" : "text-destructive")}>
+                            {passed ? <Check className="size-3.5" /> : <AlertTriangle className="size-3.5" />} {t.verificationCard}
+                        </div>
+                        <pre className="max-h-48 overflow-auto whitespace-pre-wrap font-mono text-xs text-muted-foreground">{m.content}</pre>
+                    </div>
+                </div>
+            );
+        }
         // Tool failures get a visually distinct card — an agent run that hit
         // an error mid-way should be scannable at a glance, not require
         // reading every result body to find where things went wrong.
@@ -454,6 +472,10 @@ export default function Chat() {
     const [pendingToolCalls, setPendingToolCalls] = useState<ToolCall[]>([]);
     const [pendingVariablePreset, setPendingVariablePreset] = useState<PromptPreset | null>(null);
     const [agentStepCount, setAgentStepCount] = useState(0);
+    // Separate from agentStepCount — bounds verify-fail-retry cycles
+    // specifically, so a persistently failing check can't loop forever even
+    // while agentMaxSteps still has headroom left.
+    const [verificationAttempt, setVerificationAttempt] = useState(0);
     const [autoApprovedTools, setAutoApprovedTools] = useState<Set<string>>(new Set());
     const [planSteps, setPlanSteps] = useState<PlanStep[]>([]);
     const [contextSummary, setContextSummary] = useState<string | null>(null);
@@ -550,6 +572,7 @@ export default function Chat() {
             setAgentWorkspace(session.agentWorkspace ?? null);
             setPendingToolCalls([]);
             setAgentStepCount(0);
+            setVerificationAttempt(0);
             setPlanSteps(session.planSteps ?? []);
             setContextSummary(session.contextSummary ?? null);
             setContextSummaryThroughIndex(session.contextSummaryThroughIndex ?? 0);
@@ -1073,6 +1096,18 @@ export default function Chat() {
                 for (const call of otherCalls) {
                     if (call.name !== "request_checkpoint" && autoApprovedTools.has(call.name)) respondToToolCall(call, true);
                 }
+            } else if (
+                agentMode &&
+                agentWorkspace &&
+                settings?.verificationEnabled &&
+                !result.error &&
+                last?.role === "assistant" &&
+                verificationAttempt < (settings.verificationMaxRetries ?? 3)
+            ) {
+                // Fire-and-forget: this updater must stay synchronous, the
+                // actual command runs (and any setState that follows) happen
+                // later once the awaited calls inside resolve.
+                void runVerification(finalMessages);
             } else if (settings?.ttsAutoRead && !result.error && last?.role === "assistant" && last.content) {
                 const lastIndex = finalMessages.length - 1;
                 setSpeakingIndex(lastIndex);
@@ -1090,6 +1125,59 @@ export default function Chat() {
         });
     }
 
+    // Runs once a turn ends with the model calling no more tools — i.e. it
+    // believes it's done. Deterministic and app-driven rather than asking
+    // the model whether to check its own work: runs the configured
+    // command(s) for real via the same run_command path everything else
+    // uses (inheriting whatever sandboxing/resource limits are configured),
+    // and only lets the turn actually end once they pass.
+    async function runVerification(finalMessages: ChatMessage[]) {
+        if (!agentWorkspace) return;
+        const commands = settings?.verificationCommands?.length
+            ? settings.verificationCommands
+            : [projectScripts.build, projectScripts.test].filter((c): c is string => Boolean(c));
+        if (commands.length === 0) return;
+
+        const results: { command: string; passed: boolean; output: string }[] = [];
+        for (const command of commands) {
+            const res = await window.api.agent.executeTool(agentWorkspace, "run_command", { command });
+            const output = typeof res.result === "string" ? res.result : (res.error ?? "");
+            results.push({ command, passed: !res.error && output.includes("Exit code: 0"), output });
+        }
+        const allPassed = results.every((r) => r.passed);
+        const checklist = results.map((r) => `${r.passed ? "✅" : "❌"} ${r.command}`).join("\n");
+        const failureDetail = results
+            .filter((r) => !r.passed)
+            .map((r) => `--- ${r.command} ---\n${r.output}`)
+            .join("\n\n");
+        const verificationMessage: ChatMessage = {
+            role: "tool",
+            content: allPassed
+                ? `Verification passed:\n${checklist}`
+                : `Verification failed:\n${checklist}\n\n${failureDetail}`,
+            isVerification: true,
+        };
+        const nextMessages = [...finalMessages, verificationMessage];
+        setMessages(nextMessages);
+        if (sessionId) window.api.sessions.update(sessionId, { messages: nextMessages });
+
+        if (allPassed) return; // the turn really is done
+
+        const maxRetries = settings?.verificationMaxRetries ?? 3;
+        if (verificationAttempt + 1 >= maxRetries) {
+            setMessages((m) => [
+                ...m,
+                {
+                    role: "assistant",
+                    content: `⚠️ Verification kept failing after ${maxRetries} attempt${maxRetries === 1 ? "" : "s"}. Send another message to try again.`,
+                },
+            ]);
+            return;
+        }
+        setVerificationAttempt((a) => a + 1);
+        continueAfterTools(nextMessages);
+    }
+
     // Runs after every tool call from one assistant turn has been approved or
     // denied — feeds the tool results back to the model so it can continue
     // (e.g. read a file, then act on what it found) without the user having
@@ -1261,6 +1349,7 @@ export default function Chat() {
         setRagFolders([]);
         setImageAttachments([]);
         setAgentStepCount(0);
+        setVerificationAttempt(0);
         setPlanSteps([]);
         window.api.sessions.update(sessionId, { planSteps: [] });
         await runCompletion(history, baseMessages, { isFirstMessage, titleSource });
@@ -1590,6 +1679,11 @@ export default function Chat() {
                         {t.agentStep} {agentStepCount}/{agentMaxSteps}
                     </span>
                 )}
+                {verificationAttempt > 0 && (
+                    <span className="text-xs text-muted-foreground">
+                        {t.verificationCard} {t.verificationAttemptCounter(verificationAttempt, settings?.verificationMaxRetries ?? 3)}
+                    </span>
+                )}
                 {agentMode && agentWorkspace && (
                     <Button
                         size="sm"