From 159f1c159125f439053f0fd4da8415a92446259f Mon Sep 17 00:00:00 2001 From: Alok Sharma Date: Sun, 28 Jun 2026 16:07:25 +0530 Subject: [PATCH] Block truncated/lossy file regenerations from shipping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #9 committed a README that was the existing file echoed back and cut off mid-table — syntactically fine, so it passed validation and shipped. Root causes: .md files got no integrity check (static validation only syntax-checks TS/JS/JSON), the LLM client never inspected finish_reason so a max_tokens cut was returned silently, and the 4096-token cap is too small for full-file rewrites. - static.ts: validateChanges now takes the originals map and flags a modified file that is a truncated copy of the original (exact prefix, shorter) or has lost >50% of its content — covers .md and other non-code files that get no syntax check. This deterministically catches PR #9. - index.ts: pass repoContext.keyFiles as the originals to validateChanges. - client.ts: capture choice.finish_reason and warn when a response is cut off at max_tokens, so token-cap truncation is no longer silent. - base.ts: raise the Backend/Frontend output cap from 4096 to 8192 tokens. Co-Authored-By: Claude Opus 4.8 (1M context) --- agent-core/src/agents/base.ts | 7 +++++-- agent-core/src/index.ts | 2 +- agent-core/src/llm/client.ts | 17 ++++++++++++--- agent-core/src/validation/static.ts | 32 ++++++++++++++++++++++++++++- 4 files changed, 51 insertions(+), 7 deletions(-) diff --git a/agent-core/src/agents/base.ts b/agent-core/src/agents/base.ts index f5308f2..a5c61e1 100644 --- a/agent-core/src/agents/base.ts +++ b/agent-core/src/agents/base.ts @@ -201,7 +201,9 @@ export class BackendAgent extends BaseAgent { // Single-shot with key files pre-loaded. Avoids tool-call overhead while // still giving the model the real source code it needs to match patterns. // Llama 4 Scout (30K TPM) keeps this in a separate rate-limit bucket from PM/Architect. - return callLLM(BACKEND_SYSTEM, input, SCOUT_MODEL, 4096); + // 8192 tokens: a full-file rewrite easily exceeds 4096, and a cut-off response + // produces invalid JSON (dropped silently) or a truncated file (shipped as-is). + return callLLM(BACKEND_SYSTEM, input, SCOUT_MODEL, 8192); } } @@ -243,7 +245,8 @@ export class FrontendAgent extends BaseAgent { async run(input: string, context: ExecutionContext): Promise { this.log(`Generating frontend code for ${context.repository}`); - return callLLM(FRONTEND_SYSTEM, input, SCOUT_MODEL, 4096); + // 8192 tokens — see BackendAgent: a truncated full-file response ships broken. + return callLLM(FRONTEND_SYSTEM, input, SCOUT_MODEL, 8192); } } diff --git a/agent-core/src/index.ts b/agent-core/src/index.ts index 8bda779..3250f3a 100644 --- a/agent-core/src/index.ts +++ b/agent-core/src/index.ts @@ -358,7 +358,7 @@ export class AgentCoordinator { ], context.repoContext ); - const staticValidation = validateChanges(codeChanges); + const staticValidation = validateChanges(codeChanges, context.repoContext?.keyFiles); const staticSection = formatStaticResults(staticValidation); // DevOps only needs the task, the plan, and the generated files — not the diff --git a/agent-core/src/llm/client.ts b/agent-core/src/llm/client.ts index 7b09301..3e5247d 100644 --- a/agent-core/src/llm/client.ts +++ b/agent-core/src/llm/client.ts @@ -37,6 +37,8 @@ export interface LLMToolCall { export interface ChatResultMessage { content: string | null; tool_calls?: LLMToolCall[]; + /** Why generation stopped. 'length' means the response was cut off at max_tokens. */ + finish_reason?: string | null; } export interface ChatRequest { @@ -72,7 +74,7 @@ const chatCompletion: ChatTransport = async (req) => { while (attempt <= MAX_RETRIES) { try { const response = await axios.post<{ - choices: Array<{ message: ChatResultMessage }>; + choices: Array<{ message: ChatResultMessage; finish_reason?: string | null }>; }>(`${GROQ_BASE_URL}/chat/completions`, req, { headers: { Authorization: `Bearer ${getApiKey()}`, @@ -81,11 +83,14 @@ const chatCompletion: ChatTransport = async (req) => { timeout: 120_000, }); - const message = response.data.choices[0]?.message; + const choice = response.data.choices[0]; + const message = choice?.message; if (!message) { throw new Error('Empty response from Groq'); } - return message; + // Surface the choice-level finish_reason on the message so callers can tell + // a complete response from one cut off at max_tokens. + return { ...message, finish_reason: choice.finish_reason ?? message.finish_reason }; } catch (err) { const axiosErr = err as AxiosError; const status = axiosErr.response?.status; @@ -131,6 +136,12 @@ export async function callLLM( { role: 'user', content: userMessage }, ], }); + if (message.finish_reason === 'length') { + console.warn( + `[LLM] Response cut off at max_tokens=${maxTokens} (finish_reason=length). ` + + `Output is incomplete — raise max_tokens or reduce scope. Model: ${model}.` + ); + } return message.content ?? ''; } diff --git a/agent-core/src/validation/static.ts b/agent-core/src/validation/static.ts index 83270ec..dfeae82 100644 --- a/agent-core/src/validation/static.ts +++ b/agent-core/src/validation/static.ts @@ -28,7 +28,10 @@ function extname(basename: string): string { * failure modes LLMs actually produce: malformed JSON, syntax errors, empty or * truncated content, and unsafe paths. Real ground truth to back the DevOps verdict. */ -export function validateChanges(changes: CodeChange[]): StaticValidationResult { +export function validateChanges( + changes: CodeChange[], + originals: Record = {} +): StaticValidationResult { const issues: ValidationIssue[] = []; for (const change of changes) { @@ -61,6 +64,33 @@ export function validateChanges(changes: CodeChange[]): StaticValidationResult { issues.push({ path, severity: 'error', message: 'File content appears truncated (contains a truncation marker).' }); } + // Lossy-regeneration check — when an agent modifies an EXISTING file, its + // output should not be a shrunken copy of the original. The failure mode this + // catches (and how it shipped a broken README before): the model echoes the + // file it was shown and trails off partway through, emitting syntactically + // valid output that is really the original truncated mid-document. Markdown + // and other non-code files get no syntax check, so this is their only guard. + const original = originals[path] ?? originals[normalized]; + if (original != null) { + const o = original.trimEnd(); + const n = content.trimEnd(); + // Exact prefix of the original but shorter → an echo that stopped early. + if (n.length < o.length && o.startsWith(n)) { + issues.push({ + path, + severity: 'error', + message: `Modified file is a truncated copy of the original (${n.length}/${o.length} chars, exact prefix) — the agent echoed the existing file and stopped early.`, + }); + } else if (o.length >= 400 && n.length < o.length * 0.5) { + // Not a prefix, but lost more than half its content — likely lossy. + issues.push({ + path, + severity: 'error', + message: `Modified file is ${Math.round((1 - n.length / o.length) * 100)}% shorter than the original (${n.length}/${o.length} chars) — likely truncated or lossy regeneration.`, + }); + } + } + // Type-specific syntax checks. const ext = extname(basename); if (ext === '.json') {