diff --git a/agent-core/src/agents/base.ts b/agent-core/src/agents/base.ts index f5308f2..a5c61e1 100644 --- a/agent-core/src/agents/base.ts +++ b/agent-core/src/agents/base.ts @@ -201,7 +201,9 @@ export class BackendAgent extends BaseAgent { // Single-shot with key files pre-loaded. Avoids tool-call overhead while // still giving the model the real source code it needs to match patterns. // Llama 4 Scout (30K TPM) keeps this in a separate rate-limit bucket from PM/Architect. - return callLLM(BACKEND_SYSTEM, input, SCOUT_MODEL, 4096); + // 8192 tokens: a full-file rewrite easily exceeds 4096, and a cut-off response + // produces invalid JSON (dropped silently) or a truncated file (shipped as-is). + return callLLM(BACKEND_SYSTEM, input, SCOUT_MODEL, 8192); } } @@ -243,7 +245,8 @@ export class FrontendAgent extends BaseAgent { async run(input: string, context: ExecutionContext): Promise { this.log(`Generating frontend code for ${context.repository}`); - return callLLM(FRONTEND_SYSTEM, input, SCOUT_MODEL, 4096); + // 8192 tokens — see BackendAgent: a truncated full-file response ships broken. + return callLLM(FRONTEND_SYSTEM, input, SCOUT_MODEL, 8192); } } diff --git a/agent-core/src/index.ts b/agent-core/src/index.ts index 8bda779..3250f3a 100644 --- a/agent-core/src/index.ts +++ b/agent-core/src/index.ts @@ -358,7 +358,7 @@ export class AgentCoordinator { ], context.repoContext ); - const staticValidation = validateChanges(codeChanges); + const staticValidation = validateChanges(codeChanges, context.repoContext?.keyFiles); const staticSection = formatStaticResults(staticValidation); // DevOps only needs the task, the plan, and the generated files — not the diff --git a/agent-core/src/llm/client.ts b/agent-core/src/llm/client.ts index 7b09301..3e5247d 100644 --- a/agent-core/src/llm/client.ts +++ b/agent-core/src/llm/client.ts @@ -37,6 +37,8 @@ export interface LLMToolCall { export interface ChatResultMessage { content: string | null; tool_calls?: LLMToolCall[]; + /** Why generation stopped. 'length' means the response was cut off at max_tokens. */ + finish_reason?: string | null; } export interface ChatRequest { @@ -72,7 +74,7 @@ const chatCompletion: ChatTransport = async (req) => { while (attempt <= MAX_RETRIES) { try { const response = await axios.post<{ - choices: Array<{ message: ChatResultMessage }>; + choices: Array<{ message: ChatResultMessage; finish_reason?: string | null }>; }>(`${GROQ_BASE_URL}/chat/completions`, req, { headers: { Authorization: `Bearer ${getApiKey()}`, @@ -81,11 +83,14 @@ const chatCompletion: ChatTransport = async (req) => { timeout: 120_000, }); - const message = response.data.choices[0]?.message; + const choice = response.data.choices[0]; + const message = choice?.message; if (!message) { throw new Error('Empty response from Groq'); } - return message; + // Surface the choice-level finish_reason on the message so callers can tell + // a complete response from one cut off at max_tokens. + return { ...message, finish_reason: choice.finish_reason ?? message.finish_reason }; } catch (err) { const axiosErr = err as AxiosError; const status = axiosErr.response?.status; @@ -131,6 +136,12 @@ export async function callLLM( { role: 'user', content: userMessage }, ], }); + if (message.finish_reason === 'length') { + console.warn( + `[LLM] Response cut off at max_tokens=${maxTokens} (finish_reason=length). ` + + `Output is incomplete — raise max_tokens or reduce scope. Model: ${model}.` + ); + } return message.content ?? ''; } diff --git a/agent-core/src/validation/static.ts b/agent-core/src/validation/static.ts index 83270ec..dfeae82 100644 --- a/agent-core/src/validation/static.ts +++ b/agent-core/src/validation/static.ts @@ -28,7 +28,10 @@ function extname(basename: string): string { * failure modes LLMs actually produce: malformed JSON, syntax errors, empty or * truncated content, and unsafe paths. Real ground truth to back the DevOps verdict. */ -export function validateChanges(changes: CodeChange[]): StaticValidationResult { +export function validateChanges( + changes: CodeChange[], + originals: Record = {} +): StaticValidationResult { const issues: ValidationIssue[] = []; for (const change of changes) { @@ -61,6 +64,33 @@ export function validateChanges(changes: CodeChange[]): StaticValidationResult { issues.push({ path, severity: 'error', message: 'File content appears truncated (contains a truncation marker).' }); } + // Lossy-regeneration check — when an agent modifies an EXISTING file, its + // output should not be a shrunken copy of the original. The failure mode this + // catches (and how it shipped a broken README before): the model echoes the + // file it was shown and trails off partway through, emitting syntactically + // valid output that is really the original truncated mid-document. Markdown + // and other non-code files get no syntax check, so this is their only guard. + const original = originals[path] ?? originals[normalized]; + if (original != null) { + const o = original.trimEnd(); + const n = content.trimEnd(); + // Exact prefix of the original but shorter → an echo that stopped early. + if (n.length < o.length && o.startsWith(n)) { + issues.push({ + path, + severity: 'error', + message: `Modified file is a truncated copy of the original (${n.length}/${o.length} chars, exact prefix) — the agent echoed the existing file and stopped early.`, + }); + } else if (o.length >= 400 && n.length < o.length * 0.5) { + // Not a prefix, but lost more than half its content — likely lossy. + issues.push({ + path, + severity: 'error', + message: `Modified file is ${Math.round((1 - n.length / o.length) * 100)}% shorter than the original (${n.length}/${o.length} chars) — likely truncated or lossy regeneration.`, + }); + } + } + // Type-specific syntax checks. const ext = extname(basename); if (ext === '.json') {