diff --git a/src/map/extract.ts b/src/map/extract.ts index fd83f7a7..628351c5 100644 --- a/src/map/extract.ts +++ b/src/map/extract.ts @@ -4,10 +4,11 @@ import { relative } from 'node:path'; import type { SiteInputMap, Endpoint, TsModule } from './types.js'; import { guessScriptKind } from './ast.js'; import { buildModuleBindings } from './bindings.js'; -import { collectSources, detectDeploymentShapes, detectFramework, hasEntrySignal, type WalkStats } from './sources.js'; +import { collectSources, componentScript, detectDeploymentShapes, detectFramework, hasEntrySignal, isComponentFile, type WalkStats } from './sources.js'; import { classifyServerSurface, surfaceNote } from './surface.js'; import { functionNameFromPath, routeFromFilePath } from './routes.js'; import { collectLocalSinks } from './sinks.js'; +import { boundUnprovenFlows } from './flows.js'; import { createModuleGraph } from './module-graph.js'; import { isProvenFlow } from './coordinates.js'; import { extractFromFile } from './entries.js'; @@ -15,6 +16,8 @@ import { collectFileImports, countUnresolvableImports, createImportInventory, re import { collectInvocations, createInvocationInventory } from './invocations.js'; const MAX_DEPENDENCY_INPUT_FLOWS_PER_MAP = 500; +// Unproven flows across the whole map; see `boundUnprovenFlows`. +const MAX_UNPROVEN_FLOWS_PER_MAP = 1000; // Framework-AGNOSTIC input-flow extractor. It doesn't gate on a specific stack — it walks any JS/TS // source and applies recognizer tables for (1) entry points, (2) inputs, (3) sinks, so it generalizes @@ -46,9 +49,11 @@ export async function extractInputMap(cwd: string, ts: TsModule, options: Extrac const imports = createImportInventory(readPathAliases(cwd)); const invocations = createInvocationInventory(); let dependencyInputFlowCount = 0; + let unprovenFlowCount = 0; let parsed = 0; let preFiltered = 0; let importScanFailures = 0; + let componentFiles = 0; let unresolvableImports = 0; let sourceBytes = 0; const calls = { total: 0, dependency: 0, local: 0, ambiguous: 0 }; @@ -62,6 +67,17 @@ export async function extractInputMap(cwd: string, ts: TsModule, options: Extrac const text = readFileSync(file, 'utf8'); const relFile = relative(cwd, file); sourceBytes += text.length; + // A single-file component contributes its imports, not endpoints: only its script is JavaScript. + if (isComponentFile(file)) { + componentFiles++; + preFiltered++; + const script = componentScript(text, file); + const scanned = script === null ? null : scanFileImports(script, ts); + if (scanned === null) importScanFailures++; + else imports.add(relFile, scanned, false); + if (script !== null) unresolvableImports += countUnresolvableImports(script, ts); + continue; + } // Imports are collected from EVERY file, entry point or not: the data layer of an AI-built app // usually lives in a file with no handler in it, so a pre-filtered file is exactly where the // interesting dependency is imported. @@ -108,6 +124,12 @@ export async function extractInputMap(cwd: string, ts: TsModule, options: Extrac ep.dependencyInputFlowsTruncated = true; } dependencyInputFlowCount += ep.dependencyInputFlows?.length ?? 0; + const bounded = boundUnprovenFlows(ep.flows, Math.max(0, MAX_UNPROVEN_FLOWS_PER_MAP - unprovenFlowCount)); + if (bounded.truncated) { + ep.flows = bounded.flows; + ep.flowsTruncated = true; + } + unprovenFlowCount += ep.flows.filter((f) => !isProvenFlow(f.confidence)).length; // A FILE-BASED route handler carries its URL path in its location, not in the code, so derive // it here — without this a rule can only be param-pinned, never route-scoped (`when.path`). if (ep.route === undefined && ep.entryKind === 'edge-function') { @@ -184,6 +206,10 @@ export async function extractInputMap(cwd: string, ts: TsModule, options: Extrac if (truncatedDependencyFlows > 0) { notes.push(`${truncatedDependencyFlows} endpoint(s) had more dependency-input links than the bounded map carries; those endpoint records are marked dependencyInputFlowsTruncated.`); } + const truncatedFlows = endpoints.filter((e) => e.flowsTruncated).length; + if (truncatedFlows > 0) { + notes.push(`${truncatedFlows} endpoint(s) had more unproven flows than the bounded map carries; those endpoint records are marked flowsTruncated. Every proven flow is kept.`); + } if (unresolved > 0) { notes.push(`${unresolved} endpoint(s) declare an input validator that could not be statically parsed — their inputs are UNKNOWN, not empty (marked inputsResolved: false).`); } @@ -191,6 +217,9 @@ export async function extractInputMap(cwd: string, ts: TsModule, options: Extrac if (heuristicOnly > 0) { notes.push(`${heuristicOnly} endpoint(s) have inputs and sinks but no proven data link — their flows say "may reach", not "does reach" (see each flow's confidence).`); } + if (componentFiles > 0) { + notes.push(`${componentFiles} single-file component(s) (.vue, .svelte, .astro) were scanned for imports only. Code in their script blocks and Astro frontmatter can run on the server, but it is not analyzed for endpoints, inputs or sinks.`); + } if (endpoints.length === 0) notes.push('No recognized server-side entry points found under the analyzed roots.'); const importList = imports.list(); diff --git a/src/map/flows.ts b/src/map/flows.ts index f97e9083..f25e376f 100644 --- a/src/map/flows.ts +++ b/src/map/flows.ts @@ -1,10 +1,36 @@ import type { AddressSpace, ApiInvocation, ArgumentRole, DependencyInputFlow, Flow, InputField, Limitation, Sink, TsModule } from './types.js'; import { bindingKey, calleeName, isValueRead, lineOf, rootIdentifier } from './ast.js'; import { REQ_SOURCES } from './inputs.js'; -import { addressSpaceOf } from './coordinates.js'; +import { addressSpaceOf, isProvenFlow } from './coordinates.js'; import { argumentRoleOf, CANDIDATE_FAMILIES } from './sinks.js'; const MAX_DEPENDENCY_INPUT_FLOWS_PER_ENDPOINT = 100; +/** + * Unproven flows are an inputs × sinks cross-product, so an endpoint with many fields and many sinks + * produces a map too large to ingest. Proven flows are not bounded: they are the evidence rules are + * built from, and each one stands for a read the code actually makes. + */ +const MAX_UNPROVEN_FLOWS_PER_ENDPOINT = 200; + +/** + * Keep every proven flow and at most `limit` unproven ones, in their original order. `truncated` says + * whether any unproven flow was left out. + */ +export function boundUnprovenFlows(flows: Flow[], limit: number): { flows: Flow[]; truncated: boolean } { + let unproven = 0; + let truncated = false; + const kept = flows.filter((f) => { + if (isProvenFlow(f.confidence)) return true; + if (unproven >= limit) { + truncated = true; + return false; + } + unproven++; + return true; + }); + + return { flows: kept, truncated }; +} /** A tainted binding: the path prefix it stands for, and the request region it came from if known. */ /** @@ -54,10 +80,12 @@ function spaceOfKey(key: string | undefined): AddressSpace | undefined { // the rest as "may reach". Matching is per (address space, path): a read of `query.id` is not evidence // about the body field `id`. // Spread onto an endpoint: `flows`, plus `limitations` only when there are any (keeps the common case clean). -export function linkedFlows(body: any, params: any, inputs: InputField[], sinks: Sink[], ts: TsModule, invocations: ApiInvocation[] = []): { flows: Flow[]; limitations?: Limitation[]; dependencyInputFlows?: DependencyInputFlow[]; dependencyInputFlowsTruncated?: true } { - const { flows, limitations, dependencyInputFlows, dependencyInputFlowsTruncated } = linkFlows(body, params, inputs, sinks, ts, invocations); +export function linkedFlows(body: any, params: any, inputs: InputField[], sinks: Sink[], ts: TsModule, invocations: ApiInvocation[] = []): { flows: Flow[]; flowsTruncated?: true; limitations?: Limitation[]; dependencyInputFlows?: DependencyInputFlow[]; dependencyInputFlowsTruncated?: true } { + const { flows: linked, limitations, dependencyInputFlows, dependencyInputFlowsTruncated } = linkFlows(body, params, inputs, sinks, ts, invocations); + const { flows, truncated } = boundUnprovenFlows(linked, MAX_UNPROVEN_FLOWS_PER_ENDPOINT); return { flows, + ...(truncated ? { flowsTruncated: true as const } : {}), ...(limitations.length > 0 ? { limitations } : {}), ...(dependencyInputFlows.length > 0 ? { dependencyInputFlows } : {}), ...(dependencyInputFlowsTruncated ? { dependencyInputFlowsTruncated: true as const } : {}), diff --git a/src/map/sources.ts b/src/map/sources.ts index 92dacd48..30c13a99 100644 --- a/src/map/sources.ts +++ b/src/map/sources.ts @@ -37,7 +37,46 @@ export function detectFramework(cwd: string): string { return 'unknown'; } -const isSourceFile = (name: string) => /\.(ts|tsx|mts|cts|js|jsx|mjs|cjs)$/.test(name) && !name.endsWith('.d.ts'); +// Single-file components are source too: their script blocks import packages, and Astro frontmatter and +// SvelteKit/Nuxt component scripts can run on the server. They are scanned for imports only. +const isSourceFile = (name: string) => (/\.(ts|tsx|mts|cts|js|jsx|mjs|cjs)$/.test(name) && !name.endsWith('.d.ts')) || isComponentFile(name); + +/** A `.vue`, `.svelte` or `.astro` single-file component. */ +export const isComponentFile = (name: string) => /\.(vue|svelte|astro)$/.test(name); + +/** + * The script of a single-file component, with everything else blanked to spaces so offsets and line + * numbers still point into the original file: every `\n'], + ['src/components/Panel.svelte', '\n
\n'], + ['src/pages/panel.astro', "---\nimport { format } from 'sample-lib';\nconst value = format(Astro.url);\n---\n
{value}
\n"], + ])('records the imports of %s and stays complete', async (file, source) => { + const map = await mapOf({ [file]: source }); + + expect(packages(map)).toEqual(['sample-lib']); + expect(map.coverage.importsComplete).toBe(true); + }); + + it('reads every script block of a component', async () => { + const map = await mapOf({ + 'src/App.vue': "\n\n", + }); + + expect(packages(map)).toEqual(['other-lib', 'sample-lib']); + }); + + it('reports lines in the component file', async () => { + const map = await mapOf({ 'src/App.vue': "\n\n" }); + + expect(site(map, 'sample-lib')).toMatchObject({ file: path.join('src', 'App.vue'), line: 5 }); + }); + + it('does not read imports written outside a script block', async () => { + const map = await mapOf({ 'src/App.vue': "\n\n" }); + + expect(packages(map)).toEqual([]); + }); + + it('marks the inventory incomplete when a component script cannot be delimited', async () => { + const map = await mapOf({ 'src/App.vue': "\n' }); + + expect(map.coverage.importCoverageGaps.unresolvableImports).toBe(1); + expect(map.coverage.importsComplete).toBe(false); + }); + + it('notes that components are scanned for imports only', async () => { + const map = await mapOf({ 'src/App.vue': "\n" }); + + expect(map.coverage.notes.join(' ')).toMatch(/1 single-file component\(s\).*imports only/); + }); + + it('treats a skipped directory holding only components as holding source', async () => { + const map = await mapOf({ 'vendor/widgets/Panel.vue': "\n" }); + + expect(map.coverage.importsComplete).toBe(false); + expect(map.coverage.importCoverageGaps.skippedDirsWithSource).toEqual(['vendor']); + }); +}); + +describe('component script extraction', () => { + it('keeps offsets aligned with the original file', () => { + const text = '\n\n'; + const script = componentScript(text, 'App.vue')!; + + expect(script).toHaveLength(text.length); + expect(script.indexOf('import a')).toBe(text.indexOf('import a')); + expect(script).not.toContain('template'); + }); + + it('keeps empty Astro frontmatter readable', () => { + expect(componentScript('---\n---\n
\n', 'x.astro')).not.toBeNull(); + }); +}); diff --git a/tests/map/flow-bounds.test.ts b/tests/map/flow-bounds.test.ts new file mode 100644 index 00000000..4d510e60 --- /dev/null +++ b/tests/map/flow-bounds.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it } from 'vitest'; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { buildInputMap } from '../../src/map/index.js'; +import { boundUnprovenFlows } from '../../src/map/flows.js'; + +/** + * Unproven flows are every input paired with every sink it was not shown to reach, so they grow with the + * product of the two. The map carries a bounded number of them and says when it left some out; proven + * flows, the ones rules are built from, are always kept. + */ +async function mapOf(files: Record) { + const dir = mkdtempSync(path.join(tmpdir(), 'ps-flow-bounds-')); + try { + writeFileSync(path.join(dir, 'package.json'), JSON.stringify({ name: 'x', dependencies: { next: '15.0.0' } })); + for (const [file, source] of Object.entries(files)) { + mkdirSync(path.join(dir, path.dirname(file)), { recursive: true }); + writeFileSync(path.join(dir, file), source); + } + const { map } = await buildInputMap(dir, {}); + + return map!; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +/** A handler reading `fields` body fields next to `sinks` exec calls, one of which gets `field0`. */ +const wideHandler = (fields: number, sinks: number) => { + const reads = Array.from({ length: fields }, (_, i) => `body.field${i}`).join(', '); + const calls = Array.from({ length: sinks }, (_, i) => (i === sinks - 1 ? ' exec(body.field0);' : ` exec('job-${i}');`)).join('\n'); + return `import { exec } from 'node:child_process';\nexport async function POST(request: Request) {\n const body = await request.json();\n console.log(${reads});\n${calls}\n return new Response('ok');\n}\n`; +}; + +const proven = (flows: any[]) => flows.filter((f) => f.confidence === 'exact-local' || f.confidence === 'transformed-local'); + +describe('bounded unproven flows', () => { + it('leaves a small endpoint untouched', async () => { + const map = await mapOf({ 'app/api/run/route.ts': wideHandler(5, 5) }); + const endpoint = map.endpoints[0]!; + + expect(endpoint.flows).toHaveLength(25); + expect(endpoint.flowsTruncated).toBeUndefined(); + expect(map.coverage.notes.join(' ')).not.toMatch(/flowsTruncated/); + }); + + it('bounds a wide endpoint, keeps its proven flow, and says so', async () => { + const map = await mapOf({ 'app/api/run/route.ts': wideHandler(40, 40) }); + const endpoint = map.endpoints[0]!; + + expect(endpoint.flows.length - proven(endpoint.flows).length).toBe(200); + expect(proven(endpoint.flows)).toEqual([expect.objectContaining({ input: 'field0', confidence: 'exact-local' })]); + expect(endpoint.flowsTruncated).toBe(true); + expect(map.coverage.notes.join(' ')).toMatch(/1 endpoint\(s\) had more unproven flows.*flowsTruncated/); + }); + + it('bounds unproven flows across the whole map', async () => { + const files = Object.fromEntries(Array.from({ length: 7 }, (_, i) => [`app/api/r${i}/route.ts`, wideHandler(20, 10)])); + const map = await mapOf(files); + const unproven = map.endpoints.flatMap((e) => e.flows).length - map.endpoints.flatMap((e) => proven(e.flows)).length; + + expect(unproven).toBe(1000); + expect(map.endpoints.every((e) => proven(e.flows).length === 1)).toBe(true); + expect(map.endpoints.filter((e) => e.flowsTruncated)).toHaveLength(2); + }); +}); + +describe('boundUnprovenFlows', () => { + const flow = (confidence: string, input: string) => ({ confidence, input }) as any; + + it('keeps every proven flow and the first unproven ones, in order', () => { + const flows = [flow('heuristic', 'a'), flow('exact-local', 'b'), flow('imported', 'c'), flow('transformed-local', 'd'), flow('unknown', 'e')]; + + expect(boundUnprovenFlows(flows, 1)).toEqual({ flows: [flows[0], flows[1], flows[3]], truncated: true }); + }); + + it('reports no truncation at the limit', () => { + const flows = [flow('heuristic', 'a'), flow('exact-local', 'b')]; + + expect(boundUnprovenFlows(flows, 1)).toEqual({ flows, truncated: false }); + }); +});