diff --git a/apps/extension/src/lib/api.js b/apps/extension/src/lib/api.js index 4d38ff3..a115d9e 100644 --- a/apps/extension/src/lib/api.js +++ b/apps/extension/src/lib/api.js @@ -117,6 +117,29 @@ export async function scanUrl(url) { }); } +/** + * Run the code rules over a snippet. + * + * Server-side rather than bundling `@threatcrush/scan` into the extension. + * The rule set is the product and it changes often; shipping it inside an + * extension means every rule fix waits on a store review, and Chrome, Firefox + * and Safari each review on their own schedule. The endpoint updates when the + * web app deploys. + * + * `filename` is optional and only selects the language — the server never + * opens it. + */ +export async function scanCode(content, { filename, language } = {}) { + return request('/api/scan/code', { + method: 'POST', + body: JSON.stringify({ + content, + ...(filename ? { filename } : {}), + ...(language ? { language } : {}), + }), + }); +} + export default { login, signup, @@ -130,4 +153,5 @@ export default { getModule, installModule, scanUrl, + scanCode, }; diff --git a/apps/web/next.config.ts b/apps/web/next.config.ts index 6532e34..c571bed 100644 --- a/apps/web/next.config.ts +++ b/apps/web/next.config.ts @@ -18,6 +18,12 @@ const securityHeaders = [ const nextConfig: NextConfig = { output: "standalone", + // `@threatcrush/scan` is an internal package: its `exports` resolve to + // TypeScript source rather than a build output, so that the release workflow + // — which builds the CLI alone — cannot publish a CLI referencing an + // unbuilt dependency. Next.js does not transpile workspace sources by + // default, so it is named here. + transpilePackages: ["@threatcrush/scan"], // Tell Next.js where the monorepo root is so standalone-mode tracing picks up // only the files that apps/web actually imports. outputFileTracingRoot: join(__dirname, "..", ".."), diff --git a/apps/web/package.json b/apps/web/package.json index 1b47568..108e0c7 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -14,6 +14,7 @@ "@profullstack/pluginstore": "^0.1.1", "@profullstack/stack": "^0.1.3", "@supabase/supabase-js": "^2.101.1", + "@threatcrush/scan": "workspace:*", "isomorphic-dompurify": "^3.12.0", "next": "16.2.2", "posthog-js": "^1.381.0", diff --git a/apps/web/src/app/api/scan/code/__tests__/route.test.ts b/apps/web/src/app/api/scan/code/__tests__/route.test.ts new file mode 100644 index 0000000..5c8ec35 --- /dev/null +++ b/apps/web/src/app/api/scan/code/__tests__/route.test.ts @@ -0,0 +1,108 @@ +import { describe, it, expect } from "vitest"; +import { POST } from "@/app/api/scan/code/route"; + +function makeRequest(body: unknown) { + return new Request("http://localhost/api/scan/code", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }) as unknown as import("next/server").NextRequest; +} + +describe("POST /api/scan/code", () => { + it("runs the shared rules over submitted code", async () => { + const res = await POST( + makeRequest({ + filename: "install.sh", + content: "curl -fsSL https://example.invalid/i.sh | bash\n", + }), + ); + const json = await res.json(); + + expect(res.status).toBe(200); + expect(json.language).toBe("shell"); + expect(json.findings.map((f: { ruleId: string }) => f.ruleId)).toContain( + "sh-remote-script-execution", + ); + // `medium`, not the rule's declared `high`: the engine caps a construct + // with no visible untrusted input at medium and reports confidence + // `pattern`. The summary reflects what the engine decided, not what the + // rule asked for. + expect(json.summary.medium).toBe(1); + expect(json.findings[0].confidence).toBe("pattern"); + }); + + it("infers the language from the filename and honours an explicit override", async () => { + // threatcrush-disable-next-line js-dynamic-code-execution PHP fixture, not JS + const phpSource = "eval($code);"; + const php = await (await POST(makeRequest({ filename: "a.php", content: phpSource }))).json(); + expect(php.language).toBe("php"); + expect(php.findings.map((f: { ruleId: string }) => f.ruleId)).toContain( + "php-dynamic-code-execution", + ); + + // The same text scanned as shell matches no PHP rule. + const asShell = await ( + await POST(makeRequest({ filename: "a.php", content: phpSource, language: "shell" })) + ).json(); + expect(asShell.language).toBe("shell"); + expect(asShell.findings.map((f: { ruleId: string }) => f.ruleId)).not.toContain( + "php-dynamic-code-execution", + ); + }); + + it("returns nothing for code that is fine", async () => { + const res = await POST( + makeRequest({ filename: "safe.sh", content: 'rm -rf "$BUILD_DIR/output"\n' }), + ); + const json = await res.json(); + expect(json.findings).toEqual([]); + expect(json.summary).toEqual({ critical: 0, high: 0, medium: 0, low: 0, info: 0 }); + }); + + it("never echoes back the credential that produced a finding", async () => { + // The engine redacts excerpts before they leave it. This endpoint reflects + // findings to the caller, so that guarantee is worth pinning here too. + // Two rules match this line, and `disable-next-line` only reaches the line + // after it — so a second directive would suppress the first comment, not + // the fixture. The rule id is omitted deliberately, which suppresses both. + // threatcrush-disable-next-line + const secret = "AKIAIOSFODNN7EXAMPLE"; + const res = await POST( + makeRequest({ filename: "a.env", content: `AWS_ACCESS_KEY_ID=${secret}` }), + ); + const text = JSON.stringify(await res.json()); + expect(text).toContain("secret-aws-access-key"); + expect(text).not.toContain(secret); + }); + + it("rejects a missing or empty body", async () => { + expect((await POST(makeRequest({}))).status).toBe(400); + expect((await POST(makeRequest({ content: "" }))).status).toBe(400); + }); + + it("rejects an unknown language rather than silently scanning as something else", async () => { + const res = await POST(makeRequest({ content: "x", language: "cobol" })); + expect(res.status).toBe(400); + expect((await res.json()).error).toMatch(/unknown language/); + }); + + it("bounds the input", async () => { + // Unbounded input on an unauthenticated endpoint that runs a regex per + // line is a denial of service, not a feature. + const tooBig = await POST(makeRequest({ content: "a".repeat(256 * 1024 + 1) })); + expect(tooBig.status).toBe(413); + + const tooManyLines = await POST(makeRequest({ content: "x\n".repeat(20_001) })); + expect(tooManyLines.status).toBe(413); + }); + + it("strips directories from the filename", async () => { + // It is never opened, but it is echoed back, and a caller should not be + // able to put arbitrary paths into a response. + const res = await POST( + makeRequest({ filename: "../../etc/passwd.sh", content: "echo hi\n" }), + ); + expect((await res.json()).filename).toBe("passwd.sh"); + }); +}); diff --git a/apps/web/src/app/api/scan/code/route.ts b/apps/web/src/app/api/scan/code/route.ts new file mode 100644 index 0000000..defa5b5 --- /dev/null +++ b/apps/web/src/app/api/scan/code/route.ts @@ -0,0 +1,106 @@ +import { NextRequest, NextResponse } from "next/server"; +import { languageOf, scanText, type ScanLanguage } from "@threatcrush/scan"; + +/** + * The same rules the CLI runs, over submitted text. + * + * This route exists because `@threatcrush/scan` exists. Before the engine was + * extracted from `apps/cli`, the only way to run these rules was to install the + * CLI; the web app could not have offered this without a second copy of the + * rule set, which for a rule set whose value is careful false-positive tuning + * is worse than not offering it at all. + * + * Imports the default entry point, not `@threatcrush/scan/node` — nothing here + * touches a filesystem, so there is no tree walker and no SARIF writer in this + * bundle. + */ + +/** + * Cap on submitted content. + * + * Every rule is a regular expression run per line, and the rule set includes + * `redos-nested-quantifier` precisely because catastrophic backtracking is + * real. An unbounded body on an unauthenticated endpoint is the shape of a + * cheap denial of service, so the input is bounded before any rule sees it. + */ +const MAX_BYTES = 256 * 1024; +const MAX_LINES = 20_000; + +const LANGUAGES: readonly ScanLanguage[] = [ + "javascript", + "typescript", + "python", + "ruby", + "go", + "java", + "php", + "shell", + "config", + "other", +]; + +/** + * POST /api/scan/code + * Free code scanner — no auth required, matching /api/scan. + */ +export async function POST(request: NextRequest) { + let body: { content?: string; filename?: string; language?: string }; + try { + body = await request.json(); + } catch { + return NextResponse.json({ error: "Invalid JSON" }, { status: 400 }); + } + + const { content, filename, language } = body; + + if (typeof content !== "string" || content.length === 0) { + return NextResponse.json({ error: "content is required" }, { status: 400 }); + } + + // Byte length, not string length: the limit is about how much work this + // costs to serve, and a multi-byte character is not one byte. + if (Buffer.byteLength(content, "utf8") > MAX_BYTES) { + return NextResponse.json( + { error: `content exceeds ${MAX_BYTES} bytes` }, + { status: 413 }, + ); + } + + const lineCount = content.split("\n").length; + if (lineCount > MAX_LINES) { + return NextResponse.json( + { error: `content exceeds ${MAX_LINES} lines` }, + { status: 413 }, + ); + } + + if (language !== undefined && !LANGUAGES.includes(language as ScanLanguage)) { + return NextResponse.json( + { error: `unknown language: ${language} (expected ${LANGUAGES.join(", ")})` }, + { status: 400 }, + ); + } + + // The filename is used for language detection and echoed back on each + // finding. It is never opened — this route has no filesystem access — so a + // traversal sequence in it reaches nothing. It is still bounded and stripped + // of directories, because a caller should not be able to put arbitrary text + // into a response that another user may see. + const safeName = (filename ?? "snippet.txt").split("/").pop()?.slice(0, 128) || "snippet.txt"; + const resolved = (language as ScanLanguage | undefined) ?? languageOf(safeName); + + const findings = scanText(safeName, content, resolved); + + const summary = { critical: 0, high: 0, medium: 0, low: 0, info: 0 }; + for (const finding of findings) summary[finding.severity] += 1; + + return NextResponse.json({ + filename: safeName, + language: resolved, + lines_scanned: lineCount, + // Excerpts are redacted by the engine before they reach here, so echoing a + // finding never returns the credential that produced it. + findings, + summary, + }); +} diff --git a/packages/scan/README.md b/packages/scan/README.md index 56cf685..230114f 100644 --- a/packages/scan/README.md +++ b/packages/scan/README.md @@ -45,12 +45,24 @@ Consumers therefore transpile it themselves: - **CLI** — bundled by tsup via `noExternal`, so the published package stays self-contained and gains no dependency on an unpublished package. -- **Next.js** (web) — add `transpilePackages: ['@threatcrush/scan']`. -- **Vite** (desktop, extension) — works as-is; Vite transpiles linked workspace - sources by default. +- **Next.js** (web) — `transpilePackages: ['@threatcrush/scan']`. +- **Vite** — works as-is; Vite transpiles linked workspace sources by default. -If this package is ever published standalone, add a build step and switch -`exports` to `dist` with a `publishConfig` override. Nothing else needs to move. +### Internal imports carry no file extension + +`import { … } from './types'`, not `'./types.js'`. + +Both tsconfigs here use `moduleResolution: "bundler"`, which makes that valid, +and every consumer resolves it. The `.js` form does not survive contact with +Turbopack: Next.js 16 builds with it by default, it does not rewrite `.js` to +`.ts`, and it offers no `extensionAlias` escape hatch — so every internal +import in this package resolved to nothing and the web build failed outright. +A webpack `extensionAlias` fixes the same problem but is simply ignored under +Turbopack. + +If this package is ever published standalone for Node consumers, add a build +step, put the extensions back in the emitted output, and switch `exports` to +`dist` behind a `publishConfig` override. ## Adding a rule diff --git a/packages/scan/src/__tests__/code-rules.test.ts b/packages/scan/src/__tests__/code-rules.test.ts index 316421e..745c4c9 100644 --- a/packages/scan/src/__tests__/code-rules.test.ts +++ b/packages/scan/src/__tests__/code-rules.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest'; -import { CODE_RULES, proseLines } from '../code-rules.js'; -import { languageOf, scanText } from '../text.js'; -import type { ScanLanguage } from '../types.js'; +import { CODE_RULES, proseLines } from '../code-rules'; +import { languageOf, scanText } from '../text'; +import type { ScanLanguage } from '../types'; /** * Every case here is a pair: the vulnerable shape and the *corrected* shape diff --git a/packages/scan/src/__tests__/sarif.test.ts b/packages/scan/src/__tests__/sarif.test.ts index d68bf5b..7b7465b 100644 --- a/packages/scan/src/__tests__/sarif.test.ts +++ b/packages/scan/src/__tests__/sarif.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { buildSarif, fingerprintOf, sarifLevel, securitySeverity, toArtifactUri } from '../node/sarif.js'; -import type { ScanFinding } from '../types.js'; +import { buildSarif, fingerprintOf, sarifLevel, securitySeverity, toArtifactUri } from '../node/sarif'; +import type { ScanFinding } from '../types'; const finding = (overrides: Partial = {}): ScanFinding => ({ ruleId: 'secret-aws-access-key', diff --git a/packages/scan/src/__tests__/text.test.ts b/packages/scan/src/__tests__/text.test.ts index fecb127..9924e4e 100644 --- a/packages/scan/src/__tests__/text.test.ts +++ b/packages/scan/src/__tests__/text.test.ts @@ -5,10 +5,10 @@ import { languageOfShebang, meetsFailThreshold, scanText, -} from '../text.js'; -import { detectTyposquat, editDistance, scanPackageJson, scanRequirementsTxt } from '../manifest-rules.js'; -import { isKnownPlaceholder, redactSecret } from '../secret-rules.js'; -import type { ScanFinding } from '../types.js'; +} from '../text'; +import { detectTyposquat, editDistance, scanPackageJson, scanRequirementsTxt } from '../manifest-rules'; +import { isKnownPlaceholder, redactSecret } from '../secret-rules'; +import type { ScanFinding } from '../types'; describe('language detection', () => { it('maps extensions and treats dotted env files as config', () => { diff --git a/packages/scan/src/code-rules.ts b/packages/scan/src/code-rules.ts index 021fb8f..af7958f 100644 --- a/packages/scan/src/code-rules.ts +++ b/packages/scan/src/code-rules.ts @@ -37,8 +37,8 @@ * rule that would flag every session read in the codebase. See KNOWN_GAPS. */ -import type { Confidence, ScanLanguage, Severity } from './types.js'; -import { severityFor } from './types.js'; +import type { Confidence, ScanLanguage, Severity } from './types'; +import { severityFor } from './types'; export interface CodeRule { id: string; diff --git a/packages/scan/src/index.ts b/packages/scan/src/index.ts index a729ac4..a74a538 100644 --- a/packages/scan/src/index.ts +++ b/packages/scan/src/index.ts @@ -11,13 +11,13 @@ * writing SARIF — lives behind `@threatcrush/scan/node`. */ -export { CODE_RULES, evaluateRule, GENERIC_GUARD, proseLines, untrustedPatternFor } from './code-rules.js'; -export type { CodeRule } from './code-rules.js'; +export { CODE_RULES, evaluateRule, GENERIC_GUARD, proseLines, untrustedPatternFor } from './code-rules'; +export type { CodeRule } from './code-rules'; -export { scanPackageJson, scanRequirementsTxt, detectTyposquat, editDistance } from './manifest-rules.js'; -export type { ManifestFinding, SquatVerdict } from './manifest-rules.js'; +export { scanPackageJson, scanRequirementsTxt, detectTyposquat, editDistance } from './manifest-rules'; +export type { ManifestFinding, SquatVerdict } from './manifest-rules'; -export { isKnownPlaceholder, redactSecret, SECRET_RULES, SENSITIVE_FILES } from './secret-rules.js'; +export { isKnownPlaceholder, redactSecret, SECRET_RULES, SENSITIVE_FILES } from './secret-rules'; export { collectSuppressions, @@ -30,8 +30,8 @@ export { scanManifest, scanText, SKIP_DIRS, -} from './text.js'; -export type { Suppressions } from './text.js'; +} from './text'; +export type { Suppressions } from './text'; -export { severityRank, SEVERITY_ORDER } from './types.js'; -export type { Confidence, ScanFinding, ScanLanguage, Severity } from './types.js'; +export { severityRank, SEVERITY_ORDER } from './types'; +export type { Confidence, ScanFinding, ScanLanguage, Severity } from './types'; diff --git a/packages/scan/src/manifest-rules.ts b/packages/scan/src/manifest-rules.ts index ae7445e..10b12bd 100644 --- a/packages/scan/src/manifest-rules.ts +++ b/packages/scan/src/manifest-rules.ts @@ -10,7 +10,7 @@ * privileges. Both are visible in the manifest, before anything is fetched. */ -import type { Severity } from './types.js'; +import type { Severity } from './types'; export interface ManifestFinding { ruleId: string; diff --git a/packages/scan/src/node/dependencies.ts b/packages/scan/src/node/dependencies.ts index 0ad16db..a4e1f94 100644 --- a/packages/scan/src/node/dependencies.ts +++ b/packages/scan/src/node/dependencies.ts @@ -10,7 +10,7 @@ import { existsSync, readFileSync } from 'node:fs'; import { join } from 'node:path'; -import type { ScanFinding, Severity } from '../types.js'; +import type { ScanFinding, Severity } from '../types'; interface OsvVulnerability { id: string; diff --git a/packages/scan/src/node/index.ts b/packages/scan/src/node/index.ts index f3e902f..94b1cd2 100644 --- a/packages/scan/src/node/index.ts +++ b/packages/scan/src/node/index.ts @@ -6,10 +6,10 @@ * filesystem exists: the CLI, the daemon, a server route. */ -export { scanPath } from './walk.js'; -export type { ScanOptions, ScanReport } from './walk.js'; +export { scanPath } from './walk'; +export type { ScanOptions, ScanReport } from './walk'; -export { scanDependencies } from './dependencies.js'; +export { scanDependencies } from './dependencies'; -export { buildSarif, fingerprintOf, sarifLevel, securitySeverity, toArtifactUri } from './sarif.js'; -export type { SarifOptions } from './sarif.js'; +export { buildSarif, fingerprintOf, sarifLevel, securitySeverity, toArtifactUri } from './sarif'; +export type { SarifOptions } from './sarif'; diff --git a/packages/scan/src/node/sarif.ts b/packages/scan/src/node/sarif.ts index 27777fc..f3078ff 100644 --- a/packages/scan/src/node/sarif.ts +++ b/packages/scan/src/node/sarif.ts @@ -26,7 +26,7 @@ import { createHash } from 'node:crypto'; import { isAbsolute, relative, resolve, sep } from 'node:path'; -import type { ScanFinding, Severity } from '../types.js'; +import type { ScanFinding, Severity } from '../types'; /** * The key our fingerprint is published under. diff --git a/packages/scan/src/node/walk.ts b/packages/scan/src/node/walk.ts index f9a0c82..1055872 100644 --- a/packages/scan/src/node/walk.ts +++ b/packages/scan/src/node/walk.ts @@ -12,7 +12,7 @@ import { closeSync, fstatSync, openSync, readdirSync, readFileSync, readSync, statSync, } from 'node:fs'; import { basename, dirname, extname, join, relative, sep } from 'node:path'; -import { SENSITIVE_FILES } from '../secret-rules.js'; +import { SENSITIVE_FILES } from '../secret-rules'; import { collectSuppressions, languageOf, @@ -21,9 +21,9 @@ import { scanManifest, scanText, SKIP_DIRS, -} from '../text.js'; -import type { ScanFinding, ScanLanguage } from '../types.js'; -import { severityRank } from '../types.js'; +} from '../text'; +import type { ScanFinding, ScanLanguage } from '../types'; +import { severityRank } from '../types'; export interface ScanOptions { /** Skip files larger than this. Defaults to 1 MiB. */ diff --git a/packages/scan/src/secret-rules.ts b/packages/scan/src/secret-rules.ts index ebe51e8..d5482f4 100644 --- a/packages/scan/src/secret-rules.ts +++ b/packages/scan/src/secret-rules.ts @@ -22,7 +22,7 @@ * flag both. */ -import type { Severity } from './types.js'; +import type { Severity } from './types'; export interface SecretRule { id: string; diff --git a/packages/scan/src/text.ts b/packages/scan/src/text.ts index 9ce965b..4caff73 100644 --- a/packages/scan/src/text.ts +++ b/packages/scan/src/text.ts @@ -14,11 +14,11 @@ * and everything here works on strings that somebody else read. */ -import { CODE_RULES, evaluateRule, proseLines } from './code-rules.js'; -import { scanPackageJson, scanRequirementsTxt } from './manifest-rules.js'; -import { isKnownPlaceholder, redactSecret, SECRET_RULES, SENSITIVE_FILES } from './secret-rules.js'; -import type { ScanFinding, ScanLanguage, Severity } from './types.js'; -import { severityRank } from './types.js'; +import { CODE_RULES, evaluateRule, proseLines } from './code-rules'; +import { scanPackageJson, scanRequirementsTxt } from './manifest-rules'; +import { isKnownPlaceholder, redactSecret, SECRET_RULES, SENSITIVE_FILES } from './secret-rules'; +import type { ScanFinding, ScanLanguage, Severity } from './types'; +import { severityRank } from './types'; /** * `extname` and `basename`, reimplemented in three lines each. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8e5cf54..37f8418 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -355,6 +355,9 @@ importers: '@supabase/supabase-js': specifier: ^2.101.1 version: 2.101.1 + '@threatcrush/scan': + specifier: workspace:* + version: link:../../packages/scan isomorphic-dompurify: specifier: ^3.12.0 version: 3.12.0