diff --git a/.threatcrushignore b/.threatcrushignore new file mode 100644 index 0000000..dfa30a7 --- /dev/null +++ b/.threatcrushignore @@ -0,0 +1,19 @@ +# ThreatCrush does not scan its own rule definitions or test fixtures. +# +# A scanner's detection patterns look like the vulnerable code they match, and +# its test corpus is deliberately-vulnerable sample code. Scanning either +# measures nothing and buries real findings in the product under dozens of +# self-references — the tool detecting itself. Everything else in the repo is +# still scanned. +# +# Globs are gitignore-flavoured: a bare name matches at any depth; a pattern +# with a slash is anchored to the repo root. See @threatcrush/scan. + +# Test fixtures across the monorepo. +__tests__ + +# The rule definitions themselves. +packages/scan/src/code-rules.ts +packages/scan/src/secret-rules.ts +packages/scan/src/manifest-rules.ts +modules/code-scanner/src/**/rules.ts diff --git a/apps/cli/package.json b/apps/cli/package.json index e32f243..229acd5 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -1,6 +1,6 @@ { "name": "@profullstack/threatcrush", - "version": "0.7.2", + "version": "0.8.0", "description": "All-in-one security agent daemon — monitor, detect, scan, and protect servers in real-time", "bin": { "threatcrush": "./dist/index.js" diff --git a/apps/cli/src/commands/scan.ts b/apps/cli/src/commands/scan.ts index 6128208..2bb84e1 100644 --- a/apps/cli/src/commands/scan.ts +++ b/apps/cli/src/commands/scan.ts @@ -34,6 +34,8 @@ export interface ScanCommandOptions { * discover the dependency mid-run. */ dependencies?: boolean; + /** Globs to skip, merged with any `.threatcrushignore` at the scan root. */ + exclude?: readonly string[]; } interface ScanOutcome { @@ -42,6 +44,7 @@ interface ScanOutcome { filesScanned: number; unreadable: string[]; suppressed: number; + excluded: number; root: string; } @@ -171,6 +174,7 @@ export async function scanCommand( try { let seen = 0; const report = scanPath(targetPath, { + exclude: options.exclude, onFile: () => { seen += 1; if (spinner) spinner.text = `Scanning files... (${seen} files)`; @@ -186,6 +190,7 @@ export async function scanCommand( filesScanned: report.filesScanned, unreadable: report.unreadable, suppressed: report.suppressed, + excluded: report.excluded, root: report.root, }; } catch (err) { @@ -209,6 +214,14 @@ export async function scanCommand( } } + if (outcome.excluded > 0) { + say( + chalk.gray( + ` · ${outcome.excluded} path(s) excluded by --exclude or .threatcrushignore`, + ), + ); + } + if (outcome.suppressed > 0) { say( chalk.gray( diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts index dc64921..1e9299e 100644 --- a/apps/cli/src/index.ts +++ b/apps/cli/src/index.ts @@ -254,6 +254,12 @@ program "prepend this to SARIF file URIs — use when the scan root is not the repository root", ) .option("--deps", "also query OSV.dev for advisories against lockfile versions (network)") + .option( + "--exclude ", + "skip paths matching this glob (repeatable); merged with a .threatcrushignore at the scan root", + (value: string, previous: string[]) => [...previous, value], + [] as string[], + ) .option("-v, --verbose", "list the paths that could not be read") .action(async (targetPath: string, opts: { format?: string; @@ -261,6 +267,7 @@ program failOn?: string; pathPrefix?: string; deps?: boolean; + exclude?: string[]; verbose?: boolean; }) => { const format = (opts.format ?? "text").toLowerCase(); @@ -283,6 +290,7 @@ program failOn, pathPrefix: opts.pathPrefix, dependencies: opts.deps, + exclude: opts.exclude, verbose: opts.verbose, }); }); diff --git a/packages/scan/README.md b/packages/scan/README.md index 230114f..1637cac 100644 --- a/packages/scan/README.md +++ b/packages/scan/README.md @@ -64,6 +64,27 @@ If this package is ever published standalone for Node consumers, add a build step, put the extensions back in the emitted output, and switch `exports` to `dist` behind a `publishConfig` override. +## Excluding paths + +`scanPath` (and the CLI's `threatcrush scan`) skips paths matching an exclusion +glob, from either `--exclude ` (repeatable) or a `.threatcrushignore` +file at the scan root — the two are merged. Globs are gitignore-flavoured: a +bare name (`__tests__`) matches at any depth, a pattern with a slash is +anchored to the root, `*` stays within a segment and `**` crosses them. + +``` +# .threatcrushignore +__tests__ +vendor +dist/** +``` + +Excluding is not the same as finding nothing: `ScanReport.excluded` counts the +skipped paths and the CLI prints it, so a scan quieted by a broad glob is not +mistaken for a clean one. This repository ships a `.threatcrushignore` that +excludes the scanner's own rule definitions and fixtures — vulnerable-looking +by design — from its self-scan. + ## Adding a rule Rules live in `src/code-rules.ts`, credentials in `src/secret-rules.ts`, diff --git a/packages/scan/package.json b/packages/scan/package.json index 4645517..49ae944 100644 --- a/packages/scan/package.json +++ b/packages/scan/package.json @@ -1,6 +1,6 @@ { "name": "@threatcrush/scan", - "version": "0.7.2", + "version": "0.8.0", "description": "ThreatCrush scan rules and engine, shared by the CLI, web, desktop and extension.", "license": "MIT", "type": "module", diff --git a/packages/scan/src/__tests__/exclude.test.ts b/packages/scan/src/__tests__/exclude.test.ts new file mode 100644 index 0000000..1a03a83 --- /dev/null +++ b/packages/scan/src/__tests__/exclude.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, it } from 'vitest'; +import { compileExcludes } from '../node/walk'; + +/** + * The exclusion matcher decides which paths a scan skips, so a bug here either + * hides real findings (over-matching) or defeats the feature (under-matching). + * Both directions are pinned. + */ +describe('exclude globs', () => { + const matches = (patterns: string[], path: string) => compileExcludes(patterns)(path); + + it('matches a bare name at any depth', () => { + const p = ['__tests__']; + expect(matches(p, 'packages/scan/src/__tests__/code-rules.test.ts')).toBe(true); + expect(matches(p, '__tests__/a.ts')).toBe(true); + expect(matches(p, 'src/__tests__')).toBe(true); + // Not a substring: `__tests__helper` is a different name. + expect(matches(p, 'src/__tests__helper/a.ts')).toBe(false); + }); + + it('anchors a pattern that contains a slash to the scan root', () => { + const p = ['packages/scan/src/code-rules.ts']; + expect(matches(p, 'packages/scan/src/code-rules.ts')).toBe(true); + // Same basename elsewhere is not excluded — the slash anchored it. + expect(matches(p, 'other/code-rules.ts')).toBe(false); + }); + + it('treats * as within-segment and ** as across-segments', () => { + expect(matches(['*.test.ts'], 'packages/scan/src/__tests__/x.test.ts')).toBe(true); + expect(matches(['*.test.ts'], 'x.spec.ts')).toBe(false); + // `*` does not cross a slash. + expect(matches(['packages/*/index.ts'], 'packages/scan/index.ts')).toBe(true); + expect(matches(['packages/*/index.ts'], 'packages/scan/src/index.ts')).toBe(false); + // `**` does. + expect(matches(['modules/**/rules.ts'], 'modules/code-scanner/src/secrets/rules.ts')).toBe(true); + expect(matches(['modules/**/rules.ts'], 'modules/rules.ts')).toBe(true); + }); + + it('prunes a whole subtree when the directory itself matches', () => { + const p = ['packages/scan/src/__tests__']; + expect(matches(p, 'packages/scan/src/__tests__')).toBe(true); + expect(matches(p, 'packages/scan/src/__tests__/deep/a.ts')).toBe(true); + expect(matches(p, 'packages/scan/src/text.ts')).toBe(false); + }); + + it('ignores blank lines and comments, so a .threatcrushignore passes straight in', () => { + const file = ['# scanner self-reference', '', ' __tests__ ', '#another']; + expect(matches(file, 'a/__tests__/b.ts')).toBe(true); + expect(matches(file, 'a/b.ts')).toBe(false); + }); + + it('matches nothing when there are no real patterns', () => { + const none = compileExcludes(['', ' ', '# just a comment']); + expect(none('anything/at/all.ts')).toBe(false); + }); + + it('does not let a metacharacter in the pattern match literally', () => { + // `.` in the pattern is a literal dot, not "any char". + expect(matches(['config.ts'], 'configXts')).toBe(false); + expect(matches(['config.ts'], 'config.ts')).toBe(true); + }); + + it('matches a trailing ** against everything beneath, including direct files', () => { + const p = ['dist/**']; + expect(matches(p, 'dist/index.js')).toBe(true); + expect(matches(p, 'dist/a/b/c.js')).toBe(true); + // `dist/**` is the contents of dist, not dist itself. + expect(matches(p, 'dist')).toBe(false); + expect(matches(p, 'src/dist/x.js')).toBe(false); + }); + + it('compiles a pathological pattern in linear time', () => { + // The pattern is library input too. An all-slashes value once hit a + // quadratic trailing-slash trim; a long run of `*` once compiled to + // adjacent `[^/]*[^/]*`. Both must be linear. + const start = Date.now(); + const a = compileExcludes(['/'.repeat(50_000)]); + const b = compileExcludes([`${'*'.repeat(5_000)}.ts`]); + expect(a('some/path.ts')).toBe(false); + expect(b('x/verylongname.ts')).toBe(true); + expect(Date.now() - start).toBeLessThan(1000); + }); + + it('runs in linear time on a path with many slashes', () => { + // The whole point of the segment-anchored encoding: a pathological input — + // thousands of slashes that never satisfy the pattern — must not backtrack. + // The naive `.*/` form took quadratic time here and CodeQL flagged it. + const isExcluded = compileExcludes(['__tests__', 'packages/scan/**', 'a/**/b.ts']); + const hostile = `${'/'.repeat(50_000)}x`; + const start = Date.now(); + expect(isExcluded(hostile)).toBe(false); + // A quadratic matcher blows past this by orders of magnitude; a linear one + // finishes in single-digit milliseconds. + expect(Date.now() - start).toBeLessThan(1000); + }); +}); diff --git a/packages/scan/src/node/walk.ts b/packages/scan/src/node/walk.ts index 1055872..2a9fe16 100644 --- a/packages/scan/src/node/walk.ts +++ b/packages/scan/src/node/walk.ts @@ -32,6 +32,151 @@ export interface ScanOptions { onFile?: (path: string) => void; /** Restrict to these rule categories. Defaults to all. */ categories?: readonly ScanFinding['category'][]; + /** + * Paths to skip, as globs over repository-relative POSIX paths. + * + * Merged with a `.threatcrushignore` file at the scan root, if present. The + * intended use is generated output, vendored trees, and — the case that + * prompted this — a scanner's own rule definitions and test fixtures, which + * are vulnerable-looking by design and otherwise dominate its self-scan. + * + * Excluding is not the same as finding nothing: the count of skipped paths + * is reported (`ScanReport.excluded`) and surfaced, so a scan silenced by a + * broad glob cannot be mistaken for a clean one. + */ + exclude?: readonly string[]; +} + +/** + * Compile exclusion globs into one predicate over relative POSIX paths. + * + * gitignore-flavoured: `*` matches within a path segment, `**` across + * segments, `?` a single non-slash character. A pattern with no `/` matches by + * name at any depth — `__tests__` excludes every directory so named — while a + * pattern containing `/` is anchored to the scan root. Blank lines and `#` + * comments are ignored, so a `.threatcrushignore` file can be passed straight + * in. Exported for its own tests. + */ +export function compileExcludes(patterns: readonly string[]): (relPath: string) => boolean { + const matchers = patterns + .map((p) => p.trim()) + .filter((p) => p.length > 0 && !p.startsWith('#')) + .map(compilePattern); + if (matchers.length === 0) return () => false; + return (relPath) => { + const segs = relPath.split('/'); + return matchers.some((m) => m(segs)); + }; +} + +/** + * Matching is done segment by segment, in code, rather than by compiling the + * glob into one big regex. + * + * A cross-segment regex — `(?:[^/]+/)*` for depth, a dot-star for `**` — is the + * shape that backtracks quadratically on a path full of slashes, the + * polynomial-ReDoS CodeQL flags and this scanner has its own rule for. There is + * no way to feed a whole path through a single generated pattern without that + * risk. Splitting both the pattern and the path on `/` and walking them with a + * two-pointer (the classic `**` alignment) removes it entirely: each per-segment + * regex is a trivial `^…$` with a single quantifier and is tested against one + * bounded segment, never the whole path. + */ +const GLOBSTAR = Symbol('globstar'); +type SegMatcher = RegExp | typeof GLOBSTAR; + +function compilePattern(pattern: string): (segs: readonly string[]) => boolean { + // Trailing slashes are trimmed without a regex: `/\/+$/` is unanchored, so + // `replace` retries at every start position and is quadratic on a value that + // is all slashes — the very polynomial-ReDoS this file is avoiding. + const withoutLead = pattern.replace(/^\.?\//, ''); + let end = withoutLead.length; + while (end > 0 && withoutLead[end - 1] === '/') end -= 1; + const p = withoutLead.slice(0, end); + + // A pattern with no `/` matches by name at any depth: excluded if any single + // segment matches it. + if (!p.includes('/')) { + if (p === '**') return () => true; + const rx = segToRegExp(p); + return (segs) => segs.some((s) => rx.test(s)); + } + + const raw = p.split('/'); + // A trailing `**` means "the contents of", so it must not match the directory + // itself — the path has to be strictly deeper than the leading parts. + const trailingGlobstar = raw[raw.length - 1] === '**'; + const core = trailingGlobstar ? raw.slice(0, -1) : raw; + const parts: SegMatcher[] = core.map((s) => (s === '**' ? GLOBSTAR : segToRegExp(s))); + + return (segs) => { + const end = matchPrefix(parts, segs); + if (end < 0) return false; + // `end` is where the pattern stopped; everything past it is the pruned + // subtree. A prefix match covers the path and all of its descendants; + // `dir/**` additionally requires at least one descendant. + return trailingGlobstar ? end < segs.length : true; + }; +} + +/** `*` within a segment, `?` one character; a segment contains no slash. */ +function segToRegExp(seg: string): RegExp { + let body = ''; + for (let i = 0; i < seg.length; i += 1) { + const c = seg[i]!; + if (c === '*') { + // Collapse a run of `*` into one `[^/]*`; `[^/]*[^/]*` is two adjacent + // stars over the same characters, which backtracks quadratically. + while (seg[i + 1] === '*') i += 1; + body += '[^/]*'; + } else if (c === '?') { + body += '[^/]'; + } else { + body += c.replace(/[.+^${}()|[\]\\]/g, '\\$&'); + } + } + return new RegExp(`^${body}$`); +} + +/** + * Align `parts` against a prefix of `segs`, `GLOBSTAR` consuming zero or more + * segments. Returns the index in `segs` where the match ends, or -1. A + * two-pointer with one backtrack point for the active globstar — O(segments), + * no catastrophic backtracking. + */ +function matchPrefix(parts: readonly SegMatcher[], segs: readonly string[]): number { + let pi = 0; + let si = 0; + let star = -1; + let starSi = -1; + + while (pi < parts.length) { + const part = parts[pi]!; + if (part === GLOBSTAR) { + star = pi; + starSi = si; + pi += 1; + } else if (si < segs.length && part.test(segs[si]!)) { + pi += 1; + si += 1; + } else if (star !== -1 && starSi < segs.length) { + starSi += 1; + si = starSi; + pi = star + 1; + } else { + return -1; + } + } + return si; +} + +/** The globs in a `.threatcrushignore` at `root`, or `[]` if there is none. */ +function readIgnoreFile(root: string): string[] { + try { + return readFileSync(join(root, '.threatcrushignore'), 'utf-8').split('\n'); + } catch { + return []; + } } export interface ScanReport { @@ -56,6 +201,12 @@ export interface ScanReport { * over an unread tree is the failure this scanner exists to avoid. */ unreadable: string[]; + /** + * How many paths an exclusion glob skipped — a pruned directory counts once, + * not per file inside it. Reported for the same reason `suppressed` is: a + * quiet scan produced by a broad `.threatcrushignore` is not a clean scan. + */ + excluded: number; } export function scanPath(targetPath: string, options: ScanOptions = {}): ScanReport { @@ -65,6 +216,7 @@ export function scanPath(targetPath: string, options: ScanOptions = {}): ScanRep const unreadable: string[] = []; let filesScanned = 0; let suppressed = 0; + let excluded = 0; // A file target is not a degenerate directory target. `readdirSync` on a // file throws ENOTDIR, which the walker below treats as an unreadable @@ -79,6 +231,11 @@ export function scanPath(targetPath: string, options: ScanOptions = {}): ScanRep })(); const walkRoot = rootIsDirectory ? targetPath : dirname(targetPath); + // Exclusions come from both the caller and a committed `.threatcrushignore`, + // so a repository can carry its own ignore list without every invocation + // repeating `--exclude`. Compiled once for the whole walk. + const isExcluded = compileExcludes([...(options.exclude ?? []), ...readIgnoreFile(walkRoot)]); + const scanFile = (fullPath: string, filename: string): void => { const relativePath = toRelative(walkRoot, fullPath); const extension = extname(filename).toLowerCase(); @@ -169,13 +326,24 @@ export function scanPath(targetPath: string, options: ScanOptions = {}): ScanRep for (const entry of entries) { const fullPath = join(currentPath, entry.name); + const relativePath = toRelative(walkRoot, fullPath); if (entry.isDirectory()) { if (SKIP_DIRS.has(entry.name)) continue; + // A directory match prunes the whole subtree and counts once, rather + // than descending it to skip each file — the point is not to read it. + if (isExcluded(relativePath)) { + excluded += 1; + continue; + } walk(fullPath); continue; } if (!entry.isFile()) continue; + if (isExcluded(relativePath)) { + excluded += 1; + continue; + } scanFile(fullPath, entry.name); } @@ -183,6 +351,8 @@ export function scanPath(targetPath: string, options: ScanOptions = {}): ScanRep if (rootIsDirectory) { walk(targetPath); + } else if (isExcluded(toRelative(walkRoot, targetPath))) { + excluded += 1; } else { scanFile(targetPath, basename(targetPath)); } @@ -195,7 +365,7 @@ export function scanPath(targetPath: string, options: ScanOptions = {}): ScanRep a.line - b.line, ); - return { findings: filtered, filesScanned, unreadable, suppressed, root: walkRoot }; + return { findings: filtered, filesScanned, unreadable, suppressed, excluded, root: walkRoot }; } /**