feat(scan): path exclusion via --exclude and .threatcrushignore (v0.8.0)

.threatcrushignore

@@ -0,0 +1,19 @@
+# ThreatCrush does not scan its own rule definitions or test fixtures.
+#
+# A scanner's detection patterns look like the vulnerable code they match, and
+# its test corpus is deliberately-vulnerable sample code. Scanning either
+# measures nothing and buries real findings in the product under dozens of
+# self-references — the tool detecting itself. Everything else in the repo is
+# still scanned.
+#
+# Globs are gitignore-flavoured: a bare name matches at any depth; a pattern
+# with a slash is anchored to the repo root. See @threatcrush/scan.
+
+# Test fixtures across the monorepo.
+__tests__
+
+# The rule definitions themselves.
+packages/scan/src/code-rules.ts
+packages/scan/src/secret-rules.ts
+packages/scan/src/manifest-rules.ts
+modules/code-scanner/src/**/rules.ts

apps/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@profullstack/threatcrush",
-  "version": "0.7.2",
+  "version": "0.8.0",
   "description": "All-in-one security agent daemon — monitor, detect, scan, and protect servers in real-time",
   "bin": {
     "threatcrush": "./dist/index.js"

apps/cli/src/commands/scan.ts

@@ -34,6 +34,8 @@ export interface ScanCommandOptions {
    * discover the dependency mid-run.
    */
   dependencies?: boolean;
+  /** Globs to skip, merged with any `.threatcrushignore` at the scan root. */
+  exclude?: readonly string[];
 }
 
 interface ScanOutcome {
@@ -42,6 +44,7 @@ interface ScanOutcome {
   filesScanned: number;
   unreadable: string[];
   suppressed: number;
+  excluded: number;
   root: string;
 }
 
@@ -171,6 +174,7 @@ export async function scanCommand(
   try {
     let seen = 0;
     const report = scanPath(targetPath, {
+      exclude: options.exclude,
       onFile: () => {
         seen += 1;
         if (spinner) spinner.text = `Scanning files... (${seen} files)`;
@@ -186,6 +190,7 @@ export async function scanCommand(
       filesScanned: report.filesScanned,
       unreadable: report.unreadable,
       suppressed: report.suppressed,
+      excluded: report.excluded,
       root: report.root,
     };
   } catch (err) {
@@ -209,6 +214,14 @@ export async function scanCommand(
     }
   }
 
+  if (outcome.excluded > 0) {
+    say(
+      chalk.gray(
+        `  · ${outcome.excluded} path(s) excluded by --exclude or .threatcrushignore`,
+      ),
+    );
+  }
+
   if (outcome.suppressed > 0) {
     say(
       chalk.gray(

apps/cli/src/index.ts

@@ -254,13 +254,20 @@ program
     "prepend this to SARIF file URIs — use when the scan root is not the repository root",
   )
   .option("--deps", "also query OSV.dev for advisories against lockfile versions (network)")
+  .option(
+    "--exclude <glob>",
+    "skip paths matching this glob (repeatable); merged with a .threatcrushignore at the scan root",
+    (value: string, previous: string[]) => [...previous, value],
+    [] as string[],
+  )
   .option("-v, --verbose", "list the paths that could not be read")
   .action(async (targetPath: string, opts: {
     format?: string;
     output?: string;
     failOn?: string;
     pathPrefix?: string;
     deps?: boolean;
+    exclude?: string[];
     verbose?: boolean;
   }) => {
     const format = (opts.format ?? "text").toLowerCase();
@@ -283,6 +290,7 @@ program
       failOn,
       pathPrefix: opts.pathPrefix,
       dependencies: opts.deps,
+      exclude: opts.exclude,
       verbose: opts.verbose,
     });
   });

packages/scan/README.md

@@ -64,6 +64,27 @@ If this package is ever published standalone for Node consumers, add a build
 step, put the extensions back in the emitted output, and switch `exports` to
 `dist` behind a `publishConfig` override.
 
+## Excluding paths
+
+`scanPath` (and the CLI's `threatcrush scan`) skips paths matching an exclusion
+glob, from either `--exclude <glob>` (repeatable) or a `.threatcrushignore`
+file at the scan root — the two are merged. Globs are gitignore-flavoured: a
+bare name (`__tests__`) matches at any depth, a pattern with a slash is
+anchored to the root, `*` stays within a segment and `**` crosses them.
+
+```
+# .threatcrushignore
+__tests__
+vendor
+dist/**
+```
+
+Excluding is not the same as finding nothing: `ScanReport.excluded` counts the
+skipped paths and the CLI prints it, so a scan quieted by a broad glob is not
+mistaken for a clean one. This repository ships a `.threatcrushignore` that
+excludes the scanner's own rule definitions and fixtures — vulnerable-looking
+by design — from its self-scan.
+
 ## Adding a rule
 
 Rules live in `src/code-rules.ts`, credentials in `src/secret-rules.ts`,

packages/scan/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@threatcrush/scan",
-  "version": "0.7.2",
+  "version": "0.8.0",
   "description": "ThreatCrush scan rules and engine, shared by the CLI, web, desktop and extension.",
   "license": "MIT",
   "type": "module",

packages/scan/src/__tests__/exclude.test.ts

@@ -0,0 +1,96 @@
+import { describe, expect, it } from 'vitest';
+import { compileExcludes } from '../node/walk';
+
+/**
+ * The exclusion matcher decides which paths a scan skips, so a bug here either
+ * hides real findings (over-matching) or defeats the feature (under-matching).
+ * Both directions are pinned.
+ */
+describe('exclude globs', () => {
+  const matches = (patterns: string[], path: string) => compileExcludes(patterns)(path);
+
+  it('matches a bare name at any depth', () => {
+    const p = ['__tests__'];
+    expect(matches(p, 'packages/scan/src/__tests__/code-rules.test.ts')).toBe(true);
+    expect(matches(p, '__tests__/a.ts')).toBe(true);
+    expect(matches(p, 'src/__tests__')).toBe(true);
+    // Not a substring: `__tests__helper` is a different name.
+    expect(matches(p, 'src/__tests__helper/a.ts')).toBe(false);
+  });
+
+  it('anchors a pattern that contains a slash to the scan root', () => {
+    const p = ['packages/scan/src/code-rules.ts'];
+    expect(matches(p, 'packages/scan/src/code-rules.ts')).toBe(true);
+    // Same basename elsewhere is not excluded — the slash anchored it.
+    expect(matches(p, 'other/code-rules.ts')).toBe(false);
+  });
+
+  it('treats * as within-segment and ** as across-segments', () => {
+    expect(matches(['*.test.ts'], 'packages/scan/src/__tests__/x.test.ts')).toBe(true);
+    expect(matches(['*.test.ts'], 'x.spec.ts')).toBe(false);
+    // `*` does not cross a slash.
+    expect(matches(['packages/*/index.ts'], 'packages/scan/index.ts')).toBe(true);
+    expect(matches(['packages/*/index.ts'], 'packages/scan/src/index.ts')).toBe(false);
+    // `**` does.
+    expect(matches(['modules/**/rules.ts'], 'modules/code-scanner/src/secrets/rules.ts')).toBe(true);
+    expect(matches(['modules/**/rules.ts'], 'modules/rules.ts')).toBe(true);
+  });
+
+  it('prunes a whole subtree when the directory itself matches', () => {
+    const p = ['packages/scan/src/__tests__'];
+    expect(matches(p, 'packages/scan/src/__tests__')).toBe(true);
+    expect(matches(p, 'packages/scan/src/__tests__/deep/a.ts')).toBe(true);
+    expect(matches(p, 'packages/scan/src/text.ts')).toBe(false);
+  });
+
+  it('ignores blank lines and comments, so a .threatcrushignore passes straight in', () => {
+    const file = ['# scanner self-reference', '', '  __tests__  ', '#another'];
+    expect(matches(file, 'a/__tests__/b.ts')).toBe(true);
+    expect(matches(file, 'a/b.ts')).toBe(false);
+  });
+
+  it('matches nothing when there are no real patterns', () => {
+    const none = compileExcludes(['', '  ', '# just a comment']);
+    expect(none('anything/at/all.ts')).toBe(false);
+  });
+
+  it('does not let a metacharacter in the pattern match literally', () => {
+    // `.` in the pattern is a literal dot, not "any char".
+    expect(matches(['config.ts'], 'configXts')).toBe(false);
+    expect(matches(['config.ts'], 'config.ts')).toBe(true);
+  });
+
+  it('matches a trailing ** against everything beneath, including direct files', () => {
+    const p = ['dist/**'];
+    expect(matches(p, 'dist/index.js')).toBe(true);
+    expect(matches(p, 'dist/a/b/c.js')).toBe(true);
+    // `dist/**` is the contents of dist, not dist itself.
+    expect(matches(p, 'dist')).toBe(false);
+    expect(matches(p, 'src/dist/x.js')).toBe(false);
+  });
+
+  it('compiles a pathological pattern in linear time', () => {
+    // The pattern is library input too. An all-slashes value once hit a
+    // quadratic trailing-slash trim; a long run of `*` once compiled to
+    // adjacent `[^/]*[^/]*`. Both must be linear.
+    const start = Date.now();
+    const a = compileExcludes(['/'.repeat(50_000)]);
+    const b = compileExcludes([`${'*'.repeat(5_000)}.ts`]);
+    expect(a('some/path.ts')).toBe(false);
+    expect(b('x/verylongname.ts')).toBe(true);
+    expect(Date.now() - start).toBeLessThan(1000);
+  });
+
+  it('runs in linear time on a path with many slashes', () => {
+    // The whole point of the segment-anchored encoding: a pathological input —
+    // thousands of slashes that never satisfy the pattern — must not backtrack.
+    // The naive `.*/` form took quadratic time here and CodeQL flagged it.
+    const isExcluded = compileExcludes(['__tests__', 'packages/scan/**', 'a/**/b.ts']);
+    const hostile = `${'/'.repeat(50_000)}x`;
+    const start = Date.now();
+    expect(isExcluded(hostile)).toBe(false);
+    // A quadratic matcher blows past this by orders of magnitude; a linear one
+    // finishes in single-digit milliseconds.
+    expect(Date.now() - start).toBeLessThan(1000);
+  });
+});