Stop re-evaluating modules that can't see a mock (fix unbounded ESM memory growth)

lib/import-graph.js

@@ -0,0 +1,133 @@
+// Keeps `quibble.esm()` from re-evaluating every module on every mock change.
+//
+// Node can't evict anything from its ESM module cache, so the only way to get
+// a fresh instance of a module is to import it under a new URL. Quibble does
+// that by tagging URLs with `?__quibble=<generation>`, where the generation
+// goes up whenever a mock is added or reset. Tagging *every* module that way
+// means each generation re-evaluates, and permanently retains, the whole
+// graph of every module imported while any mock exists - including packages
+// that have nothing to do with the mock.
+//
+// Only a module that is mocked, or that can reach a mocked module through its
+// imports, needs a fresh URL. This learns those imports from what the
+// resolve and load hooks already see (no source parsing, so it behaves the
+// same for TypeScript or anything else a loader transforms):
+//
+//   - recordImport() notes each parent -> child edge the resolve hook sees
+//   - recordLoad() notes the generation in which a module was first loaded
+//   - needsQuibbling() decides whether a resolved URL should be tagged
+//
+// Edges are recorded for every import, including ones made while nothing is
+// mocked and ones whose target doesn't exist on disk (a module can be mocked
+// without existing). Modules loaded earlier count as fully known by the time
+// a mock exists, so a missing edge would make a module look like it depends
+// on nothing, and it would never see a mock added later. For the same reason,
+// loads served as a mock stub are not recorded: a stub has no imports, so
+// counting one would make the real module look fully known with no edges.
+//
+// A module's imports are only fully known once it has finished linking, which
+// we take to mean "it was first loaded in an earlier generation". Until then
+// it is assumed to be affected and gets tagged, exactly as before. So each
+// unaffected module is evaluated about twice (once tagged, once at its plain
+// URL) instead of once per generation.
+//
+// Dynamic `import()` calls need no special handling: they run through the
+// resolve hook when they execute, so they see whichever mocks exist then.
+// `require()` edges inside CommonJS aren't seen by the async hooks, so a
+// CommonJS module is treated as a leaf.
+//
+// Modules are identified by URL with any query string and hash removed, the
+// same way mocks are keyed, so `a.js?v=1` and `a.js?v=2` (separate instances
+// in Node) are one node here and get one tagging decision.
+//
+// One consequence for users: a module that no mock can reach is now a single
+// instance for the life of the process, where it used to get a fresh one
+// after every mock change. Module-level state in such a module (counters,
+// caches, singletons) persists between tests.
+
+const { stripQueryAndHash } = require('./loader-helpers')
+
+/**
+ * Relies on every mock change bumping `state.stubModuleGeneration` (the
+ * decisions are memoised per generation, and "known" means loaded in an
+ * earlier generation), and on reset() replacing `state.quibbledModules`
+ * with a new Map. Those updates live in the message handler in quibble.mjs
+ * (`globalPreload`), in thisWillRunInUserThread.js (same-thread state), and
+ * nowhere else.
+ *
+ * @param {import('./loader-helpers').QuibbleLoaderState} state - read on every
+ *   call, never cached, because reset() replaces `state.quibbledModules`
+ */
+exports.createImportGraph = function createImportGraph (state) {
+  /** @type {Map<string, Set<string>>} plain URL -> plain URLs it imports */
+  const imports = new Map()
+  /** @type {Map<string, number>} plain URL -> generation of its first load */
+  const firstLoadedIn = new Map()
+  let decisionsGeneration = -1
+  /** @type {Map<string, boolean>} memoised per generation, so one URL never gets two answers */
+  let decisions = new Map()
+
+  function recordImport (parentUrl, childUrl) {
+    if (!parentUrl) {
+      return
+    }
+    const parent = stripQueryAndHash(parentUrl)
+    if (!imports.has(parent)) {
+      imports.set(parent, new Set())
+    }
+    imports.get(parent).add(stripQueryAndHash(childUrl))
+  }
+
+  function recordLoad (url) {
+    const plainUrl = stripQueryAndHash(url)
+    if (!firstLoadedIn.has(plainUrl)) {
+      firstLoadedIn.set(plainUrl, state.stubModuleGeneration)
+    }
+  }
+
+  function needsQuibbling (resolvedUrl) {
+    if (!state.quibbledModules.size) {
+      return false
+    }
+
+    const generation = state.stubModuleGeneration
+    if (decisionsGeneration !== generation) {
+      decisionsGeneration = generation
+      decisions = new Map()
+    }
+
+    const url = stripQueryAndHash(resolvedUrl)
+    if (!decisions.has(url)) {
+      decisions.set(url, mayReachMockedModule(url, generation))
+    }
+    return decisions.get(url)
+  }
+
+  function mayReachMockedModule (startUrl, generation) {
+    const seen = new Set([startUrl])
+    const queue = [startUrl]
+    while (queue.length) {
+      const url = queue.shift()
+      if (state.quibbledModules.has(url)) {
+        return true
+      }
+      if (url.startsWith('node:')) {
+        continue
+      }
+
+      const loadedIn = firstLoadedIn.get(url)
+      if (loadedIn === undefined || loadedIn >= generation) {
+        return true // its imports might not all be known yet
+      }
+      for (const child of imports.get(url) ?? []) {
+        if (!seen.has(child)) {
+          seen.add(child)
+          queue.push(child)
+        }
+      }
+    }
+    return false
+  }
+
+  return { recordImport, recordLoad, needsQuibbling }
+}

lib/loader-helpers.js

@@ -24,7 +24,6 @@
  * @returns {[string, ModuleLoaderMockInfo] | undefined}
  */
 function getStubsInfo (state, moduleUrl) {
-  if (!state.quibbledModules) return undefined
   if (!moduleUrl.includes('__quibble=')) return undefined
 
   const moduleKey = stripQueryAndHash(moduleUrl)
@@ -103,7 +102,7 @@ function planResolve (state, specifier, context) {
     return { kind: 'resolveUrl', nextSpecifier: stripMarkers(specifier) }
   }
 
-  if (!state.quibbledModules || specifier === 'quibble' || specifier.includes('__quibbleoriginal')) {
+  if (specifier === 'quibble' || specifier.includes('__quibbleoriginal')) {
     return { kind: 'passthrough', nextSpecifier: stripMarkers(specifier) }
   }
 

lib/quibble-sync-hooks.js

@@ -11,11 +11,14 @@ const {
   stubbedLoadResult,
   stripQueryAndHash
 } = require('./loader-helpers')
+const { createImportGraph } = require('./import-graph')
 
 /**
  * @param {import('./loader-helpers').QuibbleLoaderState} state
  */
 exports.createSyncHooks = function createSyncHooks (state) {
+  const importGraph = createImportGraph(state)
+
   function resolve (specifier, context, nextResolve) {
     // registerHooks() hooks also see CommonJS require() calls, which
     // register()-style hooks never did. Quibble's ESM cache-busting query
@@ -34,20 +37,41 @@ exports.createSyncHooks = function createSyncHooks (state) {
     context.__quibbleSuppressed = true
     try {
       const result = nextResolve(plan.nextSpecifier, context)
+      if (plan.kind === 'quibble') {
+        // Same off-thread-loader caveat as below: work with untagged URLs
+        const url = removeQuibbleQuery(result.url)
+        importGraph.recordImport(context.parentURL && removeQuibbleQuery(context.parentURL), url)
+        if (!importGraph.needsQuibbling(url)) {
+          return { ...result, url }
+        }
+      }
       // An explicit `--loader=quibble` can be registered (off-thread)
       // alongside these hooks and will already have tagged the resolved URL;
       // strip it before quibble re-tags it with its own generation number.
       // The async hooks never run alongside another quibble instance.
       return finishResolve(state, plan, result, removeQuibbleQuery)
     } catch (error) {
-      return recoverResolve(plan, error)
+      const recovered = recoverResolve(plan, error)
+      // The import still happened (the module is just mocked without existing),
+      // so the importer has to be seen as depending on the mock
+      if (plan.kind === 'quibble') {
+        importGraph.recordImport(context.parentURL, recovered.url)
+      }
+      return recovered
     } finally {
       context.__quibbleSuppressed = false
     }
   }
 
   function load (url, context, nextLoad) {
-    return stubbedLoadResult(state, url) ?? nextLoad(stripQueryAndHash(url), context)
+    const stub = stubbedLoadResult(state, url)
+    if (stub) {
+      return stub
+    }
+
+    // Only real loads count (see the async load hook in quibble.mjs)
+    importGraph.recordLoad(url)
+    return nextLoad(stripQueryAndHash(url), context)
   }
 
   return { resolve, load }

lib/quibble.mjs

@@ -1,6 +1,7 @@
 import quibble from './quibble.js'
 import { thisWillRunInUserThread } from './thisWillRunInUserThread.js'
 import helpers from './loader-helpers.js'
+import importGraphModule from './import-graph.js'
 
 const {
   planResolve,
@@ -28,13 +29,15 @@ const quibbleLoaderState = {
   quibbledModules: new Map(),
   stubModuleGeneration: 0
 }
+const importGraph = importGraphModule.createImportGraph(quibbleLoaderState)
 
 // The decision logic - what to do with a given specifier/URL - lives in
 // planResolve()/finishResolve()/recoverResolve() (loader-helpers.js), shared
 // with the sync hooks in quibble-sync-hooks.js. This is just the async
 // driver for it: decide the plan, call nextResolve (awaited, suppressed
 // unless the plan says not to), and let finishResolve/recoverResolve
-// interpret the outcome.
+// interpret the outcome. Modules that can't reach a mock keep their plain URL
+// (see import-graph.js) so they stay in Node's module cache.
 export async function resolve (specifier, context, nextResolve) {
   const plan = planResolve(quibbleLoaderState, specifier, context)
 
@@ -45,9 +48,21 @@ export async function resolve (specifier, context, nextResolve) {
   context.__quibbleSuppressed = true
   try {
     const result = await nextResolve(plan.nextSpecifier, context)
+    if (plan.kind === 'quibble') {
+      importGraph.recordImport(context.parentURL, result.url)
+      if (!importGraph.needsQuibbling(result.url)) {
+        return result
+      }
+    }
     return finishResolve(quibbleLoaderState, plan, result)
   } catch (error) {
-    return recoverResolve(plan, error)
+    const recovered = recoverResolve(plan, error)
+    // The import still happened (the module is just mocked without existing),
+    // so the importer has to be seen as depending on the mock
+    if (plan.kind === 'quibble') {
+      importGraph.recordImport(context.parentURL, recovered.url)
+    }
+    return recovered
   } finally {
     context.__quibbleSuppressed = false
   }
@@ -62,8 +77,15 @@ export async function resolve (specifier, context, nextResolve) {
  * @returns {Promise<{ source: !(string | SharedArrayBuffer | Uint8Array), format: string}>}
  */
 export async function load (url, context, nextLoad) {
-  return stubbedLoadResult(quibbleLoaderState, url) ??
-    await nextLoad(stripQueryAndHash(url), context)
+  const stub = stubbedLoadResult(quibbleLoaderState, url)
+  if (stub) {
+    return stub
+  }
+
+  // Only real loads count: a stub has no imports, so counting it would make
+  // the module look fully known with no edges the next time it loads for real
+  importGraph.recordLoad(url)
+  return nextLoad(stripQueryAndHash(url), context)
 }
 
 export const globalPreload = ({ port }) => {

test/esm-fixtures/import-graph-late-mock/leaf.mjs

@@ -0,0 +1 @@
+export const thing = () => 'real'

test/esm-fixtures/import-graph-late-mock/middle.mjs

@@ -0,0 +1,2 @@
+import { thing } from './leaf.mjs'
+export const fromMiddle = () => thing()

test/esm-fixtures/import-graph-late-mock/subject.mjs

@@ -0,0 +1,2 @@
+import { fromMiddle } from './middle.mjs'
+export const run = () => fromMiddle()

test/esm-fixtures/import-graph-missing-module/importer.mjs

@@ -0,0 +1,2 @@
+import { ghost } from './does-not-exist.mjs'
+export const run = () => ghost()

test/esm-fixtures/import-graph-stub-first/leaf.mjs

@@ -0,0 +1 @@
+export const thing = () => 'real'

test/esm-fixtures/import-graph-stub-first/middle.mjs

@@ -0,0 +1,2 @@
+import { thing } from './leaf.mjs'
+export const fromMiddle = () => thing()

test/esm-fixtures/import-graph-stub-first/subject.mjs

@@ -0,0 +1,2 @@
+import { fromMiddle } from './middle.mjs'
+export const run = () => fromMiddle()

test/esm-fixtures/import-graph/leaf.mjs

@@ -0,0 +1 @@
+export const thing = () => 'real'

test/esm-fixtures/import-graph/middle.mjs

@@ -0,0 +1,2 @@
+import { thing } from './leaf.mjs'
+export const fromMiddle = () => thing()

test/esm-fixtures/import-graph/subject.mjs

@@ -0,0 +1,5 @@
+import { fromMiddle } from './middle.mjs'
+import { evaluation } from './unrelated.mjs'
+
+export const run = () => fromMiddle()
+export const unrelatedEvaluation = evaluation

test/esm-fixtures/import-graph/unrelated.mjs

@@ -0,0 +1,3 @@
+// Counts how many times Node has evaluated this module in this process
+globalThis.__quibbleImportGraphEvaluations = (globalThis.__quibbleImportGraphEvaluations ?? 0) + 1
+export const evaluation = globalThis.__quibbleImportGraphEvaluations

test/esm-lib/quibble-esm-import-graph.test.mjs

@@ -0,0 +1,83 @@
+import quibble from 'quibble'
+
+const importSubject = () => import('../esm-fixtures/import-graph/subject.mjs')
+
+export default {
+  afterEach: function () { quibble.reset() },
+  'a module that depends on a mock only through another module sees the mock': async function () {
+    await quibble.esm('../esm-fixtures/import-graph/leaf.mjs', { thing: () => 'fake' })
+
+    const subject = await importSubject()
+
+    assert.equal(subject.run(), 'fake')
+  },
+  'each new mock is picked up by the modules that depend on it': async function () {
+    for (const name of ['first', 'second', 'third', 'fourth', 'fifth']) {
+      await quibble.esm('../esm-fixtures/import-graph/leaf.mjs', { thing: () => name })
+
+      assert.equal((await importSubject()).run(), name)
+
+      quibble.reset()
+    }
+  },
+  'the real module comes back after a reset': async function () {
+    await quibble.esm('../esm-fixtures/import-graph/leaf.mjs', { thing: () => 'fake' })
+    await importSubject()
+    quibble.reset()
+
+    assert.equal((await importSubject()).run(), 'real')
+  },
+  'a module in the middle of a graph can be mocked': async function () {
+    await quibble.esm('../esm-fixtures/import-graph/middle.mjs', { fromMiddle: () => 'middle fake' })
+
+    assert.equal((await importSubject()).run(), 'middle fake')
+  },
+  'modules unrelated to any mock stop being re-evaluated for every new mock': async function () {
+    const evaluations = []
+    for (let i = 0; i < 8; i++) {
+      await quibble.esm('../esm-fixtures/import-graph/leaf.mjs', { thing: () => `fake ${i}` })
+      evaluations.push((await importSubject()).unrelatedEvaluation)
+      quibble.reset()
+    }
+
+    // Each unrelated module may be evaluated once under the first (tagged)
+    // URL it is loaded with and once more at its plain URL, but no more
+    assert.ok(new Set(evaluations).size <= 2, `evaluated ${new Set(evaluations).size} times: ${evaluations}`)
+    assert.equal(new Set(evaluations.slice(-3)).size, 1, `still changing: ${evaluations}`)
+  },
+  // These fixtures are only used here, so the subject really is first loaded
+  // before any mock exists no matter what order the tests run in
+  'a module imported before any mock exists still sees a mock added later': async function () {
+    const subject = () => import('../esm-fixtures/import-graph-late-mock/subject.mjs')
+    assert.equal((await subject()).run(), 'real')
+
+    await quibble.esm('../esm-fixtures/import-graph-late-mock/leaf.mjs', { thing: () => 'fake' })
+
+    assert.equal((await subject()).run(), 'fake')
+  },
+  // Separate fixtures again, so what each module has been used for before this
+  // test is the same no matter what order the tests run in
+  'a module that was only ever loaded as a mock stub does not hide later mocks of its dependencies': async function () {
+    const subject = () => import('../esm-fixtures/import-graph-stub-first/subject.mjs')
+
+    await quibble.esm('../esm-fixtures/import-graph-stub-first/middle.mjs', { fromMiddle: () => 'middle fake' })
+    assert.equal((await subject()).run(), 'middle fake')
+    quibble.reset()
+
+    // The real middle module is loaded for the first time here, against a fake leaf
+    await quibble.esm('../esm-fixtures/import-graph-stub-first/leaf.mjs', { thing: () => 'leaf fake' })
+    assert.equal((await subject()).run(), 'leaf fake')
+    quibble.reset()
+
+    assert.equal((await subject()).run(), 'real')
+  },
+  'a module that does not exist can be mocked again by each new generation': async function () {
+    for (const name of ['first', 'second', 'third', 'fourth']) {
+      await quibble.esm('../esm-fixtures/import-graph-missing-module/does-not-exist.mjs', { ghost: () => name })
+
+      assert.equal((await import('../esm-fixtures/import-graph-missing-module/importer.mjs')).run(), name)
+
+      quibble.reset()
+    }
+  }
+}