diff --git a/packages/cli-command/src/flags.js b/packages/cli-command/src/flags.js index 56b7d7f9f..a8f8e1766 100644 --- a/packages/cli-command/src/flags.js +++ b/packages/cli-command/src/flags.js @@ -94,6 +94,16 @@ export const disableCache = { group: 'Percy' }; +export const maxCacheRam = { + name: 'max-cache-ram', + description: 'Cap asset-discovery cache memory in MB (e.g. 300)', + env: 'PERCY_MAX_CACHE_RAM', + percyrc: 'discovery.maxCacheRam', + type: 'integer', + parse: Number, + group: 'Percy' +}; + export const debug = { name: 'debug', description: 'Debug asset discovery and do not upload snapshots', @@ -134,6 +144,7 @@ export const DISCOVERY = [ disallowedHostnames, networkIdleTimeout, disableCache, + maxCacheRam, debug ]; diff --git a/packages/cli-command/test/flags.test.js b/packages/cli-command/test/flags.test.js index d0c21c87e..b630e493d 100644 --- a/packages/cli-command/test/flags.test.js +++ b/packages/cli-command/test/flags.test.js @@ -104,6 +104,7 @@ describe('Built-in flags:', () => { --disallowed-hostname Disallowed hostnames to abort in asset discovery -t, --network-idle-timeout Asset discovery network idle timeout --disable-cache Disable asset discovery caches + --max-cache-ram Cap asset-discovery cache memory in MB (e.g. 300) --debug Debug asset discovery and do not upload snapshots `); diff --git a/packages/core/README.md b/packages/core/README.md index 04c8989af..24622e7ed 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -53,6 +53,7 @@ The following options can also be defined within a Percy config file - `requestHeaders` — Request headers used when discovering snapshot assets - `authorization` — Basic auth `username` and `password` for protected snapshot assets - `disableCache` — Disable asset caching (**default** `false`) + - `maxCacheRam` — Cap the asset-discovery cache at this many MB (**default** unset/unbounded). When set, least-recently-used resources are evicted to stay within the cap. MB is decimal (1 MB = 1,000,000 bytes), not binary MiB (1,048,576). The cap measures cache body bytes only; process RSS is typically 1.5–2× the cap due to Node's Buffer slab allocator. Values below 25 MB are clamped to 25 MB with a warn log (the per-resource ceiling is 25 MB, so smaller caps would reject every resource). Also settable via the `--max-cache-ram ` CLI flag or the `PERCY_MAX_CACHE_RAM` env var - `userAgent` — Custom user-agent string used when requesting assets - `cookies` — Browser cookies to use when requesting assets - `networkIdleTimeout` — Milliseconds to wait for the network to idle (**default** `100`) diff --git a/packages/core/src/cache/byte-lru.js b/packages/core/src/cache/byte-lru.js new file mode 100644 index 000000000..2e14966da --- /dev/null +++ b/packages/core/src/cache/byte-lru.js @@ -0,0 +1,224 @@ +// Two-tier cache used by asset discovery: +// ByteLRU — byte-budget in-memory LRU; Map insertion order = LRU order. +// DiskSpillStore — on-disk overflow tier. RAM evictions spill here; lookups +// fall back to disk before refetching from origin. +// All operations are synchronous; callers (network intercept, ByteLRU.set) +// cannot yield to the event loop mid-op. Per-entry size is capped at 25MB +// upstream so disk I/O latency is bounded. + +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import crypto from 'crypto'; + +const DEFAULT_PER_ENTRY_OVERHEAD = 512; + +export class ByteLRU { + #map = new Map(); + #bytes = 0; + #max; + #stats = { hits: 0, misses: 0, evictions: 0, peakBytes: 0 }; + onEvict; + + constructor(maxBytes, { onEvict } = {}) { + this.#max = maxBytes; + this.onEvict = onEvict; + } + + get(key) { + if (!this.#map.has(key)) { + this.#stats.misses++; + return undefined; + } + const rec = this.#map.get(key); + this.#map.delete(key); + this.#map.set(key, rec); + this.#stats.hits++; + return rec.value; + } + + set(key, value, size) { + if (!Number.isFinite(size) || size < 0) return false; + + // Reject oversize BEFORE touching any existing entry — a failed set on an + // existing key must not evict the prior (valid) entry. + if (this.#max !== undefined && size > this.#max) { + if (this.onEvict) this.onEvict(key, 'too-big', value); + return false; + } + + if (this.#map.has(key)) { + this.#bytes -= this.#map.get(key).size; + this.#map.delete(key); + } + + this.#map.set(key, { value, size }); + this.#bytes += size; + if (this.#bytes > this.#stats.peakBytes) this.#stats.peakBytes = this.#bytes; + + while (this.#max !== undefined && this.#bytes > this.#max) { + const oldestKey = this.#map.keys().next().value; + const rec = this.#map.get(oldestKey); + this.#bytes -= rec.size; + this.#map.delete(oldestKey); + this.#stats.evictions++; + if (this.onEvict) this.onEvict(oldestKey, 'lru', rec.value); + } + + return true; + } + + has(key) { return this.#map.has(key); } + + delete(key) { + if (!this.#map.has(key)) return false; + this.#bytes -= this.#map.get(key).size; + return this.#map.delete(key); + } + + clear() { + this.#map.clear(); + this.#bytes = 0; + } + + values() { + const iter = this.#map.values(); + return { + next: () => { + const r = iter.next(); + return r.done ? r : { value: r.value.value, done: false }; + }, + [Symbol.iterator]() { return this; } + }; + } + + get size() { return this.#map.size; } + get calculatedSize() { return this.#bytes; } + get stats() { return { ...this.#stats, currentBytes: this.#bytes }; } +} + +// Handles the two Percy cache-entry shapes: single resource, or array of +// roots captured at multiple widths (see discovery.js parseDomResources). +export function entrySize(resource, overhead = DEFAULT_PER_ENTRY_OVERHEAD) { + if (Array.isArray(resource)) { + return resource.reduce((n, r) => n + (r?.content?.length ?? 0) + overhead, 0); + } + return (resource?.content?.length ?? 0) + overhead; +} + +export class DiskSpillStore { + #index = new Map(); + #bytes = 0; + #peakBytes = 0; + #stats = { spilled: 0, restored: 0, spillFailures: 0, readFailures: 0 }; + #counter = 0; + #ready = false; + + constructor(dir, { log } = {}) { + this.dir = dir; + this.log = log; + try { + fs.mkdirSync(dir, { recursive: true }); + this.#ready = true; + } catch (err) { + this.log?.debug?.(`disk-spill init failed for ${dir}: ${err.message}`); + } + } + + // Returns true on success; false on any failure so caller falls back to drop. + // Overwrites prior spill for the same URL — a fresh discovery write wins. + set(url, resource) { + if (!this.#ready) return false; + + let content = resource?.content; + if (content == null) return false; + if (!Buffer.isBuffer(content)) { + try { content = Buffer.from(content); } catch { return false; } + } + + // Counter-based filename keeps URL-derived data out of path.join — + // avoids any path-traversal surface even though sha256 would be safe. + const filepath = path.join(this.dir, String(++this.#counter)); + + try { + fs.writeFileSync(filepath, content); + } catch (err) { + this.#stats.spillFailures++; + this.log?.debug?.(`disk-spill write failed for ${url}: ${err.message}`); + return false; + } + + if (this.#index.has(url)) { + const prev = this.#index.get(url); + this.#bytes -= prev.size; + try { fs.unlinkSync(prev.path); } catch { /* best-effort */ } + } + + const meta = { ...resource }; + delete meta.content; + this.#index.set(url, { path: filepath, size: content.length, meta }); + this.#bytes += content.length; + if (this.#bytes > this.#peakBytes) this.#peakBytes = this.#bytes; + this.#stats.spilled++; + return true; + } + + get(url) { + const entry = this.#index.get(url); + if (!entry) return undefined; + let content; + try { + content = fs.readFileSync(entry.path); + } catch (err) { + this.#stats.readFailures++; + this.log?.debug?.(`disk-spill read failed for ${url}: ${err.message}`); + this.#removeEntry(url, entry); + return undefined; + } + this.#stats.restored++; + return { ...entry.meta, content }; + } + + has(url) { return this.#index.has(url); } + + delete(url) { + const entry = this.#index.get(url); + if (!entry) return false; + this.#removeEntry(url, entry); + return true; + } + + destroy() { + try { + if (this.#ready) fs.rmSync(this.dir, { recursive: true, force: true }); + } catch (err) { + this.log?.debug?.(`disk-spill cleanup failed for ${this.dir}: ${err.message}`); + } + this.#index.clear(); + this.#bytes = 0; + this.#ready = false; + } + + get size() { return this.#index.size; } + get bytes() { return this.#bytes; } + get ready() { return this.#ready; } + get stats() { + return { + ...this.#stats, + currentBytes: this.#bytes, + peakBytes: this.#peakBytes, + entries: this.#index.size + }; + } + + #removeEntry(url, entry) { + this.#bytes -= entry.size; + this.#index.delete(url); + try { fs.unlinkSync(entry.path); } catch { /* best-effort */ } + } +} + +export function createSpillDir() { + const suffix = `${process.pid}-${crypto.randomBytes(4).toString('hex')}`; + return path.join(os.tmpdir(), `percy-cache-${suffix}`); +} diff --git a/packages/core/src/config.js b/packages/core/src/config.js index c34e45427..cd1a8f13c 100644 --- a/packages/core/src/config.js +++ b/packages/core/src/config.js @@ -369,6 +369,10 @@ export const configSchema = { disableCache: { type: 'boolean' }, + maxCacheRam: { + type: ['integer', 'null'], + minimum: 0 + }, captureMockedServiceWorker: { type: 'boolean', default: false diff --git a/packages/core/src/discovery.js b/packages/core/src/discovery.js index a36f0df2d..173b5ce4b 100644 --- a/packages/core/src/discovery.js +++ b/packages/core/src/discovery.js @@ -16,6 +16,7 @@ import { isGzipped, maybeScrollToBottom } from './utils.js'; +import { ByteLRU, entrySize, DiskSpillStore, createSpillDir } from './cache/byte-lru.js'; import { sha256hash } from '@percy/client/utils'; @@ -391,8 +392,67 @@ export async function* discoverSnapshotResources(queue, options, callback) { }, [])); } -// Used to cache resources across core instances export const RESOURCE_CACHE_KEY = Symbol('resource-cache'); +export const CACHE_STATS_KEY = Symbol('resource-cache-stats'); +export const DISK_SPILL_KEY = Symbol('resource-cache-disk-spill'); + +const BYTES_PER_MB = 1_000_000; +// MAX_RESOURCE_SIZE in network.js is 25MB; caps below that would skip every +// resource, so we clamp. MIN_REASONABLE_CAP_MB warns on near-useless caps. +const MAX_RESOURCE_SIZE_MB = 25; +const MIN_REASONABLE_CAP_MB = 50; +const DEFAULT_WARN_THRESHOLD_BYTES = 500 * BYTES_PER_MB; + +function makeCacheStats() { + return { + effectiveMaxCacheRamMB: null, + oversizeSkipped: 0, + firstEvictionEventFired: false, + warningFired: false, + unsetModeBytes: 0 + }; +} + +function readWarnThresholdBytes() { + const raw = Number(process.env.PERCY_CACHE_WARN_THRESHOLD_BYTES); + return Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_WARN_THRESHOLD_BYTES; +} + +// Cache lookup shared by the network intercept path. RAM miss falls through +// to the disk tier; read failures return undefined so the browser refetches. +// Also resolves the array-valued root-resource shape used for multi-width +// DOM snapshots, regardless of which tier returned it. +export function lookupCacheResource(percy, snapshotResources, cache, url, width) { + let resource = snapshotResources.get(url) || cache.get(url); + const disk = percy[DISK_SPILL_KEY]; + if (!resource && disk) { + resource = disk.get(url); + if (resource) { + percy.log.debug( + `cache disk-hit: ${url} (disk=${disk.size}/` + + `${Math.round(disk.bytes / BYTES_PER_MB)}MB)` + ); + } + } + if (resource && Array.isArray(resource) && resource[0].root) { + const rootResource = resource.find(r => r.widths?.includes(width)); + resource = rootResource || resource[0]; + } + return resource; +} + +// Fire-and-forget: telemetry loss must not fail a build. +function fireCacheEventSafe(percy, message, extra) { + if (!percy.build?.id) return; + Promise.resolve() + .then(() => percy.client.sendBuildEvents(percy.build.id, { + message, + cliVersion: percy.client.cliVersion, + clientInfo: percy.clientInfo, + extra + })) + .catch(err => percy.log.debug(`${message} telemetry failed`, err)); +} // Creates an asset discovery queue that uses the percy browser instance to create a page for each // snapshot which is used to intercept and capture snapshot resource requests. @@ -400,21 +460,103 @@ export function createDiscoveryQueue(percy) { let { concurrency } = percy.config.discovery; let queue = new Queue('discovery'); let cache; + let capBytes = null; + // Read once: saveResource consults this on every call. + const warnThreshold = readWarnThresholdBytes(); return queue .set({ concurrency }) - // on start, launch the browser and run the queue .handle('start', async () => { - cache = percy[RESOURCE_CACHE_KEY] = new Map(); + const configuredMaxCacheRamMB = percy.config.discovery.maxCacheRam; + let effectiveMaxCacheRamMB = configuredMaxCacheRamMB; + + // User's config is not mutated; the post-clamp value lives on stats. + if (configuredMaxCacheRamMB != null) { + if (configuredMaxCacheRamMB < MAX_RESOURCE_SIZE_MB) { + percy.log.warn( + `--max-cache-ram=${configuredMaxCacheRamMB}MB is below the ${MAX_RESOURCE_SIZE_MB}MB minimum ` + + '(individual resources up to 25MB would otherwise be dropped). ' + + `Continuing with the minimum: ${MAX_RESOURCE_SIZE_MB}MB.` + ); + effectiveMaxCacheRamMB = MAX_RESOURCE_SIZE_MB; + } else if (configuredMaxCacheRamMB < MIN_REASONABLE_CAP_MB) { + percy.log.warn( + `--max-cache-ram=${configuredMaxCacheRamMB}MB is very small; ` + + 'most resources will not fit and hit rate will be near zero.' + ); + } + if (percy.config.discovery.disableCache) { + percy.log.info('--max-cache-ram is ignored because --disable-cache is set.'); + } + capBytes = effectiveMaxCacheRamMB * BYTES_PER_MB; + } + + if (warnThreshold !== DEFAULT_WARN_THRESHOLD_BYTES) { + percy.log.debug( + `PERCY_CACHE_WARN_THRESHOLD_BYTES override active: ${warnThreshold} bytes ` + + `(default ${DEFAULT_WARN_THRESHOLD_BYTES}).` + ); + } + + percy[CACHE_STATS_KEY] = makeCacheStats(); + percy[CACHE_STATS_KEY].effectiveMaxCacheRamMB = capBytes != null ? effectiveMaxCacheRamMB : null; + + if (capBytes != null) { + // Overflow tier: RAM evictions spill here. diskStore.set returns + // false on any I/O failure → caller falls back to drop automatically. + const diskStore = new DiskSpillStore(createSpillDir(), { log: percy.log }); + percy[DISK_SPILL_KEY] = diskStore; + + cache = percy[RESOURCE_CACHE_KEY] = new ByteLRU(capBytes, { + onEvict: (key, reason, value) => { + if (reason === 'too-big') { + percy[CACHE_STATS_KEY].oversizeSkipped++; + percy.log.debug(`cache skip (oversize): ${key}`); + return; + } + const spilled = diskStore.set(key, value); + percy.log.debug( + `cache ${spilled ? 'spill' : 'evict'}: ${key} ` + + `(cache ${Math.round(cache.calculatedSize / BYTES_PER_MB)}` + + `/${effectiveMaxCacheRamMB}MB, entries=${cache.size}, ` + + `disk=${diskStore.size}/${Math.round(diskStore.bytes / BYTES_PER_MB)}MB)` + ); + const stats = percy[CACHE_STATS_KEY]; + if (stats && !stats.firstEvictionEventFired) { + stats.firstEvictionEventFired = true; + percy.log.info( + 'Cache eviction active — cap reached, oldest entries spilling to disk.' + ); + fireCacheEventSafe(percy, 'cache_eviction_started', { + cache_budget_ram_mb: effectiveMaxCacheRamMB, + cache_peak_bytes_seen: cache.stats.peakBytes, + eviction_count: cache.stats.evictions, + disk_spill_enabled: diskStore.ready + }); + } + } + }); + } else { + cache = percy[RESOURCE_CACHE_KEY] = new Map(); + } - // If browser.launch() fails it will get captured in - // *percy.start() await percy.browser.launch(); queue.run(); }) - // on end, close the browser .handle('end', async () => { await percy.browser.close(); + const diskStore = percy[DISK_SPILL_KEY]; + if (diskStore) { + // Snapshot final stats before destroy() clears them — sendCacheSummary + // runs after this handler, and reads from stats.finalDiskStats. + // CACHE_STATS_KEY is always set alongside DISK_SPILL_KEY in 'start'. + percy[CACHE_STATS_KEY].finalDiskStats = { + ...diskStore.stats, + ready: diskStore.ready + }; + diskStore.destroy(); + delete percy[DISK_SPILL_KEY]; + } }) // snapshots are unique by name and testCase; when deferred also by widths .handle('find', ({ name, testCase, widths }, snapshot) => ( @@ -460,14 +602,9 @@ export function createDiscoveryQueue(percy) { disableCache: snapshot.discovery.disableCache, allowedHostnames: snapshot.discovery.allowedHostnames, disallowedHostnames: snapshot.discovery.disallowedHostnames, - getResource: (u, width = null) => { - let resource = snapshot.resources.get(u) || cache.get(u); - if (resource && Array.isArray(resource) && resource[0].root) { - const rootResource = resource.find(r => r.widths?.includes(width)); - resource = rootResource || resource[0]; - } - return resource; - }, + getResource: (u, width = null) => ( + lookupCacheResource(percy, snapshot.resources, cache, u, width) + ), saveResource: r => { const limitResources = process.env.LIMIT_SNAPSHOT_RESOURCES || false; const MAX_RESOURCES = Number(process.env.MAX_SNAPSHOT_RESOURCES) || 749; @@ -476,8 +613,32 @@ export function createDiscoveryQueue(percy) { return; } snapshot.resources.set(r.url, r); - if (!snapshot.discovery.disableCache) { + if (snapshot.discovery.disableCache) return; + + // Fresh write supersedes any prior spill — prevents races + // where getResource could serve a stale disk copy. + if (percy[DISK_SPILL_KEY]?.has(r.url)) { + percy[DISK_SPILL_KEY].delete(r.url); + } + + if (capBytes != null) { + // ByteLRU fires onEvict('too-big') for oversize entries; + // the oversize_skipped stat + debug log live there. + cache.set(r.url, r, entrySize(r)); + } else { cache.set(r.url, r); + // Track bytes unconditionally so peak is accurate; + // one-shot warn emission is gated by warningFired. + const stats = percy[CACHE_STATS_KEY]; + stats.unsetModeBytes += entrySize(r); + if (!stats.warningFired && stats.unsetModeBytes >= warnThreshold) { + stats.warningFired = true; + percy.log.warn( + `Percy cache is using ${(stats.unsetModeBytes / BYTES_PER_MB).toFixed(1)}MB. ` + + 'If your CI is memory-constrained, set --max-cache-ram. ' + + 'See https://www.browserstack.com/docs/percy/cli/managing-cache-memory' + ); + } } } } diff --git a/packages/core/src/percy.js b/packages/core/src/percy.js index 3f1a9eb48..b1f96b338 100644 --- a/packages/core/src/percy.js +++ b/packages/core/src/percy.js @@ -26,7 +26,10 @@ import { } from './snapshot.js'; import { discoverSnapshotResources, - createDiscoveryQueue + createDiscoveryQueue, + RESOURCE_CACHE_KEY, + CACHE_STATS_KEY, + DISK_SPILL_KEY } from './discovery.js'; import Monitoring from '@percy/monitoring'; import { WaitForJob } from './wait-for-job.js'; @@ -376,7 +379,53 @@ export class Percy { // This issue doesn't comes under regular error logs, // it's detected if we just and stop percy server await this.checkForNoSnapshotCommandError(); + // sendBuildLogs goes first — it's the primary egress. cache_summary is + // analytics, ordered after so a slow pager hop cannot delay the logs. await this.sendBuildLogs(); + await this.sendCacheSummary(); + } + } + + // Fire-and-forget cache-usage summary. Telemetry loss is preferable to a + // failed stop, so errors are swallowed. + async sendCacheSummary() { + try { + if (!this.build?.id) return; + const cache = this[RESOURCE_CACHE_KEY]; + const stats = this[CACHE_STATS_KEY]; + if (!cache || !stats) return; + + const cacheStats = typeof cache.stats === 'object' ? cache.stats : null; + // diskStore is destroyed by discovery 'end' before this runs, so fall + // back to the snapshot captured in stats.finalDiskStats. + const diskStore = this[DISK_SPILL_KEY]; + const diskSnap = diskStore?.stats ?? stats.finalDiskStats; + const diskReady = diskStore ? diskStore.ready : !!stats.finalDiskStats?.ready; + await this.client.sendBuildEvents(this.build.id, { + message: 'cache_summary', + cliVersion: this.client.cliVersion, + clientInfo: this.clientInfo, + extra: { + cache_budget_ram_mb: stats.effectiveMaxCacheRamMB, + hits: cacheStats?.hits ?? 0, + misses: cacheStats?.misses ?? 0, + evictions: cacheStats?.evictions ?? 0, + peak_bytes: cacheStats?.peakBytes ?? stats.unsetModeBytes, + final_bytes: cache.calculatedSize ?? stats.unsetModeBytes, + entry_count: cache.size ?? 0, + oversize_skipped: stats.oversizeSkipped, + disk_spill_enabled: diskReady, + disk_spilled_count: diskSnap?.spilled ?? 0, + disk_restored_count: diskSnap?.restored ?? 0, + disk_spill_failures: diskSnap?.spillFailures ?? 0, + disk_read_failures: diskSnap?.readFailures ?? 0, + disk_peak_bytes: diskSnap?.peakBytes ?? 0, + disk_final_bytes: diskSnap?.currentBytes ?? 0, + disk_final_entries: diskSnap?.entries ?? 0 + } + }); + } catch (err) { + this.log.debug('cache_summary telemetry failed', err); } } diff --git a/packages/core/test/discovery.test.js b/packages/core/test/discovery.test.js index 898bc8b42..daca8243a 100644 --- a/packages/core/test/discovery.test.js +++ b/packages/core/test/discovery.test.js @@ -1,7 +1,9 @@ import { sha256hash } from '@percy/client/utils'; import { logger, api, setupTest, createTestServer, dedent, mockRequests } from './helpers/index.js'; import Percy from '@percy/core'; -import { RESOURCE_CACHE_KEY } from '../src/discovery.js'; +import { RESOURCE_CACHE_KEY, CACHE_STATS_KEY, DISK_SPILL_KEY } from '../src/discovery.js'; +import { ByteLRU, DiskSpillStore } from '../src/cache/byte-lru.js'; +import fs from 'fs'; import Session from '../src/session.js'; import Pako from 'pako'; import * as CoreConfig from '@percy/core/config'; @@ -2314,6 +2316,523 @@ describe('Discovery', () => { }); }); + describe('with --max-cache-ram', () => { + async function startPercyWith(discoveryExtras = {}) { + await percy.stop(true); + percy = await Percy.start({ + token: 'PERCY_TOKEN', + snapshot: { widths: [1000] }, + discovery: { concurrency: 1, ...discoveryExtras } + }); + } + + afterEach(() => { + delete process.env.PERCY_CACHE_WARN_THRESHOLD_BYTES; + }); + + it('installs a ByteLRU when maxCacheRam is set', async () => { + await startPercyWith({ maxCacheRam: 25 }); + const cache = percy[RESOURCE_CACHE_KEY]; + expect(cache instanceof ByteLRU).toBe(true); + expect(cache.calculatedSize).toEqual(0); + expect(cache.stats).toEqual(jasmine.objectContaining({ + hits: 0, misses: 0, evictions: 0, peakBytes: 0 + })); + }); + + it('uses a plain Map when maxCacheRam is unset (backward compat)', () => { + // percy was started in beforeEach without maxCacheRam + expect(percy[RESOURCE_CACHE_KEY] instanceof Map).toBe(true); + }); + + it('clamps a cap below the 25MB floor, warns, and leaves user config intact', async () => { + await startPercyWith({ maxCacheRam: 10 }); + expect(percy[RESOURCE_CACHE_KEY] instanceof ByteLRU).toBe(true); + // User config is NOT mutated — the effective value lives on stats. + expect(percy.config.discovery.maxCacheRam).toEqual(10); + expect(percy[CACHE_STATS_KEY].effectiveMaxCacheRamMB).toEqual(25); + expect(logger.stderr).toEqual(jasmine.arrayContaining([ + jasmine.stringContaining('--max-cache-ram=10MB is below the 25MB minimum') + ])); + }); + + it('emits an info log when cap and --disable-cache are both set', async () => { + await startPercyWith({ maxCacheRam: 50, disableCache: true }); + expect(logger.stdout).toEqual(jasmine.arrayContaining([ + jasmine.stringContaining('--max-cache-ram is ignored because --disable-cache is set') + ])); + }); + + it('warns when cap is >= 25MB but below the reasonable floor', async () => { + // Between MAX_RESOURCE_SIZE_MB (25) and MIN_REASONABLE_CAP_MB (50). + // No clamp, but surface the likely-too-tight hit-rate warning. + await startPercyWith({ maxCacheRam: 30 }); + expect(percy[CACHE_STATS_KEY].effectiveMaxCacheRamMB).toEqual(30); + expect(logger.stderr).toEqual(jasmine.arrayContaining([ + jasmine.stringContaining('--max-cache-ram=30MB is very small') + ])); + }); + + it('fires cache_eviction_started info log + sets gate when an LRU eviction happens', async () => { + await startPercyWith({ maxCacheRam: 25 }); + const cache = percy[RESOURCE_CACHE_KEY]; + // Drive the cache past cap so onEvict fires the lru branch. + const capBytes = 25 * 1_000_000; + const chunk = Math.floor(capBytes / 3); + cache.set('a', { content: Buffer.alloc(chunk) }, chunk + 512); + cache.set('b', { content: Buffer.alloc(chunk) }, chunk + 512); + cache.set('c', { content: Buffer.alloc(chunk) }, chunk + 512); + cache.set('d', { content: Buffer.alloc(chunk) }, chunk + 512); + expect(cache.stats.evictions).toBeGreaterThan(0); + expect(percy[CACHE_STATS_KEY].firstEvictionEventFired).toBe(true); + expect(logger.stdout).toEqual(jasmine.arrayContaining([ + jasmine.stringContaining('Cache eviction active — cap reached, oldest entries spilling to disk') + ])); + }); + + it('fireCacheEventSafe short-circuits when percy.build is not yet set', async () => { + await startPercyWith({ maxCacheRam: 25 }); + percy.build = undefined; + const spy = spyOn(percy.client, 'sendBuildEvents'); + const cache = percy[RESOURCE_CACHE_KEY]; + const chunk = Math.floor(25_000_000 / 3); + cache.set('a', { content: Buffer.alloc(chunk) }, chunk + 512); + cache.set('b', { content: Buffer.alloc(chunk) }, chunk + 512); + cache.set('c', { content: Buffer.alloc(chunk) }, chunk + 512); + cache.set('d', { content: Buffer.alloc(chunk) }, chunk + 512); + expect(spy).not.toHaveBeenCalled(); + }); + + it('fireCacheEventSafe debug-logs and swallows pager rejections', async () => { + await logger.mock({ level: 'debug' }); + await startPercyWith({ maxCacheRam: 25 }); + percy.build = { id: '123' }; + spyOn(percy.client, 'sendBuildEvents').and.rejectWith(new Error('pager down')); + const cache = percy[RESOURCE_CACHE_KEY]; + const capBytes = 25 * 1_000_000; + const chunk = Math.floor(capBytes / 3); + cache.set('a', { content: Buffer.alloc(chunk) }, chunk + 512); + cache.set('b', { content: Buffer.alloc(chunk) }, chunk + 512); + cache.set('c', { content: Buffer.alloc(chunk) }, chunk + 512); + cache.set('d', { content: Buffer.alloc(chunk) }, chunk + 512); + // fireCacheEventSafe is fire-and-forget — wait a microtask tick for the + // catch handler to run. + await new Promise(r => setImmediate(r)); + expect(percy.client.sendBuildEvents).toHaveBeenCalled(); + }); + + it('records oversize_skipped in stats and logs when an entry is bigger than cap', async () => { + await logger.mock({ level: 'debug' }); + await startPercyWith({ maxCacheRam: 25 }); + const cache = percy[RESOURCE_CACHE_KEY]; + // Oversize path: ByteLRU fires onEvict('too-big') which increments + // the stat and logs — upstream network caps at ~16.5MB so this is + // the only way to exercise the branch from core code. + const saved = cache.set('http://x/huge', { content: Buffer.alloc(26_000_001) }, 26_000_001 + 512); + expect(saved).toBe(false); + expect(cache.calculatedSize).toEqual(0); + expect(percy[CACHE_STATS_KEY].oversizeSkipped).toEqual(1); + expect(logger.stderr).toEqual(jasmine.arrayContaining([ + jasmine.stringContaining('cache skip (oversize): http://x/huge') + ])); + }); + }); + + describe('with --max-cache-ram disk-spill tier', () => { + async function startWith(discoveryExtras = {}) { + await percy.stop(true); + percy = await Percy.start({ + token: 'PERCY_TOKEN', + snapshot: { widths: [1000] }, + discovery: { concurrency: 1, ...discoveryExtras } + }); + } + + it('installs a DiskSpillStore alongside ByteLRU when cap is set', async () => { + await startWith({ maxCacheRam: 25 }); + expect(percy[DISK_SPILL_KEY] instanceof DiskSpillStore).toBe(true); + expect(percy[DISK_SPILL_KEY].ready).toBe(true); + expect(fs.existsSync(percy[DISK_SPILL_KEY].dir)).toBe(true); + }); + + it('does not install a DiskSpillStore when no cap is set', () => { + // percy was started in beforeEach without maxCacheRam + expect(percy[DISK_SPILL_KEY]).toBeUndefined(); + }); + + it('spills an LRU-evicted resource to disk instead of dropping it', async () => { + await logger.mock({ level: 'debug' }); + await startWith({ maxCacheRam: 25 }); + const cache = percy[RESOURCE_CACHE_KEY]; + const disk = percy[DISK_SPILL_KEY]; + const chunk = Math.floor(25_000_000 / 3); + cache.set('http://x/a', { url: 'http://x/a', content: Buffer.alloc(chunk, 0xaa) }, chunk + 512); + cache.set('http://x/b', { url: 'http://x/b', content: Buffer.alloc(chunk, 0xbb) }, chunk + 512); + cache.set('http://x/c', { url: 'http://x/c', content: Buffer.alloc(chunk, 0xcc) }, chunk + 512); + cache.set('http://x/d', { url: 'http://x/d', content: Buffer.alloc(chunk, 0xdd) }, chunk + 512); + + expect(cache.has('http://x/a')).toBe(false); + expect(disk.has('http://x/a')).toBe(true); + expect(disk.stats.spilled).toBeGreaterThan(0); + expect(logger.stderr).toEqual(jasmine.arrayContaining([ + jasmine.stringContaining('cache spill: http://x/a') + ])); + }); + + it('rehydrates a spilled resource byte-for-byte on disk-hit', async () => { + await startWith({ maxCacheRam: 25 }); + const disk = percy[DISK_SPILL_KEY]; + const content = Buffer.from([0, 1, 2, 3, 254, 255, 128]); + disk.set('http://x/spilled', { + url: 'http://x/spilled', mimetype: 'image/png', sha: 'abc', content + }); + const got = disk.get('http://x/spilled'); + expect(got.content.equals(content)).toBe(true); + expect(got.sha).toEqual('abc'); + }); + + it('falls back to drop when disk write fails and emits cache evict debug log', async () => { + await logger.mock({ level: 'debug' }); + await startWith({ maxCacheRam: 25 }); + const cache = percy[RESOURCE_CACHE_KEY]; + const disk = percy[DISK_SPILL_KEY]; + spyOn(fs, 'writeFileSync').and.throwError(new Error('ENOSPC')); + const chunk = Math.floor(25_000_000 / 3); + cache.set('http://x/a', { url: 'http://x/a', content: Buffer.alloc(chunk) }, chunk + 512); + cache.set('http://x/b', { url: 'http://x/b', content: Buffer.alloc(chunk) }, chunk + 512); + cache.set('http://x/c', { url: 'http://x/c', content: Buffer.alloc(chunk) }, chunk + 512); + cache.set('http://x/d', { url: 'http://x/d', content: Buffer.alloc(chunk) }, chunk + 512); + + expect(cache.has('http://x/a')).toBe(false); + expect(disk.has('http://x/a')).toBe(false); + expect(disk.stats.spillFailures).toBeGreaterThan(0); + expect(logger.stderr).toEqual(jasmine.arrayContaining([ + jasmine.stringContaining('cache evict:') + ])); + }); + + it('saveResource clears a stale disk entry so fresh writes are not shadowed', async () => { + await startWith({ maxCacheRam: 25 }); + const disk = percy[DISK_SPILL_KEY]; + disk.set('http://localhost:8000/style.css', { + url: 'http://localhost:8000/style.css', + mimetype: 'text/css', + content: Buffer.from('STALE') + }); + expect(disk.has('http://localhost:8000/style.css')).toBe(true); + + await percy.snapshot({ + name: 'stale disk test', + url: 'http://localhost:8000', + domSnapshot: testDOM + }); + await percy.idle(); + + expect(disk.has('http://localhost:8000/style.css')).toBe(false); + }); + + it('calls diskStore.destroy and clears the key in the queue end handler', async () => { + await startWith({ maxCacheRam: 25 }); + const disk = percy[DISK_SPILL_KEY]; + const destroySpy = spyOn(disk, 'destroy').and.callThrough(); + await percy.stop(); + expect(destroySpy).toHaveBeenCalled(); + expect(percy[DISK_SPILL_KEY]).toBeUndefined(); + }); + + it('gracefully handles a DiskSpillStore that fails to init', async () => { + spyOn(fs, 'mkdirSync').and.throwError(new Error('EACCES')); + await startWith({ maxCacheRam: 25 }); + const disk = percy[DISK_SPILL_KEY]; + expect(disk.ready).toBe(false); + const cache = percy[RESOURCE_CACHE_KEY]; + const chunk = Math.floor(25_000_000 / 3); + expect(() => { + cache.set('http://x/a', { url: 'http://x/a', content: Buffer.alloc(chunk) }, chunk + 512); + cache.set('http://x/b', { url: 'http://x/b', content: Buffer.alloc(chunk) }, chunk + 512); + cache.set('http://x/c', { url: 'http://x/c', content: Buffer.alloc(chunk) }, chunk + 512); + cache.set('http://x/d', { url: 'http://x/d', content: Buffer.alloc(chunk) }, chunk + 512); + }).not.toThrow(); + expect(cache.has('http://x/a')).toBe(false); + expect(disk.has('http://x/a')).toBe(false); + }); + }); + + describe('warning-at-threshold (unset cap)', () => { + afterEach(() => { + delete process.env.PERCY_CACHE_WARN_THRESHOLD_BYTES; + }); + + it('emits a debug log when PERCY_CACHE_WARN_THRESHOLD_BYTES is overridden', async () => { + process.env.PERCY_CACHE_WARN_THRESHOLD_BYTES = '100'; + await percy.stop(true); + await logger.mock({ level: 'debug' }); + percy = await Percy.start({ + token: 'PERCY_TOKEN', + snapshot: { widths: [1000] }, + discovery: { concurrency: 1 } + }); + expect(logger.stderr).toEqual(jasmine.arrayContaining([ + jasmine.stringContaining('PERCY_CACHE_WARN_THRESHOLD_BYTES override active') + ])); + }); + + it('fires a warn-level log once when cache crosses the threshold', async () => { + // Override to a tiny threshold so the snapshot's real resources trip it. + process.env.PERCY_CACHE_WARN_THRESHOLD_BYTES = '100'; + await percy.stop(true); + percy = await Percy.start({ + token: 'PERCY_TOKEN', + snapshot: { widths: [1000] }, + discovery: { concurrency: 1 } + }); + + await percy.snapshot({ + name: 'warning snapshot', + url: 'http://localhost:8000', + domSnapshot: testDOM + }); + await percy.idle(); + + const warnHits = logger.stderr.filter(line => + line.includes('Percy cache is using') && + line.includes('--max-cache-ram') + ); + expect(warnHits.length).toEqual(1); + + // Trigger another resource save; the gate should stay closed. + await percy.snapshot({ + name: 'warning snapshot second', + url: 'http://localhost:8000', + domSnapshot: testDOM + }); + await percy.idle(); + + const warnHitsAfter = logger.stderr.filter(line => + line.includes('Percy cache is using') && + line.includes('--max-cache-ram') + ); + expect(warnHitsAfter.length).toEqual(1); + }); + + it('sendCacheSummary swallows telemetry failures (never throws)', async () => { + // Ensure sendCacheSummary's early-return guards don't fire so the + // sendBuildEvents call actually runs and enters the catch. + percy.build = { id: '123' }; + percy[RESOURCE_CACHE_KEY] = new Map(); + percy[CACHE_STATS_KEY] = { + effectiveMaxCacheRamMB: null, + oversizeSkipped: 0, + unsetModeBytes: 0 + }; + const spy = spyOn(percy.client, 'sendBuildEvents') + .and.rejectWith(new Error('pager down')); + await expectAsync(percy.sendCacheSummary()).toBeResolved(); + expect(spy).toHaveBeenCalled(); + }); + + it('sendCacheSummary short-circuits when the build has not been created', async () => { + percy.build = undefined; + const spy = spyOn(percy.client, 'sendBuildEvents'); + await percy.sendCacheSummary(); + expect(spy).not.toHaveBeenCalled(); + }); + + it('sendCacheSummary short-circuits when the cache or stats are missing', async () => { + percy.build = { id: '123' }; + percy[RESOURCE_CACHE_KEY] = undefined; + percy[CACHE_STATS_KEY] = undefined; + const spy = spyOn(percy.client, 'sendBuildEvents'); + await percy.sendCacheSummary(); + expect(spy).not.toHaveBeenCalled(); + }); + + it('sendCacheSummary falls back to 0 when cache has no size field', async () => { + percy.build = { id: '123' }; + // Defensive-path cache shape with no .size — exercises the `?? 0` + // fallback on entry_count. + percy[RESOURCE_CACHE_KEY] = { stats: {} }; + percy[CACHE_STATS_KEY] = { + effectiveMaxCacheRamMB: 25, + oversizeSkipped: 0, + unsetModeBytes: 0 + }; + const spy = spyOn(percy.client, 'sendBuildEvents').and.resolveTo(); + await percy.sendCacheSummary(); + expect(spy).toHaveBeenCalledWith('123', jasmine.objectContaining({ + extra: jasmine.objectContaining({ entry_count: 0 }) + })); + }); + + it('sendCacheSummary reports disk-tier stats when a DiskSpillStore is present', async () => { + percy.build = { id: '123' }; + percy[RESOURCE_CACHE_KEY] = new Map(); + percy[CACHE_STATS_KEY] = { + effectiveMaxCacheRamMB: 25, + oversizeSkipped: 0, + unsetModeBytes: 0 + }; + // destroy is a no-op so the afterEach percy.stop(true) 'end' handler + // does not TypeError when it reaches diskStore.destroy(). + percy[DISK_SPILL_KEY] = { + ready: true, + destroy: () => {}, + stats: { + spilled: 3, + restored: 2, + spillFailures: 1, + readFailures: 0, + currentBytes: 4096, + peakBytes: 8192, + entries: 2 + } + }; + const spy = spyOn(percy.client, 'sendBuildEvents').and.resolveTo(); + await percy.sendCacheSummary(); + expect(spy).toHaveBeenCalledWith('123', jasmine.objectContaining({ + extra: jasmine.objectContaining({ + disk_spill_enabled: true, + disk_spilled_count: 3, + disk_restored_count: 2, + disk_spill_failures: 1, + disk_read_failures: 0, + disk_peak_bytes: 8192, + disk_final_bytes: 4096, + disk_final_entries: 2 + }) + })); + }); + + it('sendCacheSummary falls back to stats.finalDiskStats after discovery.end destroys the diskStore', async () => { + // Real-build ordering: discovery 'end' destroys the diskStore and + // deletes DISK_SPILL_KEY before sendCacheSummary runs. The discovery + // 'end' handler is responsible for snapshotting the disk stats onto + // stats.finalDiskStats so sendCacheSummary can still populate the + // telemetry payload. + percy.build = { id: '123' }; + percy[RESOURCE_CACHE_KEY] = new Map(); + percy[CACHE_STATS_KEY] = { + effectiveMaxCacheRamMB: 25, + oversizeSkipped: 0, + unsetModeBytes: 0, + finalDiskStats: { + ready: true, + spilled: 97, + restored: 96, + spillFailures: 0, + readFailures: 0, + currentBytes: 0, + peakBytes: 36003060, + entries: 0 + } + }; + // DISK_SPILL_KEY intentionally unset — simulates post-destroy state. + const spy = spyOn(percy.client, 'sendBuildEvents').and.resolveTo(); + await percy.sendCacheSummary(); + expect(spy).toHaveBeenCalledWith('123', jasmine.objectContaining({ + extra: jasmine.objectContaining({ + disk_spill_enabled: true, + disk_spilled_count: 97, + disk_restored_count: 96, + disk_peak_bytes: 36003060, + disk_final_bytes: 0, + disk_final_entries: 0 + }) + })); + }); + + it('discovery end handler snapshots diskStore.stats onto stats.finalDiskStats before destroy', async () => { + // Ensures the fix covering the sendCacheSummary ordering is wired into + // the discovery queue's 'end' handler so real builds preserve the data. + await percy.stop(true); + percy = await Percy.start({ + token: 'PERCY_TOKEN', + snapshot: { widths: [1000] }, + discovery: { concurrency: 1, maxCacheRam: 25 } + }); + const stats = percy[CACHE_STATS_KEY]; + const disk = percy[DISK_SPILL_KEY]; + spyOn(disk, 'destroy').and.callThrough(); + // Seed fake stats onto the disk store so we can verify snapshot copies them. + disk._testStats = { spilled: 5, restored: 3, peakBytes: 1234 }; + Object.defineProperty(disk, 'stats', { + configurable: true, + get: () => ({ + ...disk._testStats, + spillFailures: 0, + readFailures: 0, + currentBytes: 0, + entries: 0 + }) + }); + await percy.stop(); + expect(stats.finalDiskStats).toEqual(jasmine.objectContaining({ + ready: true, + spilled: 5, + restored: 3, + peakBytes: 1234 + })); + expect(disk.destroy).toHaveBeenCalled(); + expect(percy[DISK_SPILL_KEY]).toBeUndefined(); + }); + + it('sendCacheSummary reports zeroed disk-tier fields when no DiskSpillStore is present', async () => { + percy.build = { id: '123' }; + percy[RESOURCE_CACHE_KEY] = new Map(); + percy[CACHE_STATS_KEY] = { + effectiveMaxCacheRamMB: null, + oversizeSkipped: 0, + unsetModeBytes: 0 + }; + const spy = spyOn(percy.client, 'sendBuildEvents').and.resolveTo(); + await percy.sendCacheSummary(); + expect(spy).toHaveBeenCalledWith('123', jasmine.objectContaining({ + extra: jasmine.objectContaining({ + disk_spill_enabled: false, + disk_spilled_count: 0, + disk_restored_count: 0, + disk_spill_failures: 0, + disk_read_failures: 0, + disk_peak_bytes: 0, + disk_final_bytes: 0, + disk_final_entries: 0 + }) + })); + }); + + it('keeps incrementing unsetModeBytes after the warning has fired', async () => { + // Regression: the byte counter used to freeze as soon as the warning + // flag flipped, so cache_summary.peak_bytes was always pinned to the + // threshold. Now it grows through the whole run. + process.env.PERCY_CACHE_WARN_THRESHOLD_BYTES = '100'; + await percy.stop(true); + percy = await Percy.start({ + token: 'PERCY_TOKEN', + snapshot: { widths: [1000] }, + discovery: { concurrency: 1 } + }); + + await percy.snapshot({ + name: 'counter grows 1', + url: 'http://localhost:8000', + domSnapshot: testDOM + }); + await percy.idle(); + const afterFirst = percy[CACHE_STATS_KEY].unsetModeBytes; + expect(afterFirst).toBeGreaterThan(100); + expect(percy[CACHE_STATS_KEY].warningFired).toBe(true); + + await percy.snapshot({ + name: 'counter grows 2', + url: 'http://localhost:8000', + domSnapshot: testDOM + }); + await percy.idle(); + // Counter continues to climb even though warningFired is already true. + expect(percy[CACHE_STATS_KEY].unsetModeBytes).toBeGreaterThan(afterFirst); + }); + }); + describe('with resource errors', () => { // sabotage this method to trigger unexpected error handling async function triggerSessionEventError(event, error) { diff --git a/packages/core/test/unit/byte-lru.test.js b/packages/core/test/unit/byte-lru.test.js new file mode 100644 index 000000000..eb927b2e9 --- /dev/null +++ b/packages/core/test/unit/byte-lru.test.js @@ -0,0 +1,574 @@ +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import { + ByteLRU, + entrySize, + DiskSpillStore, + createSpillDir +} from '../../src/cache/byte-lru.js'; +import { DISK_SPILL_KEY, lookupCacheResource } from '../../src/discovery.js'; + +describe('Unit / ByteLRU', () => { + describe('unbounded mode (no cap)', () => { + it('behaves like a Map', () => { + const c = new ByteLRU(); + c.set('a', { body: 'A' }, 100); + c.set('b', { body: 'B' }, 200); + expect(c.size).toEqual(2); + expect(c.get('a')).toEqual({ body: 'A' }); + expect(c.calculatedSize).toEqual(300); + }); + }); + + describe('eviction', () => { + it('evicts LRU when adding over-budget entry', () => { + const c = new ByteLRU(300); + c.set('a', 'A', 100); + c.set('b', 'B', 100); + c.set('c', 'C', 100); + c.set('d', 'D', 100); + expect(c.has('a')).toBe(false); + expect(c.has('d')).toBe(true); + expect(c.calculatedSize).toEqual(300); + }); + + it('evicts multiple entries if new one needs room', () => { + const c = new ByteLRU(500); + c.set('a', 'A', 100); + c.set('b', 'B', 100); + c.set('c', 'C', 100); + c.set('d', 'D', 100); + c.set('e', 'E', 100); + c.set('big', 'BIG', 300); + expect(c.has('a')).toBe(false); + expect(c.has('b')).toBe(false); + expect(c.has('c')).toBe(false); + expect(c.has('d')).toBe(true); + expect(c.has('e')).toBe(true); + expect(c.has('big')).toBe(true); + expect(c.calculatedSize).toEqual(500); + }); + }); + + describe('recency', () => { + it('.get() bumps recency', () => { + const c = new ByteLRU(300); + c.set('a', 'A', 100); + c.set('b', 'B', 100); + c.set('c', 'C', 100); + c.get('a'); + c.set('d', 'D', 100); + expect(c.has('a')).toBe(true); + expect(c.has('b')).toBe(false); + }); + + it('re-set same key updates size & recency', () => { + const c = new ByteLRU(500); + c.set('a', 'A1', 100); + c.set('b', 'B', 100); + c.set('a', 'A2', 200); + expect(c.get('a')).toEqual('A2'); + expect(c.calculatedSize).toEqual(300); + }); + }); + + describe('oversized entry', () => { + it('is skipped; cache unaffected', () => { + const c = new ByteLRU(100); + c.set('a', 'A', 50); + const ok = c.set('huge', 'HUGE', 200); + expect(ok).toBe(false); + expect(c.has('huge')).toBe(false); + expect(c.has('a')).toBe(true); + expect(c.calculatedSize).toEqual(50); + }); + + it('oversized re-set of an existing key leaves the prior entry intact', () => { + const c = new ByteLRU(100); + c.set('k', 'small', 50); + const ok = c.set('k', 'huge', 200); + expect(ok).toBe(false); + expect(c.has('k')).toBe(true); + expect(c.get('k')).toEqual('small'); + expect(c.calculatedSize).toEqual(50); + }); + }); + + describe('onEvict', () => { + it('fires with reason "too-big" and the value on oversize', () => { + const evicted = []; + const c = new ByteLRU(100, { + onEvict: (k, r, v) => evicted.push({ k, r, v }) + }); + c.set('huge', { body: 'HUGE' }, 200); + expect(evicted).toEqual([{ k: 'huge', r: 'too-big', v: { body: 'HUGE' } }]); + }); + + it('fires with reason "lru" and the evicted value when over budget', () => { + const evicted = []; + const c = new ByteLRU(200, { + onEvict: (k, r, v) => evicted.push({ k, r, v }) + }); + c.set('a', { body: 'A' }, 100); + c.set('b', { body: 'B' }, 100); + c.set('c', { body: 'C' }, 100); + expect(evicted).toEqual([{ k: 'a', r: 'lru', v: { body: 'A' } }]); + }); + }); + + describe('.values()', () => { + it('iterates yielding plain values', () => { + const c = new ByteLRU(); + c.set('a', { root: true }, 100); + c.set('b', { root: false }, 100); + c.set('c', { root: true }, 100); + const rootResources = Array.from(c.values()).filter(r => !!r.root); + expect(rootResources.length).toEqual(2); + }); + }); + + describe('.clear()', () => { + it('resets bytes and map', () => { + const c = new ByteLRU(1000); + c.set('a', 'A', 100); + c.set('b', 'B', 200); + c.clear(); + expect(c.size).toEqual(0); + expect(c.calculatedSize).toEqual(0); + expect(c.has('a')).toBe(false); + }); + }); + + describe('.delete()', () => { + it('updates bytes correctly and prevents double-count on re-insert', () => { + const c = new ByteLRU(1000); + c.set('a', 'A', 100); + c.set('b', 'B', 200); + c.delete('a'); + expect(c.has('a')).toBe(false); + expect(c.calculatedSize).toEqual(200); + c.set('a', 'A', 100); + expect(c.calculatedSize).toEqual(300); + }); + + it('returns false when the key is not in the cache', () => { + const c = new ByteLRU(1000); + c.set('a', 'A', 100); + expect(c.delete('missing')).toBe(false); + expect(c.calculatedSize).toEqual(100); + }); + }); + + describe('stats', () => { + it('peakBytes captures transient high-water before eviction', () => { + const c = new ByteLRU(300); + c.set('a', 'A', 100); + c.set('b', 'B', 100); + c.set('c', 'C', 100); + c.set('d', 'D', 100); + c.delete('b'); + c.delete('c'); + c.delete('d'); + expect(c.calculatedSize).toEqual(0); + expect(c.stats.peakBytes).toEqual(400); + }); + + it('tracks hits / misses / evictions', () => { + const c = new ByteLRU(300); + c.set('a', 'A', 100); + c.set('b', 'B', 100); + c.get('a'); c.get('a'); c.get('missing'); + c.set('c', 'C', 100); + c.set('d', 'D', 100); + const s = c.stats; + expect(s.hits).toEqual(2); + expect(s.misses).toEqual(1); + expect(s.evictions).toBeGreaterThan(0); + expect(s.currentBytes).toEqual(300); + }); + }); + + describe('sanity under alternating get/set', () => { + it('calculated bytes stay consistent across heavy churn', () => { + const c = new ByteLRU(1000); + for (let i = 0; i < 100; i++) c.set(`k${i}`, i, 10); + expect(c.calculatedSize).toEqual(1000); + for (let i = 0; i < 100; i++) c.get(`k${i}`); + for (let i = 0; i < 50; i++) c.set(`n${i}`, i, 20); + expect(c.calculatedSize).toEqual(1000); + for (let i = 0; i < 50; i++) expect(c.has(`n${i}`)).toBe(true); + for (let i = 0; i < 100; i++) expect(c.has(`k${i}`)).toBe(false); + }); + }); + + describe('input guards', () => { + it('refuses NaN/negative sizes', () => { + const c = new ByteLRU(1000); + expect(c.set('a', 'A', NaN)).toBe(false); + expect(c.set('b', 'B', -1)).toBe(false); + expect(c.size).toEqual(0); + }); + }); +}); + +describe('Unit / entrySize', () => { + it('sums content.length + overhead for a single resource', () => { + const r = { content: Buffer.alloc(1000) }; + expect(entrySize(r)).toEqual(1000 + 512); + }); + + it('sums across array-valued root-resource-with-widths', () => { + const arr = [ + { root: true, content: Buffer.alloc(100) }, + { root: true, content: Buffer.alloc(150) }, + { root: true, content: Buffer.alloc(200) } + ]; + expect(entrySize(arr)).toEqual(450 + 3 * 512); + }); + + it('tolerates missing content field', () => { + expect(entrySize({})).toEqual(512); + expect(entrySize(null)).toEqual(512); + }); + + it('tolerates null entries and missing content fields inside an array', () => { + const arr = [ + null, + {}, + { content: Buffer.alloc(100) } + ]; + expect(entrySize(arr)).toEqual(100 + 3 * 512); + }); + + it('accepts custom overhead', () => { + expect(entrySize({ content: Buffer.alloc(100) }, 0)).toEqual(100); + }); +}); + +describe('Unit / DiskSpillStore', () => { + function makeResource(url, content, extra = {}) { + return { + url, + sha: 'deadbeef', + mimetype: 'text/css', + content: Buffer.isBuffer(content) ? content : Buffer.from(content), + ...extra + }; + } + + function makeLog() { + const calls = []; + return { calls, debug: (m) => calls.push(m) }; + } + + function freshDir() { + return path.join( + os.tmpdir(), + `disk-spill-test-${process.pid}-${Math.random().toString(36).slice(2, 10)}` + ); + } + + let dir; + let store; + + afterEach(() => { + store?.destroy(); + if (dir && fs.existsSync(dir)) fs.rmSync(dir, { recursive: true, force: true }); + }); + + describe('construction', () => { + it('creates the target directory', () => { + dir = freshDir(); + store = new DiskSpillStore(dir); + expect(fs.existsSync(dir)).toBe(true); + expect(store.ready).toBe(true); + }); + + it('swallows mkdir failure and marks itself not-ready', () => { + const log = makeLog(); + store = new DiskSpillStore('/dev/null/cannot-mkdir-here', { log }); + expect(store.ready).toBe(false); + expect(log.calls.some(m => m.includes('init failed'))).toBe(true); + }); + + it('short-circuits set() when not ready', () => { + store = new DiskSpillStore('/dev/null/cannot-mkdir-here'); + expect(store.set('http://x/a', makeResource('http://x/a', 'A'))).toBe(false); + expect(store.stats.spillFailures).toEqual(0); + }); + + it('works without a log option', () => { + // Covers the optional-chain branches on this.log?.debug?.() calls. + const badDir = '/dev/null/cannot-mkdir-here'; + const silent = new DiskSpillStore(badDir); + expect(silent.ready).toBe(false); + expect(silent.set('http://x/a', makeResource('http://x/a', 'A'))).toBe(false); + }); + }); + + describe('set + get round-trip', () => { + beforeEach(() => { + dir = freshDir(); + store = new DiskSpillStore(dir); + }); + + it('preserves binary content byte-for-byte', () => { + const bin = Buffer.from([0, 1, 2, 253, 254, 255, 0, 127]); + store.set('http://x/bin', makeResource('http://x/bin', bin, { mimetype: 'image/png' })); + const got = store.get('http://x/bin'); + expect(got.content.equals(bin)).toBe(true); + expect(got.mimetype).toEqual('image/png'); + expect(got.url).toEqual('http://x/bin'); + }); + + it('coerces non-Buffer content via Buffer.from', () => { + store.set('http://x/str', { url: 'http://x/str', mimetype: 'text/html', content: 'hello' }); + expect(store.get('http://x/str').content.toString()).toEqual('hello'); + }); + + it('returns false when Buffer.from coercion throws', () => { + // A symbol cannot be coerced to a Buffer. + const badContent = Symbol('x'); + expect(store.set('http://x/bad', { content: badContent })).toBe(false); + }); + + it('returns undefined for unknown urls', () => { + expect(store.get('http://x/missing')).toBeUndefined(); + }); + + it('returns false when content is null/undefined', () => { + expect(store.set('http://x/nil', { url: 'http://x/nil' })).toBe(false); + expect(store.set('http://x/nil2', null)).toBe(false); + }); + + it('carries resource metadata through the round-trip', () => { + store.set( + 'http://x/root', + makeResource('http://x/root', 'root-html', { root: true, widths: [1280], sha: 'abc123' }) + ); + const got = store.get('http://x/root'); + expect(got.root).toBe(true); + expect(got.widths).toEqual([1280]); + expect(got.sha).toEqual('abc123'); + }); + + it('increments spilled/restored counters', () => { + store.set('http://x/a', makeResource('http://x/a', 'A')); + store.set('http://x/b', makeResource('http://x/b', 'B')); + store.get('http://x/a'); + store.get('http://x/a'); + store.get('http://x/missing'); + expect(store.stats.spilled).toEqual(2); + expect(store.stats.restored).toEqual(2); + }); + }); + + describe('accounting', () => { + beforeEach(() => { + dir = freshDir(); + store = new DiskSpillStore(dir); + }); + + it('tracks bytes and peak', () => { + store.set('http://x/a', makeResource('http://x/a', Buffer.alloc(1000))); + store.set('http://x/b', makeResource('http://x/b', Buffer.alloc(2000))); + expect(store.bytes).toEqual(3000); + expect(store.stats.peakBytes).toEqual(3000); + store.delete('http://x/a'); + expect(store.bytes).toEqual(2000); + expect(store.stats.peakBytes).toEqual(3000); + }); + + it('replaces an existing URL and fixes up byte accounting', () => { + store.set('http://x/a', makeResource('http://x/a', Buffer.alloc(1000))); + store.set('http://x/a', makeResource('http://x/a', Buffer.alloc(500))); + expect(store.bytes).toEqual(500); + expect(store.size).toEqual(1); + expect(store.get('http://x/a').content.length).toEqual(500); + }); + + it('silently tolerates unlinkSync errors during overwrite', () => { + // Covers the best-effort unlink branch in the overwrite path. + store.set('http://x/a', makeResource('http://x/a', 'A')); + const spy = spyOn(fs, 'unlinkSync').and.throwError(new Error('EBUSY')); + try { + expect(() => store.set('http://x/a', makeResource('http://x/a', 'B'))).not.toThrow(); + } finally { spy.and.callThrough(); } + expect(store.get('http://x/a').content.toString()).toEqual('B'); + }); + }); + + describe('failure handling', () => { + beforeEach(() => { + dir = freshDir(); + store = new DiskSpillStore(dir); + }); + + it('returns false and increments spillFailures on write error', () => { + const log = makeLog(); + const localStore = new DiskSpillStore(dir, { log }); + const spy = spyOn(fs, 'writeFileSync').and.throwError(new Error('EACCES')); + try { + const ok = localStore.set('http://x/a', makeResource('http://x/a', 'A')); + expect(ok).toBe(false); + expect(localStore.stats.spillFailures).toEqual(1); + expect(log.calls.some(m => m.includes('write failed'))).toBe(true); + } finally { spy.and.callThrough(); } + }); + + it('self-heals the index on read failure', () => { + const log = makeLog(); + const localStore = new DiskSpillStore(dir, { log }); + localStore.set('http://x/a', makeResource('http://x/a', 'A')); + expect(localStore.has('http://x/a')).toBe(true); + + const spy = spyOn(fs, 'readFileSync').and.throwError(new Error('ENOENT')); + try { + const got = localStore.get('http://x/a'); + expect(got).toBeUndefined(); + expect(localStore.has('http://x/a')).toBe(false); + expect(localStore.stats.readFailures).toEqual(1); + expect(log.calls.some(m => m.includes('read failed'))).toBe(true); + } finally { spy.and.callThrough(); } + }); + }); + + describe('delete + destroy', () => { + beforeEach(() => { + dir = freshDir(); + store = new DiskSpillStore(dir); + }); + + it('delete removes both file and index entry, is idempotent', () => { + store.set('http://x/a', makeResource('http://x/a', 'A')); + expect(fs.readdirSync(dir).length).toEqual(1); + expect(store.delete('http://x/a')).toBe(true); + expect(fs.readdirSync(dir).length).toEqual(0); + expect(store.has('http://x/a')).toBe(false); + expect(store.delete('http://x/a')).toBe(false); + }); + + it('delete silently tolerates unlinkSync errors', () => { + store.set('http://x/a', makeResource('http://x/a', 'A')); + const spy = spyOn(fs, 'unlinkSync').and.throwError(new Error('EBUSY')); + try { + expect(() => store.delete('http://x/a')).not.toThrow(); + } finally { spy.and.callThrough(); } + }); + + it('destroy removes the entire dir and clears index', () => { + store.set('http://x/a', makeResource('http://x/a', 'A')); + store.set('http://x/b', makeResource('http://x/b', 'B')); + store.destroy(); + expect(fs.existsSync(dir)).toBe(false); + expect(store.size).toEqual(0); + expect(store.ready).toBe(false); + }); + + it('destroy swallows rm errors', () => { + const log = makeLog(); + const localStore = new DiskSpillStore(dir, { log }); + const spy = spyOn(fs, 'rmSync').and.throwError(new Error('EBUSY')); + try { + expect(() => localStore.destroy()).not.toThrow(); + expect(log.calls.some(m => m.includes('cleanup failed'))).toBe(true); + } finally { spy.and.callThrough(); } + }); + + it('destroy is a no-op when the store was not ready', () => { + const notReady = new DiskSpillStore('/dev/null/cannot-mkdir-here'); + const rmSpy = spyOn(fs, 'rmSync'); + try { + notReady.destroy(); + expect(rmSpy).not.toHaveBeenCalled(); + } finally { rmSpy.and.callThrough(); } + }); + }); + + describe('createSpillDir', () => { + it('returns a unique path under os.tmpdir() with a percy-cache prefix', () => { + const a = createSpillDir(); + const b = createSpillDir(); + expect(a).not.toEqual(b); + expect(a.startsWith(os.tmpdir())).toBe(true); + expect(path.basename(a).startsWith('percy-cache-')).toBe(true); + }); + }); +}); + +describe('Unit / lookupCacheResource', () => { + function makePercy(disk) { + const logs = []; + const percy = { + log: { debug: (m) => logs.push(m) }, + [DISK_SPILL_KEY]: disk + }; + return { percy, logs }; + } + + it('returns a snapshot-local resource first', () => { + const { percy } = makePercy(undefined); + const local = { url: 'a', mimetype: 'text/css', content: Buffer.from('L') }; + const snapshotResources = new Map([['a', local]]); + const cache = new ByteLRU(); + expect(lookupCacheResource(percy, snapshotResources, cache, 'a')).toBe(local); + }); + + it('falls through to RAM cache when snapshot has no entry', () => { + const { percy } = makePercy(undefined); + const cache = new ByteLRU(); + const cached = { url: 'a', content: Buffer.from('C') }; + cache.set('a', cached, 100); + expect(lookupCacheResource(percy, new Map(), cache, 'a')).toBe(cached); + }); + + it('falls through to disk when both snapshot and RAM miss', () => { + const dir = path.join(os.tmpdir(), `lookup-test-${process.pid}-${Math.random().toString(36).slice(2, 8)}`); + const disk = new DiskSpillStore(dir); + try { + disk.set('a', { url: 'a', mimetype: 'text/css', content: Buffer.from('DISK') }); + const { percy, logs } = makePercy(disk); + const got = lookupCacheResource(percy, new Map(), new ByteLRU(), 'a'); + expect(got.content.toString()).toEqual('DISK'); + expect(logs.some(m => m.includes('cache disk-hit: a'))).toBe(true); + } finally { disk.destroy(); } + }); + + it('returns undefined on full miss', () => { + const { percy } = makePercy(undefined); + expect(lookupCacheResource(percy, new Map(), new ByteLRU(), 'missing')).toBeUndefined(); + }); + + it('returns undefined when disk is present but url is absent', () => { + const dir = path.join(os.tmpdir(), `lookup-test-${process.pid}-${Math.random().toString(36).slice(2, 8)}`); + const disk = new DiskSpillStore(dir); + try { + const { percy, logs } = makePercy(disk); + expect(lookupCacheResource(percy, new Map(), new ByteLRU(), 'missing')).toBeUndefined(); + expect(logs.length).toEqual(0); + } finally { disk.destroy(); } + }); + + it('picks the width-matching entry from an array-valued root resource', () => { + const { percy } = makePercy(undefined); + const arr = [ + { root: true, widths: [375], content: Buffer.from('small') }, + { root: true, widths: [1280], content: Buffer.from('wide') } + ]; + const snapshotResources = new Map([['root', arr]]); + const cache = new ByteLRU(); + const got = lookupCacheResource(percy, snapshotResources, cache, 'root', 1280); + expect(got.content.toString()).toEqual('wide'); + }); + + it('falls back to the first array entry when no width matches', () => { + const { percy } = makePercy(undefined); + const arr = [ + { root: true, widths: [375], content: Buffer.from('A') }, + { root: true, widths: [1280], content: Buffer.from('B') } + ]; + const got = lookupCacheResource(percy, new Map([['root', arr]]), new ByteLRU(), 'root', 9999); + expect(got.content.toString()).toEqual('A'); + }); +});