Skip to content

feat(logger): disk-backed hybrid log store for bounded CLI memory (PER-7809) #2193

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pranavz28 wants to merge 9 commits into
base: master
Choose a base branch
from
Open

feat(logger): disk-backed hybrid log store for bounded CLI memory (PER-7809) #2193

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
4e4aa2d
feat(logger): disk-backed hybrid log store for bounded CLI memory (PE…
pranavz28 Apr 22, 2026
fb45efe
fix(per-7809): address test-suite regressions from hybrid log store
pranavz28 Apr 23, 2026
616825a
refactor(per-7809): consolidate log store files and strip verbose com…
pranavz28 Apr 23, 2026
9db0b3e
test(per-7809): add genuine coverage for redactor, readBack fallback,…
pranavz28 Apr 23, 2026
63f130f
refactor(per-7809): extract disk-write helpers so every branch has a …
pranavz28 Apr 23, 2026
626e6b5
fix(logger): swallow read-stream errors in readBack to prevent uncaug…
pranavz28 Apr 23, 2026
f268b21
fix(core): restore published-CLI parity in checkForNoSnapshotCommandE…
pranavz28 Apr 23, 2026
426cb1a
fix(logger): pre-flight spill-file existence to prevent readline hang
pranavz28 Apr 23, 2026
ea03fe8
fix(logger): brace-style lint on readBack pre-flight
pranavz28 Apr 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions packages/core/src/api.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,9 +228,11 @@ export function createPercyServer(percy, port) {
body = Buffer.isBuffer(body) ? body.toString() : body;

if (cmd === 'reset') {
// the reset command will reset testing mode and clear any logs
// Sync clearMemory — the HTTP handler must return synchronously
// and tearing down the disk writer mid-request would race with
// concurrent log writes.
percy.testing = {};
logger.instance.messages.clear();
logger.clearMemory();
} else if (cmd === 'version') {
// the version command will update the api version header for testing
percy.testing.version = body;
Expand Down Expand Up @@ -260,9 +262,9 @@ export function createPercyServer(percy, port) {
.route('get', '/test/requests', (req, res) => res.json(200, {
requests: percy.testing.requests
}))
// returns an array of raw logs from the logger
// in-memory view only; callers needing disk-backed enumeration use readBack()
.route('get', '/test/logs', (req, res) => res.json(200, {
logs: Array.from(logger.instance.messages)
logs: logger.toArray()
}))
// serves a very basic html page for testing snapshots
.route('get', '/test/snapshot', (req, res) => {
Expand Down
26 changes: 15 additions & 11 deletions packages/core/src/percy.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import PercyClient from '@percy/client';
import PercyConfig from '@percy/config';
import logger from '@percy/logger';
import { readBack } from '@percy/logger/internal';
import { getProxy } from '@percy/client/utils';
import Browser from './browser.js';
import Pako from 'pako';
Expand All @@ -9,7 +10,6 @@ import {
generatePromise,
yieldAll,
yieldTo,
redactSecrets,
detectSystemProxyAndLog,
checkSDKVersion,
processCorsIframes
Expand Down Expand Up @@ -596,7 +596,6 @@ export class Percy {
return syncMode;
}

// This specific error will be hard coded
async checkForNoSnapshotCommandError() {
let isPercyStarted = false;
let containsSnapshotTaken = false;
Expand Down Expand Up @@ -682,16 +681,22 @@ export class Percy {
async sendBuildLogs() {
if (!process.env.PERCY_TOKEN) return;
try {
const logsObject = {
clilogs: logger.query(log => !['ci'].includes(log.debug))
};

// Only add CI logs if not disabled voluntarily.
// Stream directly from the on-disk JSONL (or memory fallback) so the
// POST body is the only unbounded-in-memory object, not a full copy
// of every entry. Redaction already happened at write-time.
const clilogs = [];
const cilogs = [];
const sendCILogs = process.env.PERCY_CLIENT_ERROR_LOGS !== 'false';
if (sendCILogs) {
const redactedContent = redactSecrets(logger.query(log => ['ci'].includes(log.debug)));
logsObject.cilogs = redactedContent;

for await (const entry of readBack()) {
if (entry.debug === 'ci') {
if (sendCILogs) cilogs.push(entry);
} else {
clilogs.push(entry);
}
}

const logsObject = sendCILogs ? { clilogs, cilogs } : { clilogs };
const content = base64encode(Pako.gzip(JSON.stringify(logsObject)));
const referenceId = this.build?.id ? `build_${this.build?.id}` : this.build?.id;
const eventObject = {
Expand All @@ -701,7 +706,6 @@ export class Percy {
service_name: 'cli',
base64encoded: true
};
// Ignore this will update once I implement logs controller.
const logsSHA = await this.client.sendBuildLogs(eventObject);
this.log.info(`Build's CLI${sendCILogs ? ' and CI' : ''} logs sent successfully. Please share this log ID with Percy team in case of any issues - ${logsSHA}`);
} catch (err) {
Expand Down
Loading
Loading