From ab714144a86f7e98069c19102100861a1383ffc7 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Thu, 24 Jul 2025 15:19:23 +0200 Subject: [PATCH 01/29] wip --- test/logs.test.js | 54 ++++++++++++++++++++++++++++++++++++++ test/mock_server/consts.js | 15 +++++++++++ test/mock_server/server.js | 8 ++++++ 3 files changed, 77 insertions(+) create mode 100644 test/mock_server/consts.js diff --git a/test/logs.test.js b/test/logs.test.js index 386f986e5..dcb4207c2 100644 --- a/test/logs.test.js +++ b/test/logs.test.js @@ -1,5 +1,6 @@ const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); const { ApifyClient } = require('apify-client'); +const MOCKED_ACTOR_LOGS = require('./mock_server/consts'); const mockServer = require('./mock_server/server'); describe('Log methods', () => { @@ -58,3 +59,56 @@ describe('Log methods', () => { }); }); }); + + +describe('Redirect logs', () => { + let baseUrl; + const browser = new Browser(); + + beforeAll(async () => { + const server = await mockServer.start(); + await browser.start(); + baseUrl = `http://localhost:${server.address().port}`; + }); + + afterAll(async () => { + await Promise.all([mockServer.close(), browser.cleanUpBrowser()]); + }); + + let client; + let page; + beforeEach(async () => { + page = await browser.getInjectedPage(baseUrl, DEFAULT_OPTIONS); + client = new ApifyClient({ + baseUrl, + maxRetries: 0, + ...DEFAULT_OPTIONS, + }); + }); + afterEach(async () => { + client = null; + page.close().catch(() => {}); + }); + + describe('log(buildOrRunId)', () => { + test('get() works', async () => { + const logId = 'redirect-log-id'; + + const res = await client.log(logId).get(); + expect(res).toBe(MOCKED_ACTOR_LOGS); + + }); + + test('stream() works', async () => { + const logId = 'redirect-log-id'; + + const res = await client.log(logId).stream(); + const chunks = []; + for await (const chunk of res) { + chunks.push(chunk); + } + const id = Buffer.concat(chunks).toString(); + expect(id).toBe('get-log'); + }); + }); +}); diff --git a/test/mock_server/consts.js b/test/mock_server/consts.js new file mode 100644 index 000000000..eb9ec6929 --- /dev/null +++ b/test/mock_server/consts.js @@ -0,0 +1,15 @@ +const MOCKED_ACTOR_LOGS = `2025-05-13T07:24:12.588Z ACTOR: Pulling Docker image of build. +2025-05-13T07:24:12.686Z ACTOR: Creating Docker container. +2025-05-13T07:24:12.745Z ACTOR: Starting Docker container. +2025-05-13T07:26:14.132Z [apify] DEBUG \xc3 +\xa1\n +2025-05-13T07:24:14.132Z [apify] INFO multiline \n log +2025-05-13T07:25:14.132Z [apify] WARNING some warning +2025-05-13T07:26:14.132Z [apify] DEBUG +2025-05-13T0 +7:26:14.132Z [apify] DEBUG d +2025-05-13T07:27:14.132Z [apify] DEB +UG e +2025-05-13T07:28:14.132Z [apify.redirect-logger runId:4U1oAnKau6jpzjUuA] -> 2025-05-13T07:27:14.132Z ACTOR:...\n`; + +module.exports = MOCKED_ACTOR_LOGS; diff --git a/test/mock_server/server.js b/test/mock_server/server.js index 0e0576a71..d6bed1fec 100644 --- a/test/mock_server/server.js +++ b/test/mock_server/server.js @@ -21,6 +21,9 @@ const userRouter = require('./routes/users'); const webhookDispatches = require('./routes/webhook_dispatches'); const webhooks = require('./routes/webhooks'); +// Consts +const MOCKED_ACTOR_LOGS = require('./consts'); + const app = express(); const v2Router = express.Router(); const mockServer = { @@ -74,12 +77,17 @@ app.set('mockServer', mockServer); app.use('/v2', v2Router); app.use('/external', external); + + // Attaching V2 routers v2Router.use('/acts', actorRouter); v2Router.use('/actor-builds', buildRouter); v2Router.use('/actor-runs', runRouter); v2Router.use('/actor-tasks', taskRouter); v2Router.use('/users', userRouter); +v2Router.use('/logs/redirect-log-id', (req, res) => { + res.status(200).send(MOCKED_ACTOR_LOGS); +}); v2Router.use('/logs', logRouter); v2Router.use('/datasets', datasetRouter); v2Router.use('/key-value-stores', keyValueStores); From d328ada06d4817aeec4cd924047a34075f111b98 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Mon, 11 Aug 2025 15:49:29 +0200 Subject: [PATCH 02/29] More wip, todo figure out streaming buffer --- src/resource_clients/log.ts | 150 ++++++++++++++++++++++++++++++++++++ test/logs.test.js | 12 +-- test/mock_server/consts.js | 27 +++---- test/mock_server/server.js | 18 ++++- 4 files changed, 182 insertions(+), 25 deletions(-) diff --git a/src/resource_clients/log.ts b/src/resource_clients/log.ts index b9589a19e..301193e08 100644 --- a/src/resource_clients/log.ts +++ b/src/resource_clients/log.ts @@ -5,6 +5,8 @@ import type { ApiClientSubResourceOptions } from '../base/api_client'; import { ResourceClient } from '../base/resource_client'; import type { ApifyRequestConfig } from '../http_client'; import { cast, catchNotFoundOrThrow } from '../utils'; +import { Log, LogLevel} from "@apify/log"; +//import logger from '@apify/log'; export class LogClient extends ResourceClient { /** @@ -63,3 +65,151 @@ export class LogClient extends ResourceClient { return undefined; } } + +export class StreamedLog { + protected toLogger: Log; + protected streamBuffer: Buffer[] = []; + protected splitMarker = /(?:\n|^)(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z)/g; + protected relevancyTimeLimit: Date | null; + + constructor(toLogger: Log, fromStart: boolean = true) { + this.toLogger = toLogger; + this.relevancyTimeLimit = fromStart ? null : new Date(); + } + + protected processNewData(data: Buffer): void { + this.streamBuffer.push(data); + if (this.splitMarker.test(data.toString())) { + this.logBufferContent(false); + } + } + + protected logBufferContent(includeLastPart: boolean = false): void { + const allParts = Buffer.concat(this.streamBuffer).toString().split(this.splitMarker); + const messageMarkers = includeLastPart ? allParts.filter((_, i) => i % 2 === 0) : allParts.slice(0, -2).filter((_, i) => i % 2 === 0); + const messageContents = includeLastPart ? allParts.filter((_, i) => i % 2 !== 0) : allParts.slice(0, -2).filter((_, i) => i % 2 !== 0); + + this.streamBuffer = includeLastPart ? [] : [Buffer.from(allParts.slice(-2).join(''))]; + + messageMarkers.forEach((marker, index) => { + const decodedMarker = marker; + const decodedContent = messageContents[index]; + if (this.relevancyTimeLimit) { + const logTime = new Date(decodedMarker); + if (logTime < this.relevancyTimeLimit) { + return; + } + } + const message = decodedMarker + decodedContent; + this.toLogger.internal(this.guessLogLevelFromMessage(message), message.trim()); + }); + } + + protected guessLogLevelFromMessage(message: string): LogLevel { + // Original log level information does not have to be included in the message at all. + // This is methods just guesses. + if (message.includes("ERROR")) return LogLevel.ERROR + if (message.includes("SOFT_FAIL")) return LogLevel.SOFT_FAIL; + if (message.includes("WARNING")) return LogLevel.WARNING; + if (message.includes("INFO")) return LogLevel.INFO; + if (message.includes("DEBUG")) return LogLevel.DEBUG; + if (message.includes("PERF")) return LogLevel.PERF; + // Fallback in case original log message does not indicate known log level. + return LogLevel.INFO; + } +} + +export class StreamedLogAsync extends StreamedLog{ + private logClient: { stream: (options: { raw: boolean }) => Promise }; + private streamingTask: Promise | null = null; + private stopLogging = false; + + constructor( + toLogger: Log, + logClient: { stream: (options: { raw: boolean }) => Promise }, + fromStart: boolean = true, + ) { + super(toLogger, fromStart); + this.logClient = logClient; + this.relevancyTimeLimit = fromStart ? null : new Date(); + } + + public async start(): Promise { + if (this.streamingTask) { + throw new Error('Streaming task already active'); + } + this.stopLogging = false; + this.streamingTask = this._streamLog(); + return this.streamingTask; + } + + public async stop(): Promise { + if (!this.streamingTask) { + throw new Error('Streaming task is not active'); + } + this.stopLogging = true; + try { + await this.streamingTask; + } catch (err) { + if (!(err instanceof Error && err.name === 'AbortError')) { + throw err; + } + } finally { + this.streamingTask = null; + } + } + + public async withContext(callback: () => Promise): Promise { + await this.start(); + try { + return await callback(); + } finally { + await this.stop(); + } + } + + private async _streamLog(): Promise { + const logStream = await this.logClient.stream({ raw: true }); + if (!logStream) { + return; + } + + for await (const chunk of logStream) { + this._processNewData(chunk as Buffer); + if (this.stopLogging) { + break; + } + } + + // Process the remaining buffer + this._logBufferContent(true); + } + + private _processNewData(data: Buffer): void { + this.streamBuffer.push(data); + if (this.splitMarker.test(data.toString())) { + this._logBufferContent(false); + } + } + + private _logBufferContent(includeLastPart: boolean): void { + const allParts = Buffer.concat(this.streamBuffer).toString().split(this.splitMarker); + const messageMarkers = includeLastPart ? allParts.filter((_, i) => i % 2 === 0) : allParts.slice(0, -2).filter((_, i) => i % 2 === 0); + const messageContents = includeLastPart ? allParts.filter((_, i) => i % 2 !== 0) : allParts.slice(0, -2).filter((_, i) => i % 2 !== 0); + + this.streamBuffer = includeLastPart ? [] : [Buffer.from(allParts.slice(-2).join(''))]; + + messageMarkers.forEach((marker, index) => { + const decodedMarker = marker; + const decodedContent = messageContents[index]; + if (this.relevancyTimeLimit) { + const logTime = new Date(decodedMarker); + if (logTime < this.relevancyTimeLimit) { + return; + } + } + const message = decodedMarker + decodedContent; + this.toLogger.internal(this.guessLogLevelFromMessage(message), message.trim()); + }); + } +} diff --git a/test/logs.test.js b/test/logs.test.js index dcb4207c2..3576459b4 100644 --- a/test/logs.test.js +++ b/test/logs.test.js @@ -91,14 +91,6 @@ describe('Redirect logs', () => { }); describe('log(buildOrRunId)', () => { - test('get() works', async () => { - const logId = 'redirect-log-id'; - - const res = await client.log(logId).get(); - expect(res).toBe(MOCKED_ACTOR_LOGS); - - }); - test('stream() works', async () => { const logId = 'redirect-log-id'; @@ -107,8 +99,8 @@ describe('Redirect logs', () => { for await (const chunk of res) { chunks.push(chunk); } - const id = Buffer.concat(chunks).toString(); - expect(id).toBe('get-log'); + const log = Buffer.concat(chunks).toString(); + expect(log).toBe('get-log'); }); }); }); diff --git a/test/mock_server/consts.js b/test/mock_server/consts.js index eb9ec6929..1a5a2348b 100644 --- a/test/mock_server/consts.js +++ b/test/mock_server/consts.js @@ -1,15 +1,16 @@ -const MOCKED_ACTOR_LOGS = `2025-05-13T07:24:12.588Z ACTOR: Pulling Docker image of build. -2025-05-13T07:24:12.686Z ACTOR: Creating Docker container. -2025-05-13T07:24:12.745Z ACTOR: Starting Docker container. -2025-05-13T07:26:14.132Z [apify] DEBUG \xc3 -\xa1\n -2025-05-13T07:24:14.132Z [apify] INFO multiline \n log -2025-05-13T07:25:14.132Z [apify] WARNING some warning -2025-05-13T07:26:14.132Z [apify] DEBUG -2025-05-13T0 -7:26:14.132Z [apify] DEBUG d -2025-05-13T07:27:14.132Z [apify] DEB -UG e -2025-05-13T07:28:14.132Z [apify.redirect-logger runId:4U1oAnKau6jpzjUuA] -> 2025-05-13T07:27:14.132Z ACTOR:...\n`; +const MOCKED_ACTOR_LOGS = ['a'.repeat(17000),'2025-05-13T07:24:12.588Z ACTOR: Pulling Docker image of build.\n', + '2025-05-13T07:24:12.686Z ACTOR: Creating Docker container.\n', + '2025-05-13T07:24:12.745Z ACTOR: Starting Docker container.\n', + '2025-05-13T07:26:14.132Z [apify] DEBUG \xc3', + '\xa1\n', + '2025-05-13T07:24:14.132Z [apify] INFO multiline \n log\n', + '2025-05-13T07:25:14.132Z [apify] WARNING some warning\n', + '2025-05-13T07:26:14.132Z [apify] DEBUG c\n', + '2025-05-13T0', + '7:26:14.132Z [apify] DEBUG d \n', + '2025-05-13T07:27:14.132Z [apify] DEB', + 'UG e\n', + '2025-05-13T07:28:14.132Z [apify.redirect-logger runId:4U1oAnKau6jpzjUuA] -> 2025-05-13T07:27:14.132Z ACTOR:...\n' +]; module.exports = MOCKED_ACTOR_LOGS; diff --git a/test/mock_server/server.js b/test/mock_server/server.js index d6bed1fec..1dade5f94 100644 --- a/test/mock_server/server.js +++ b/test/mock_server/server.js @@ -85,8 +85,22 @@ v2Router.use('/actor-builds', buildRouter); v2Router.use('/actor-runs', runRouter); v2Router.use('/actor-tasks', taskRouter); v2Router.use('/users', userRouter); -v2Router.use('/logs/redirect-log-id', (req, res) => { - res.status(200).send(MOCKED_ACTOR_LOGS); +v2Router.use('/logs/redirect-log-id', async (req, res) => { + // Set the appropriate headers for a Server-Sent Events (SSE) stream + res.setHeader('Content-Type', 'text/event-stream'); + res.setHeader('Cache-Control', 'no-cache'); + res.setHeader('Connection', 'keep-alive'); + res.flushHeaders(); // Send the headers immediately + + // Asynchronously write each chunk to the response stream + for (const chunk of MOCKED_ACTOR_LOGS) { + res.write(chunk); + // Wait for a short period to simulate work being done on the server + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + + // End the response stream once all chunks have been sent + res.end(); }); v2Router.use('/logs', logRouter); v2Router.use('/datasets', datasetRouter); From a8ae55f6a2a811090d0bb3413f903433e62e5124 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Mon, 18 Aug 2025 16:03:16 +0200 Subject: [PATCH 03/29] WIP, polish the tests. Add those that exists in Python version. Properly populate actor id and name --- src/resource_clients/log.ts | 109 ++++++++++++++---------------------- src/resource_clients/run.ts | 16 +++++- test/logs.test.js | 20 ++++++- test/mock_server/consts.js | 29 +++++++--- test/mock_server/server.js | 23 +++++--- 5 files changed, 111 insertions(+), 86 deletions(-) diff --git a/src/resource_clients/log.ts b/src/resource_clients/log.ts index 301193e08..98e2bcaa1 100644 --- a/src/resource_clients/log.ts +++ b/src/resource_clients/log.ts @@ -5,7 +5,7 @@ import type { ApiClientSubResourceOptions } from '../base/api_client'; import { ResourceClient } from '../base/resource_client'; import type { ApifyRequestConfig } from '../http_client'; import { cast, catchNotFoundOrThrow } from '../utils'; -import { Log, LogLevel} from "@apify/log"; +import { Log} from "@apify/log"; //import logger from '@apify/log'; export class LogClient extends ResourceClient { @@ -66,70 +66,23 @@ export class LogClient extends ResourceClient { } } -export class StreamedLog { - protected toLogger: Log; - protected streamBuffer: Buffer[] = []; - protected splitMarker = /(?:\n|^)(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z)/g; - protected relevancyTimeLimit: Date | null; - constructor(toLogger: Log, fromStart: boolean = true) { - this.toLogger = toLogger; - this.relevancyTimeLimit = fromStart ? null : new Date(); - } - - protected processNewData(data: Buffer): void { - this.streamBuffer.push(data); - if (this.splitMarker.test(data.toString())) { - this.logBufferContent(false); - } - } - - protected logBufferContent(includeLastPart: boolean = false): void { - const allParts = Buffer.concat(this.streamBuffer).toString().split(this.splitMarker); - const messageMarkers = includeLastPart ? allParts.filter((_, i) => i % 2 === 0) : allParts.slice(0, -2).filter((_, i) => i % 2 === 0); - const messageContents = includeLastPart ? allParts.filter((_, i) => i % 2 !== 0) : allParts.slice(0, -2).filter((_, i) => i % 2 !== 0); +export class StreamedLog{ + private toLogger: Log; + private streamBuffer: Buffer[] = []; + private splitMarker = /(?:\n|^)(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z)/g; + private relevancyTimeLimit: Date | null; - this.streamBuffer = includeLastPart ? [] : [Buffer.from(allParts.slice(-2).join(''))]; - - messageMarkers.forEach((marker, index) => { - const decodedMarker = marker; - const decodedContent = messageContents[index]; - if (this.relevancyTimeLimit) { - const logTime = new Date(decodedMarker); - if (logTime < this.relevancyTimeLimit) { - return; - } - } - const message = decodedMarker + decodedContent; - this.toLogger.internal(this.guessLogLevelFromMessage(message), message.trim()); - }); - } - - protected guessLogLevelFromMessage(message: string): LogLevel { - // Original log level information does not have to be included in the message at all. - // This is methods just guesses. - if (message.includes("ERROR")) return LogLevel.ERROR - if (message.includes("SOFT_FAIL")) return LogLevel.SOFT_FAIL; - if (message.includes("WARNING")) return LogLevel.WARNING; - if (message.includes("INFO")) return LogLevel.INFO; - if (message.includes("DEBUG")) return LogLevel.DEBUG; - if (message.includes("PERF")) return LogLevel.PERF; - // Fallback in case original log message does not indicate known log level. - return LogLevel.INFO; - } -} - -export class StreamedLogAsync extends StreamedLog{ - private logClient: { stream: (options: { raw: boolean }) => Promise }; + private logClient: { stream: (options: { raw: boolean }) => Promise }; private streamingTask: Promise | null = null; private stopLogging = false; constructor( + logClient: { stream: (options: { raw: boolean }) => Promise }, toLogger: Log, - logClient: { stream: (options: { raw: boolean }) => Promise }, - fromStart: boolean = true, + fromStart = true, ) { - super(toLogger, fromStart); + this.toLogger = toLogger; this.logClient = logClient; this.relevancyTimeLimit = fromStart ? null : new Date(); } @@ -159,7 +112,7 @@ export class StreamedLogAsync extends StreamedLog{ } } - public async withContext(callback: () => Promise): Promise { + public async with(callback: () => Promise): Promise { await this.start(); try { return await callback(); @@ -182,22 +135,32 @@ export class StreamedLogAsync extends StreamedLog{ } // Process the remaining buffer - this._logBufferContent(true); + this.logBufferContent(true); } private _processNewData(data: Buffer): void { this.streamBuffer.push(data); if (this.splitMarker.test(data.toString())) { - this._logBufferContent(false); + this.logBufferContent(false); } } - private _logBufferContent(includeLastPart: boolean): void { - const allParts = Buffer.concat(this.streamBuffer).toString().split(this.splitMarker); - const messageMarkers = includeLastPart ? allParts.filter((_, i) => i % 2 === 0) : allParts.slice(0, -2).filter((_, i) => i % 2 === 0); - const messageContents = includeLastPart ? allParts.filter((_, i) => i % 2 !== 0) : allParts.slice(0, -2).filter((_, i) => i % 2 !== 0); + private logBufferContent(includeLastPart = false): void { + const allParts = Buffer.concat(this.streamBuffer).toString().split(this.splitMarker).slice(1); + let messageMarkers; + let messageContents; + if (includeLastPart) { + messageMarkers = allParts.filter((_, i) => i % 2 === 0); + messageContents = allParts.filter((_, i) => i % 2 !== 0); + this.streamBuffer = []; + } + else{ + messageMarkers = allParts.filter((_, i) => i % 2 === 0).slice(-2); + messageContents = allParts.filter((_, i) => i % 2 !== 0).slice(-2); - this.streamBuffer = includeLastPart ? [] : [Buffer.from(allParts.slice(-2).join(''))]; + // The last two parts (marker and message) are possibly not complete and will be left in the buffer + this.streamBuffer = [Buffer.from(allParts.slice(-2).join(''))]; + } messageMarkers.forEach((marker, index) => { const decodedMarker = marker; @@ -209,7 +172,21 @@ export class StreamedLogAsync extends StreamedLog{ } } const message = decodedMarker + decodedContent; - this.toLogger.internal(this.guessLogLevelFromMessage(message), message.trim()); + this.logAtGuessedLevel(message); }); } + + private logAtGuessedLevel(message: string) { + // Original log level information does not have to be included in the message at all. + // This is methods just guesses. + message = message.trim(); + if (message.includes("ERROR")) this.toLogger.error(message); + if (message.includes("SOFT_FAIL")) this.toLogger.softFail(message); + if (message.includes("WARNING")) this.toLogger.warning(message); + if (message.includes("INFO")) this.toLogger.info(message) + if (message.includes("DEBUG")) this.toLogger.debug(message); + if (message.includes("PERF")) this.toLogger.perf(message); + // Fallback in case original log message does not indicate known log level. + this.toLogger.info(message); + } } diff --git a/src/resource_clients/run.ts b/src/resource_clients/run.ts index f165c8dfb..eae282887 100644 --- a/src/resource_clients/run.ts +++ b/src/resource_clients/run.ts @@ -10,8 +10,9 @@ import { cast, parseDateFields, pluckData } from '../utils'; import type { ActorRun } from './actor'; import { DatasetClient } from './dataset'; import { KeyValueStoreClient } from './key_value_store'; -import { LogClient } from './log'; +import { LogClient, StreamedLog} from "./log"; import { RequestQueueClient } from './request_queue'; +import { LEVELS, Log } from "@apify/log"; const RUN_CHARGE_IDEMPOTENCY_HEADER = 'idempotency-key'; @@ -265,6 +266,19 @@ export class RunClient extends ResourceClient { }), ); } + + /** + * Get StreamedLog for convenient streaming of the run log and their redirection. + */ + getStreamedLog(toLogger?: Log, fromStart = true): StreamedLog { + + if (!toLogger){ + // Get actor name and run id + toLogger = new Log({level:LEVELS.DEBUG, prefix:"\x1b[36m actor_name runId:some_id -> \x1b[0m"}) + } + + return new StreamedLog(this.log(), toLogger, fromStart); + } } export interface RunGetOptions { diff --git a/test/logs.test.js b/test/logs.test.js index 3576459b4..7731dde54 100644 --- a/test/logs.test.js +++ b/test/logs.test.js @@ -1,6 +1,6 @@ const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); const { ApifyClient } = require('apify-client'); -const MOCKED_ACTOR_LOGS = require('./mock_server/consts'); +const {MOCKED_ACTOR_LOGS, MOCKED_ACTOR_LOGS_PROCESSED} = require('./mock_server/consts'); const mockServer = require('./mock_server/server'); describe('Log methods', () => { @@ -90,7 +90,7 @@ describe('Redirect logs', () => { page.close().catch(() => {}); }); - describe('log(buildOrRunId)', () => { + describe('run log', () => { test('stream() works', async () => { const logId = 'redirect-log-id'; @@ -100,7 +100,21 @@ describe('Redirect logs', () => { chunks.push(chunk); } const log = Buffer.concat(chunks).toString(); - expect(log).toBe('get-log'); + expect(log).toBe(MOCKED_ACTOR_LOGS.join("")); + }); + + test('StreamedLog', async () => { + const runId = 'redirect-run-id'; + const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); + + const streamedLog = await client.run(runId).getStreamedLog(); + + await streamedLog.start() + await streamedLog.stop() + logger_prefix = 'apify.{_MOCKED_ACTOR_NAME}-{_MOCKED_RUN_ID}' + expect(logSpy.mock.calls).toEqual(MOCKED_ACTOR_LOGS_PROCESSED.map(item => [logger_prefix + item])); + logSpy.mockRestore(); + }); }); }); diff --git a/test/mock_server/consts.js b/test/mock_server/consts.js index 1a5a2348b..35a80e1c9 100644 --- a/test/mock_server/consts.js +++ b/test/mock_server/consts.js @@ -1,16 +1,29 @@ -const MOCKED_ACTOR_LOGS = ['a'.repeat(17000),'2025-05-13T07:24:12.588Z ACTOR: Pulling Docker image of build.\n', +const MOCKED_ACTOR_LOGS = ['2025-05-13T07:24:12.588Z ACTOR: Pulling Docker image of build.\n', '2025-05-13T07:24:12.686Z ACTOR: Creating Docker container.\n', - '2025-05-13T07:24:12.745Z ACTOR: Starting Docker container.\n', - '2025-05-13T07:26:14.132Z [apify] DEBUG \xc3', - '\xa1\n', + '2025-05-13T07:24:12.745Z ACTOR: Starting Docker container.\n', // Several logs merged into one chunk + Buffer.from('2025-05-13T07:26:14.132Z [apify] DEBUG \xc3', 'binary'), // Chunked log split in the middle of the multibyte character + Buffer.from([0xa1, 0x0a]), '2025-05-13T07:24:14.132Z [apify] INFO multiline \n log\n', '2025-05-13T07:25:14.132Z [apify] WARNING some warning\n', '2025-05-13T07:26:14.132Z [apify] DEBUG c\n', - '2025-05-13T0', + '2025-05-13T0', // Chunked log that got split in the marker '7:26:14.132Z [apify] DEBUG d \n', - '2025-05-13T07:27:14.132Z [apify] DEB', + '2025-05-13T07:27:14.132Z [apify] DEB', // Chunked log that got split outside of marker 'UG e\n', - '2025-05-13T07:28:14.132Z [apify.redirect-logger runId:4U1oAnKau6jpzjUuA] -> 2025-05-13T07:27:14.132Z ACTOR:...\n' + '2025-05-13T07:28:14.132Z [apify.redirect-logger runId:4U1oAnKau6jpzjUuA] -> 2025-05-13T07:27:14.132Z ACTOR:...\n' // Already redirected message ]; -module.exports = MOCKED_ACTOR_LOGS; + +const MOCKED_ACTOR_LOGS_PROCESSED= ['2025-05-13T07:24:12.588Z ACTOR: Pulling Docker image of build.', + '2025-05-13T07:24:12.686Z ACTOR: Creating Docker container.', + '2025-05-13T07:24:12.745Z ACTOR: Starting Docker container.', + '2025-05-13T07:26:14.132Z [apify] DEBUG á', + '2025-05-13T07:24:14.132Z [apify] INFO multiline \n log', + '2025-05-13T07:25:14.132Z [apify] WARNING some warning', + '2025-05-13T07:26:14.132Z [apify] DEBUG c', + '2025-05-13T07:26:14.132Z [apify] DEBUG d', + '2025-05-13T07:27:14.132Z [apify] DEBUG e', + '2025-05-13T07:28:14.132Z [apify.redirect-logger runId:4U1oAnKau6jpzjUuA] -> 2025-05-13T07:27:14.132Z ACTOR:...' +]; + +module.exports = {MOCKED_ACTOR_LOGS, MOCKED_ACTOR_LOGS_PROCESSED} ; diff --git a/test/mock_server/server.js b/test/mock_server/server.js index 1dade5f94..80cf47223 100644 --- a/test/mock_server/server.js +++ b/test/mock_server/server.js @@ -22,7 +22,7 @@ const webhookDispatches = require('./routes/webhook_dispatches'); const webhooks = require('./routes/webhooks'); // Consts -const MOCKED_ACTOR_LOGS = require('./consts'); +const {MOCKED_ACTOR_LOGS} = require('./consts'); const app = express(); const v2Router = express.Router(); @@ -82,21 +82,28 @@ app.use('/external', external); // Attaching V2 routers v2Router.use('/acts', actorRouter); v2Router.use('/actor-builds', buildRouter); +v2Router.use('/actor-runs/redirect-run-id/log', async (req, res) => { + // Asynchronously write each chunk to the response stream + for (const chunk of MOCKED_ACTOR_LOGS) { + res.write(chunk); + res.flush(); // Flush the buffer and send the chunk immediately + // Wait for a short period to simulate work being done on the server + await new Promise((resolve) => setTimeout(resolve, 10)); + } + + // End the response stream once all chunks have been sent + res.end(); +}); v2Router.use('/actor-runs', runRouter); v2Router.use('/actor-tasks', taskRouter); v2Router.use('/users', userRouter); v2Router.use('/logs/redirect-log-id', async (req, res) => { - // Set the appropriate headers for a Server-Sent Events (SSE) stream - res.setHeader('Content-Type', 'text/event-stream'); - res.setHeader('Cache-Control', 'no-cache'); - res.setHeader('Connection', 'keep-alive'); - res.flushHeaders(); // Send the headers immediately - // Asynchronously write each chunk to the response stream for (const chunk of MOCKED_ACTOR_LOGS) { res.write(chunk); + res.flush(); // Flush the buffer and send the chunk immediately // Wait for a short period to simulate work being done on the server - await new Promise((resolve) => setTimeout(resolve, 1000)); + await new Promise((resolve) => setTimeout(resolve, 10)); } // End the response stream once all chunks have been sent From 4f2146d8467e5c791cda6543b071d7f5274ad51a Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Mon, 25 Aug 2025 16:20:58 +0200 Subject: [PATCH 04/29] Do I have to split logger into formater + logger? --- package-lock.json | 1 + package.json | 1 + src/resource_clients/log.ts | 26 ++++++++++++++------------ src/resource_clients/run.ts | 19 +++++++++++++++---- test/logs.test.js | 9 +++++---- test/mock_server/consts.js | 4 +++- test/mock_server/server.js | 6 ++++++ 7 files changed, 45 insertions(+), 21 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3384c8667..a40e223d7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "@apify/log": "^2.2.6", "@crawlee/types": "^3.3.0", "agentkeepalive": "^4.2.1", + "ansi-colors": "^4.1.1", "async-retry": "^1.3.3", "axios": "^1.6.7", "content-type": "^1.0.5", diff --git a/package.json b/package.json index 70d1017a9..bb4d8ebf2 100644 --- a/package.json +++ b/package.json @@ -66,6 +66,7 @@ "@apify/log": "^2.2.6", "@crawlee/types": "^3.3.0", "agentkeepalive": "^4.2.1", + "ansi-colors": "^4.1.1", "async-retry": "^1.3.3", "axios": "^1.6.7", "content-type": "^1.0.5", diff --git a/src/resource_clients/log.ts b/src/resource_clients/log.ts index 98e2bcaa1..de322d2dc 100644 --- a/src/resource_clients/log.ts +++ b/src/resource_clients/log.ts @@ -5,8 +5,7 @@ import type { ApiClientSubResourceOptions } from '../base/api_client'; import { ResourceClient } from '../base/resource_client'; import type { ApifyRequestConfig } from '../http_client'; import { cast, catchNotFoundOrThrow } from '../utils'; -import { Log} from "@apify/log"; -//import logger from '@apify/log'; +import { Log } from '@apify/log'; export class LogClient extends ResourceClient { /** @@ -67,7 +66,7 @@ export class LogClient extends ResourceClient { } -export class StreamedLog{ +export class StreamedLog { private toLogger: Log; private streamBuffer: Buffer[] = []; private splitMarker = /(?:\n|^)(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z)/g; @@ -153,8 +152,7 @@ export class StreamedLog{ messageMarkers = allParts.filter((_, i) => i % 2 === 0); messageContents = allParts.filter((_, i) => i % 2 !== 0); this.streamBuffer = []; - } - else{ + } else { messageMarkers = allParts.filter((_, i) => i % 2 === 0).slice(-2); messageContents = allParts.filter((_, i) => i % 2 !== 0).slice(-2); @@ -179,14 +177,18 @@ export class StreamedLog{ private logAtGuessedLevel(message: string) { // Original log level information does not have to be included in the message at all. // This is methods just guesses. + message = message.trim(); - if (message.includes("ERROR")) this.toLogger.error(message); - if (message.includes("SOFT_FAIL")) this.toLogger.softFail(message); - if (message.includes("WARNING")) this.toLogger.warning(message); - if (message.includes("INFO")) this.toLogger.info(message) - if (message.includes("DEBUG")) this.toLogger.debug(message); - if (message.includes("PERF")) this.toLogger.perf(message); + + if (message.includes('ERROR')) this.toLogger.error(message); + else if (message.includes('SOFT_FAIL')) this.toLogger.softFail(message); + else if (message.includes('WARNING')) this.toLogger.warning(message); + else if (message.includes('INFO')) this.toLogger.info(message); + else if (message.includes('DEBUG')) this.toLogger.debug(message); + else if (message.includes('PERF')) this.toLogger.perf(message); // Fallback in case original log message does not indicate known log level. - this.toLogger.info(message); + else this.toLogger.info(message); + + // How to preserve log level filtering, but remove formatting??? } } diff --git a/src/resource_clients/run.ts b/src/resource_clients/run.ts index eae282887..e24f188d0 100644 --- a/src/resource_clients/run.ts +++ b/src/resource_clients/run.ts @@ -1,6 +1,6 @@ import type { AxiosRequestConfig } from 'axios'; import ow from 'ow'; - +import c from 'ansi-colors'; import type { RUN_GENERAL_ACCESS } from '@apify/consts'; import type { ApiClientOptionsWithOptionalResourcePath } from '../base/api_client'; @@ -12,7 +12,8 @@ import { DatasetClient } from './dataset'; import { KeyValueStoreClient } from './key_value_store'; import { LogClient, StreamedLog} from "./log"; import { RequestQueueClient } from './request_queue'; -import { LEVELS, Log } from "@apify/log"; +import { LEVELS, Log} from "@apify/log"; + const RUN_CHARGE_IDEMPOTENCY_HEADER = 'idempotency-key'; @@ -270,11 +271,21 @@ export class RunClient extends ResourceClient { /** * Get StreamedLog for convenient streaming of the run log and their redirection. */ - getStreamedLog(toLogger?: Log, fromStart = true): StreamedLog { + async getStreamedLog(toLogger?: Log, fromStart = true): Promise { if (!toLogger){ // Get actor name and run id - toLogger = new Log({level:LEVELS.DEBUG, prefix:"\x1b[36m actor_name runId:some_id -> \x1b[0m"}) + const runData = await this.get(); + const runId = runData ? `${runData.id ?? ''}` : ''; + + const actorId = runData?.actId ?? ''; + const actorData = (await this.apifyClient.actor(actorId).get()) || {name:""}; + + const actorName = runData ? actorData.name ?? '' : ''; + const name = [actorName, `runId:${runId}`].filter(Boolean).join(' '); + + + toLogger = new Log({level:LEVELS.DEBUG, prefix: c.cyan(`${name} -> `)}) } return new StreamedLog(this.log(), toLogger, fromStart); diff --git a/test/logs.test.js b/test/logs.test.js index 7731dde54..74564d545 100644 --- a/test/logs.test.js +++ b/test/logs.test.js @@ -2,6 +2,8 @@ const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); const { ApifyClient } = require('apify-client'); const {MOCKED_ACTOR_LOGS, MOCKED_ACTOR_LOGS_PROCESSED} = require('./mock_server/consts'); const mockServer = require('./mock_server/server'); +const c = require("ansi-colors"); +const { Logger } = require("@apify/log"); describe('Log methods', () => { let baseUrl; @@ -104,14 +106,13 @@ describe('Redirect logs', () => { }); test('StreamedLog', async () => { - const runId = 'redirect-run-id'; - const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); + const logSpy = jest.spyOn(Logger.prototype, '_outputWithConsole').mockImplementation(() => {}); - const streamedLog = await client.run(runId).getStreamedLog(); + const streamedLog = await client.run('redirect-run-id').getStreamedLog(); await streamedLog.start() await streamedLog.stop() - logger_prefix = 'apify.{_MOCKED_ACTOR_NAME}-{_MOCKED_RUN_ID}' + logger_prefix = c.cyan('redirect-actor-name runId:redirect-run-id ->') expect(logSpy.mock.calls).toEqual(MOCKED_ACTOR_LOGS_PROCESSED.map(item => [logger_prefix + item])); logSpy.mockRestore(); diff --git a/test/mock_server/consts.js b/test/mock_server/consts.js index 35a80e1c9..fa84851fa 100644 --- a/test/mock_server/consts.js +++ b/test/mock_server/consts.js @@ -1,3 +1,5 @@ +const c = require('ansi-colors'); + const MOCKED_ACTOR_LOGS = ['2025-05-13T07:24:12.588Z ACTOR: Pulling Docker image of build.\n', '2025-05-13T07:24:12.686Z ACTOR: Creating Docker container.\n', '2025-05-13T07:24:12.745Z ACTOR: Starting Docker container.\n', // Several logs merged into one chunk @@ -15,7 +17,7 @@ const MOCKED_ACTOR_LOGS = ['2025-05-13T07:24:12.588Z ACTOR: Pulling Docker image const MOCKED_ACTOR_LOGS_PROCESSED= ['2025-05-13T07:24:12.588Z ACTOR: Pulling Docker image of build.', - '2025-05-13T07:24:12.686Z ACTOR: Creating Docker container.', + `${c.gray('2025-05-13T07:24:12.686Z') }ACTOR: Creating Docker container.`, '2025-05-13T07:24:12.745Z ACTOR: Starting Docker container.', '2025-05-13T07:26:14.132Z [apify] DEBUG á', '2025-05-13T07:24:14.132Z [apify] INFO multiline \n log', diff --git a/test/mock_server/server.js b/test/mock_server/server.js index 80cf47223..561ed9920 100644 --- a/test/mock_server/server.js +++ b/test/mock_server/server.js @@ -80,6 +80,9 @@ app.use('/external', external); // Attaching V2 routers +v2Router.use('/acts/redirect-actor-id', async (req, res) => { + res.json({'data':{'name':'redirect-actor-name'}}); +}); v2Router.use('/acts', actorRouter); v2Router.use('/actor-builds', buildRouter); v2Router.use('/actor-runs/redirect-run-id/log', async (req, res) => { @@ -94,6 +97,9 @@ v2Router.use('/actor-runs/redirect-run-id/log', async (req, res) => { // End the response stream once all chunks have been sent res.end(); }); +v2Router.use('/actor-runs/redirect-run-id', async (req, res) => { + res.json({'data':{'id':'redirect-run-id', 'actId':'redirect-actor-id'}}); +}); v2Router.use('/actor-runs', runRouter); v2Router.use('/actor-tasks', taskRouter); v2Router.use('/users', userRouter); From cc7aba9bf29a1516cbe0e8ce4d4a7216f8166f7f Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Tue, 21 Oct 2025 15:11:03 +0200 Subject: [PATCH 05/29] Some working draft TODO: Try in actor --- src/resource_clients/log.ts | 71 ++++++++++++++++++++++++++++--------- src/resource_clients/run.ts | 5 ++- test/logs.test.js | 6 ++-- test/mock_server/consts.js | 2 +- 4 files changed, 61 insertions(+), 23 deletions(-) diff --git a/src/resource_clients/log.ts b/src/resource_clients/log.ts index de322d2dc..fc4630c0d 100644 --- a/src/resource_clients/log.ts +++ b/src/resource_clients/log.ts @@ -1,11 +1,12 @@ import type { Readable } from 'node:stream'; +import c from 'ansi-colors'; import type { ApifyApiError } from '../apify_api_error'; import type { ApiClientSubResourceOptions } from '../base/api_client'; import { ResourceClient } from '../base/resource_client'; import type { ApifyRequestConfig } from '../http_client'; import { cast, catchNotFoundOrThrow } from '../utils'; -import { Log } from '@apify/log'; +import { Log, Logger, LogLevel } from "@apify/log"; export class LogClient extends ResourceClient { /** @@ -66,8 +67,46 @@ export class LogClient extends ResourceClient { } +// Temp create it here and ask Martin where to put it + +const DEFAULT_OPTIONS = { + skipTime: true, + level: LogLevel.DEBUG, +}; + +export class LoggerActorRedirect extends Logger { + constructor(options = {}) { + super({ ...DEFAULT_OPTIONS, ...options }); + } + + _console_log(line:string){ + console.log(line) + } + + override _log(level: LogLevel, message: string, data?: any, exception?: unknown, opts: Record = {}) { + if (level > this.options.level) { + return; + } + if (data || exception){ + throw new Error("Redirect logger does not use other arguments than level and message"); + } + let { prefix } = opts; + prefix = prefix ? `${prefix}` : ''; + + let maybeDate = ''; + if (!this.options.skipTime) { + maybeDate = `${new Date().toISOString().replace('Z', '').replace('T', ' ')} `; + } + + const line = `${c.gray(maybeDate)}${c.cyan(prefix)}${message || ''}`; + this._console_log(line); + return line; + } +} + + export class StreamedLog { - private toLogger: Log; + private toLog: Log; private streamBuffer: Buffer[] = []; private splitMarker = /(?:\n|^)(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z)/g; private relevancyTimeLimit: Date | null; @@ -78,10 +117,10 @@ export class StreamedLog { constructor( logClient: { stream: (options: { raw: boolean }) => Promise }, - toLogger: Log, + toLog: Log, fromStart = true, ) { - this.toLogger = toLogger; + this.toLog = toLog; this.logClient = logClient; this.relevancyTimeLimit = fromStart ? null : new Date(); } @@ -153,8 +192,8 @@ export class StreamedLog { messageContents = allParts.filter((_, i) => i % 2 !== 0); this.streamBuffer = []; } else { - messageMarkers = allParts.filter((_, i) => i % 2 === 0).slice(-2); - messageContents = allParts.filter((_, i) => i % 2 !== 0).slice(-2); + messageMarkers = allParts.filter((_, i) => i % 2 === 0).slice(0,-1); + messageContents = allParts.filter((_, i) => i % 2 !== 0).slice(0,-1); // The last two parts (marker and message) are possibly not complete and will be left in the buffer this.streamBuffer = [Buffer.from(allParts.slice(-2).join(''))]; @@ -176,19 +215,19 @@ export class StreamedLog { private logAtGuessedLevel(message: string) { // Original log level information does not have to be included in the message at all. - // This is methods just guesses. + // This is methods just guesses, exotic formating or specific keywords can break the guessing logic. message = message.trim(); - if (message.includes('ERROR')) this.toLogger.error(message); - else if (message.includes('SOFT_FAIL')) this.toLogger.softFail(message); - else if (message.includes('WARNING')) this.toLogger.warning(message); - else if (message.includes('INFO')) this.toLogger.info(message); - else if (message.includes('DEBUG')) this.toLogger.debug(message); - else if (message.includes('PERF')) this.toLogger.perf(message); - // Fallback in case original log message does not indicate known log level. - else this.toLogger.info(message); + // TODO: Guessing can use the coloring symbols - // How to preserve log level filtering, but remove formatting??? + if (message.includes('ERROR')) this.toLog.error(message); + else if (message.includes('SOFT_FAIL')) this.toLog.softFail(message); + else if (message.includes('WARNING')) this.toLog.warning(message); + else if (message.includes('INFO')) this.toLog.info(message); + else if (message.includes('DEBUG')) this.toLog.debug(message); + else if (message.includes('PERF')) this.toLog.perf(message); + // Fallback in case original log message does not indicate known log level. + else this.toLog.info(message); } } diff --git a/src/resource_clients/run.ts b/src/resource_clients/run.ts index e24f188d0..30eab45da 100644 --- a/src/resource_clients/run.ts +++ b/src/resource_clients/run.ts @@ -1,6 +1,5 @@ import type { AxiosRequestConfig } from 'axios'; import ow from 'ow'; -import c from 'ansi-colors'; import type { RUN_GENERAL_ACCESS } from '@apify/consts'; import type { ApiClientOptionsWithOptionalResourcePath } from '../base/api_client'; @@ -10,7 +9,7 @@ import { cast, parseDateFields, pluckData } from '../utils'; import type { ActorRun } from './actor'; import { DatasetClient } from './dataset'; import { KeyValueStoreClient } from './key_value_store'; -import { LogClient, StreamedLog} from "./log"; +import { LogClient, LoggerActorRedirect, StreamedLog } from "./log"; import { RequestQueueClient } from './request_queue'; import { LEVELS, Log} from "@apify/log"; @@ -285,7 +284,7 @@ export class RunClient extends ResourceClient { const name = [actorName, `runId:${runId}`].filter(Boolean).join(' '); - toLogger = new Log({level:LEVELS.DEBUG, prefix: c.cyan(`${name} -> `)}) + toLogger = new Log({level:LEVELS.DEBUG, prefix: `${name} -> `, logger:new LoggerActorRedirect()}) } return new StreamedLog(this.log(), toLogger, fromStart); diff --git a/test/logs.test.js b/test/logs.test.js index 74564d545..912c8315f 100644 --- a/test/logs.test.js +++ b/test/logs.test.js @@ -1,5 +1,5 @@ const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); -const { ApifyClient } = require('apify-client'); +const { ApifyClient, LoggerActorRedirect } = require('apify-client'); const {MOCKED_ACTOR_LOGS, MOCKED_ACTOR_LOGS_PROCESSED} = require('./mock_server/consts'); const mockServer = require('./mock_server/server'); const c = require("ansi-colors"); @@ -106,13 +106,13 @@ describe('Redirect logs', () => { }); test('StreamedLog', async () => { - const logSpy = jest.spyOn(Logger.prototype, '_outputWithConsole').mockImplementation(() => {}); + const logSpy = jest.spyOn(LoggerActorRedirect.prototype, '_console_log').mockImplementation(() => {}); const streamedLog = await client.run('redirect-run-id').getStreamedLog(); await streamedLog.start() await streamedLog.stop() - logger_prefix = c.cyan('redirect-actor-name runId:redirect-run-id ->') + let logger_prefix = c.cyan('redirect-actor-name runId:redirect-run-id -> '); expect(logSpy.mock.calls).toEqual(MOCKED_ACTOR_LOGS_PROCESSED.map(item => [logger_prefix + item])); logSpy.mockRestore(); diff --git a/test/mock_server/consts.js b/test/mock_server/consts.js index fa84851fa..e4913eb7d 100644 --- a/test/mock_server/consts.js +++ b/test/mock_server/consts.js @@ -17,7 +17,7 @@ const MOCKED_ACTOR_LOGS = ['2025-05-13T07:24:12.588Z ACTOR: Pulling Docker image const MOCKED_ACTOR_LOGS_PROCESSED= ['2025-05-13T07:24:12.588Z ACTOR: Pulling Docker image of build.', - `${c.gray('2025-05-13T07:24:12.686Z') }ACTOR: Creating Docker container.`, + '2025-05-13T07:24:12.686Z ACTOR: Creating Docker container.', '2025-05-13T07:24:12.745Z ACTOR: Starting Docker container.', '2025-05-13T07:26:14.132Z [apify] DEBUG á', '2025-05-13T07:24:14.132Z [apify] INFO multiline \n log', From 50b27323d400140d1ec6030ddd600f962124fcc0 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Tue, 21 Oct 2025 15:39:29 +0200 Subject: [PATCH 06/29] Try to exclude files, to make it possible to install/build from branch --- package.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/package.json b/package.json index bb4d8ebf2..3ef692771 100644 --- a/package.json +++ b/package.json @@ -43,10 +43,6 @@ "url": "https://github.com/apify/apify-client-js/issues" }, "homepage": "https://docs.apify.com/api/client/js/", - "files": [ - "dist", - "!dist/*.tsbuildinfo" - ], "scripts": { "build": "npm run clean && npm run build:node && npm run build:browser", "postbuild": "gen-esm-wrapper dist/index.js dist/index.mjs", From f25935cb07197a593dfdbfdc4bc91a488d15ae92 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Wed, 22 Oct 2025 08:35:33 +0200 Subject: [PATCH 07/29] Add log option to ActorClient.call --- src/resource_clients/actor.ts | 34 +++++++++++++++++++++++++++++++--- src/resource_clients/log.ts | 14 ++++++++------ src/resource_clients/run.ts | 15 ++++++++++----- 3 files changed, 49 insertions(+), 14 deletions(-) diff --git a/src/resource_clients/actor.ts b/src/resource_clients/actor.ts index 18ba66240..647123ce2 100644 --- a/src/resource_clients/actor.ts +++ b/src/resource_clients/actor.ts @@ -17,6 +17,8 @@ import { RunClient } from './run'; import { RunCollectionClient } from './run_collection'; import type { WebhookUpdateData } from './webhook'; import { WebhookCollectionClient } from './webhook_collection'; +import { Log } from "@apify/log"; +import { StreamedLog } from "./log"; export class ActorClient extends ResourceClient { /** @@ -112,7 +114,7 @@ export class ActorClient extends ResourceClient { * It waits indefinitely, unless the `waitSecs` option is provided. * https://docs.apify.com/api/v2#/reference/actors/run-collection/run-actor */ - async call(input?: unknown, options: ActorCallOptions = {}): Promise { + async call(input?: unknown, options: ActorCallOptions = { log: "default"}): Promise { // input can be anything, so no point in validating it. E.g. if you set content-type to application/pdf // then it will process input as a buffer. ow( @@ -126,16 +128,41 @@ export class ActorClient extends ResourceClient { webhooks: ow.optional.array.ofType(ow.object), maxItems: ow.optional.number.not.negative, maxTotalChargeUsd: ow.optional.number.not.negative, + log: ow.optional.any( + ow.string.oneOf(['default']), + ow.null, + ow.object.instanceOf(Log), + ), }), ); - const { waitSecs, ...startOptions } = options; + const { waitSecs, log, ...startOptions } = options; const { id } = await this.start(input, startOptions); // Calling root client because we need access to top level API. // Creating a new instance of RunClient here would only allow // setting it up as a nested route under actor API. - return this.apifyClient.run(id).waitForFinish({ waitSecs }); + const newRunClient = this.apifyClient.run(id) + + if (!log) { + return newRunClient.waitForFinish({ waitSecs }); + } + let streamedLog: StreamedLog + + if (log === 'default') { + streamedLog = await newRunClient.getStreamedLog(); + } else { + streamedLog = await newRunClient.getStreamedLog({ toLog: log }); + } + + try{ + await streamedLog.start() + return this.apifyClient.run(id).waitForFinish({ waitSecs }); + } + finally { + await streamedLog.stop() + } + } /** @@ -397,6 +424,7 @@ export interface ActorStartOptions { export interface ActorCallOptions extends Omit { waitSecs?: number; + log?: Log | null | 'default'; } export interface ActorRunListItem { diff --git a/src/resource_clients/log.ts b/src/resource_clients/log.ts index fc4630c0d..edf920aad 100644 --- a/src/resource_clients/log.ts +++ b/src/resource_clients/log.ts @@ -6,7 +6,8 @@ import type { ApiClientSubResourceOptions } from '../base/api_client'; import { ResourceClient } from '../base/resource_client'; import type { ApifyRequestConfig } from '../http_client'; import { cast, catchNotFoundOrThrow } from '../utils'; -import { Log, Logger, LogLevel } from "@apify/log"; +import type { Log} from "@apify/log"; +import { Logger, LogLevel } from "@apify/log"; export class LogClient extends ResourceClient { /** @@ -22,11 +23,11 @@ export class LogClient extends ResourceClient { /** * https://docs.apify.com/api/v2#/reference/logs/log/get-log */ - async get(): Promise { + async get(options = {raw:false}): Promise { const requestOpts: ApifyRequestConfig = { url: this._url(), method: 'GET', - params: this._params(), + params: this._params(options), }; try { @@ -43,9 +44,10 @@ export class LogClient extends ResourceClient { * Gets the log in a Readable stream format. Only works in Node.js. * https://docs.apify.com/api/v2#/reference/logs/log/get-log */ - async stream(): Promise { + async stream(options = {raw:false}): Promise { const params = { stream: true, + raw: options.raw }; const requestOpts: ApifyRequestConfig = { @@ -192,8 +194,8 @@ export class StreamedLog { messageContents = allParts.filter((_, i) => i % 2 !== 0); this.streamBuffer = []; } else { - messageMarkers = allParts.filter((_, i) => i % 2 === 0).slice(0,-1); - messageContents = allParts.filter((_, i) => i % 2 !== 0).slice(0,-1); + messageMarkers = allParts.filter((_, i) => i % 2 === 0).slice(0, -1); + messageContents = allParts.filter((_, i) => i % 2 !== 0).slice(0, -1); // The last two parts (marker and message) are possibly not complete and will be left in the buffer this.streamBuffer = [Buffer.from(allParts.slice(-2).join(''))]; diff --git a/src/resource_clients/run.ts b/src/resource_clients/run.ts index 30eab45da..f0798e21c 100644 --- a/src/resource_clients/run.ts +++ b/src/resource_clients/run.ts @@ -270,9 +270,9 @@ export class RunClient extends ResourceClient { /** * Get StreamedLog for convenient streaming of the run log and their redirection. */ - async getStreamedLog(toLogger?: Log, fromStart = true): Promise { - - if (!toLogger){ + async getStreamedLog(options: GetStreamedLogOptions = {}): Promise { + let { toLog, fromStart = true } = options; + if (!toLog){ // Get actor name and run id const runData = await this.get(); const runId = runData ? `${runData.id ?? ''}` : ''; @@ -284,13 +284,18 @@ export class RunClient extends ResourceClient { const name = [actorName, `runId:${runId}`].filter(Boolean).join(' '); - toLogger = new Log({level:LEVELS.DEBUG, prefix: `${name} -> `, logger:new LoggerActorRedirect()}) + toLog = new Log({level:LEVELS.DEBUG, prefix: `${name} -> `, logger:new LoggerActorRedirect()}) } - return new StreamedLog(this.log(), toLogger, fromStart); + return new StreamedLog(this.log(), toLog, fromStart); } } +export interface GetStreamedLogOptions { + toLog?: Log; + fromStart?: boolean; +} + export interface RunGetOptions { waitForFinish?: number; } From 4aa012565143228ebbda4d77148938409f32a8bf Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Wed, 22 Oct 2025 10:59:13 +0200 Subject: [PATCH 08/29] Update type hint --- src/resource_clients/log.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/resource_clients/log.ts b/src/resource_clients/log.ts index edf920aad..f1e438f96 100644 --- a/src/resource_clients/log.ts +++ b/src/resource_clients/log.ts @@ -113,12 +113,12 @@ export class StreamedLog { private splitMarker = /(?:\n|^)(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z)/g; private relevancyTimeLimit: Date | null; - private logClient: { stream: (options: { raw: boolean }) => Promise }; + private logClient: LogClient; private streamingTask: Promise | null = null; private stopLogging = false; constructor( - logClient: { stream: (options: { raw: boolean }) => Promise }, + logClient: LogClient, toLog: Log, fromStart = true, ) { From c410190adff1e0c77ee134ccc4798cf95d546b1f Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Wed, 22 Oct 2025 14:50:50 +0200 Subject: [PATCH 09/29] Format and lint and add test --- src/resource_clients/actor.ts | 30 +++++++++------------ src/resource_clients/log.ts | 29 +++++++++----------- src/resource_clients/run.ts | 18 ++++++------- test/logs.test.js | 51 ++++++++++++++++++++--------------- test/mock_server/consts.js | 15 +++++------ test/mock_server/server.js | 16 ++++++----- 6 files changed, 79 insertions(+), 80 deletions(-) diff --git a/src/resource_clients/actor.ts b/src/resource_clients/actor.ts index 647123ce2..768eb9cb8 100644 --- a/src/resource_clients/actor.ts +++ b/src/resource_clients/actor.ts @@ -3,6 +3,7 @@ import ow from 'ow'; import type { RUN_GENERAL_ACCESS } from '@apify/consts'; import { ACT_JOB_STATUSES, META_ORIGINS } from '@apify/consts'; +import { Log } from '@apify/log'; import type { ApiClientSubResourceOptions } from '../base/api_client'; import { ResourceClient } from '../base/resource_client'; @@ -13,12 +14,11 @@ import { ActorVersionCollectionClient } from './actor_version_collection'; import type { Build, BuildClientGetOptions } from './build'; import { BuildClient } from './build'; import { BuildCollectionClient } from './build_collection'; +import type { StreamedLog } from './log'; import { RunClient } from './run'; import { RunCollectionClient } from './run_collection'; import type { WebhookUpdateData } from './webhook'; import { WebhookCollectionClient } from './webhook_collection'; -import { Log } from "@apify/log"; -import { StreamedLog } from "./log"; export class ActorClient extends ResourceClient { /** @@ -114,7 +114,7 @@ export class ActorClient extends ResourceClient { * It waits indefinitely, unless the `waitSecs` option is provided. * https://docs.apify.com/api/v2#/reference/actors/run-collection/run-actor */ - async call(input?: unknown, options: ActorCallOptions = { log: "default"}): Promise { + async call(input?: unknown, options: ActorCallOptions = {}): Promise { // input can be anything, so no point in validating it. E.g. if you set content-type to application/pdf // then it will process input as a buffer. ow( @@ -128,26 +128,22 @@ export class ActorClient extends ResourceClient { webhooks: ow.optional.array.ofType(ow.object), maxItems: ow.optional.number.not.negative, maxTotalChargeUsd: ow.optional.number.not.negative, - log: ow.optional.any( - ow.string.oneOf(['default']), - ow.null, - ow.object.instanceOf(Log), - ), + log: ow.optional.any(ow.null, ow.object.instanceOf(Log)), }), ); - const { waitSecs, log, ...startOptions } = options; + const { waitSecs, log = 'default', ...startOptions } = options; const { id } = await this.start(input, startOptions); // Calling root client because we need access to top level API. // Creating a new instance of RunClient here would only allow // setting it up as a nested route under actor API. - const newRunClient = this.apifyClient.run(id) + const newRunClient = this.apifyClient.run(id); if (!log) { return newRunClient.waitForFinish({ waitSecs }); } - let streamedLog: StreamedLog + let streamedLog: StreamedLog; if (log === 'default') { streamedLog = await newRunClient.getStreamedLog(); @@ -155,14 +151,12 @@ export class ActorClient extends ResourceClient { streamedLog = await newRunClient.getStreamedLog({ toLog: log }); } - try{ - await streamedLog.start() + try { + await streamedLog.start(); return this.apifyClient.run(id).waitForFinish({ waitSecs }); + } finally { + await streamedLog.stop(); } - finally { - await streamedLog.stop() - } - } /** @@ -424,7 +418,7 @@ export interface ActorStartOptions { export interface ActorCallOptions extends Omit { waitSecs?: number; - log?: Log | null | 'default'; + log?: Log | null; } export interface ActorRunListItem { diff --git a/src/resource_clients/log.ts b/src/resource_clients/log.ts index f1e438f96..4402c846a 100644 --- a/src/resource_clients/log.ts +++ b/src/resource_clients/log.ts @@ -1,13 +1,16 @@ +// eslint-disable-next-line max-classes-per-file import type { Readable } from 'node:stream'; + import c from 'ansi-colors'; +import type { Log } from '@apify/log'; +import { Logger, LogLevel } from '@apify/log'; + import type { ApifyApiError } from '../apify_api_error'; import type { ApiClientSubResourceOptions } from '../base/api_client'; import { ResourceClient } from '../base/resource_client'; import type { ApifyRequestConfig } from '../http_client'; import { cast, catchNotFoundOrThrow } from '../utils'; -import type { Log} from "@apify/log"; -import { Logger, LogLevel } from "@apify/log"; export class LogClient extends ResourceClient { /** @@ -23,7 +26,7 @@ export class LogClient extends ResourceClient { /** * https://docs.apify.com/api/v2#/reference/logs/log/get-log */ - async get(options = {raw:false}): Promise { + async get(options = { raw: false }): Promise { const requestOpts: ApifyRequestConfig = { url: this._url(), method: 'GET', @@ -44,10 +47,10 @@ export class LogClient extends ResourceClient { * Gets the log in a Readable stream format. Only works in Node.js. * https://docs.apify.com/api/v2#/reference/logs/log/get-log */ - async stream(options = {raw:false}): Promise { + async stream(options = { raw: false }): Promise { const params = { stream: true, - raw: options.raw + raw: options.raw, }; const requestOpts: ApifyRequestConfig = { @@ -68,7 +71,6 @@ export class LogClient extends ResourceClient { } } - // Temp create it here and ask Martin where to put it const DEFAULT_OPTIONS = { @@ -81,16 +83,16 @@ export class LoggerActorRedirect extends Logger { super({ ...DEFAULT_OPTIONS, ...options }); } - _console_log(line:string){ - console.log(line) + _console_log(line: string) { + console.log(line); // eslint-disable-line no-console } override _log(level: LogLevel, message: string, data?: any, exception?: unknown, opts: Record = {}) { if (level > this.options.level) { return; } - if (data || exception){ - throw new Error("Redirect logger does not use other arguments than level and message"); + if (data || exception) { + throw new Error('Redirect logger does not use other arguments than level and message'); } let { prefix } = opts; prefix = prefix ? `${prefix}` : ''; @@ -106,7 +108,6 @@ export class LoggerActorRedirect extends Logger { } } - export class StreamedLog { private toLog: Log; private streamBuffer: Buffer[] = []; @@ -117,11 +118,7 @@ export class StreamedLog { private streamingTask: Promise | null = null; private stopLogging = false; - constructor( - logClient: LogClient, - toLog: Log, - fromStart = true, - ) { + constructor(logClient: LogClient, toLog: Log, fromStart = true) { this.toLog = toLog; this.logClient = logClient; this.relevancyTimeLimit = fromStart ? null : new Date(); diff --git a/src/resource_clients/run.ts b/src/resource_clients/run.ts index f0798e21c..e12bd791e 100644 --- a/src/resource_clients/run.ts +++ b/src/resource_clients/run.ts @@ -1,6 +1,8 @@ import type { AxiosRequestConfig } from 'axios'; import ow from 'ow'; + import type { RUN_GENERAL_ACCESS } from '@apify/consts'; +import { LEVELS, Log } from '@apify/log'; import type { ApiClientOptionsWithOptionalResourcePath } from '../base/api_client'; import { ResourceClient } from '../base/resource_client'; @@ -9,10 +11,8 @@ import { cast, parseDateFields, pluckData } from '../utils'; import type { ActorRun } from './actor'; import { DatasetClient } from './dataset'; import { KeyValueStoreClient } from './key_value_store'; -import { LogClient, LoggerActorRedirect, StreamedLog } from "./log"; +import { LogClient, LoggerActorRedirect, StreamedLog } from './log'; import { RequestQueueClient } from './request_queue'; -import { LEVELS, Log} from "@apify/log"; - const RUN_CHARGE_IDEMPOTENCY_HEADER = 'idempotency-key'; @@ -271,20 +271,20 @@ export class RunClient extends ResourceClient { * Get StreamedLog for convenient streaming of the run log and their redirection. */ async getStreamedLog(options: GetStreamedLogOptions = {}): Promise { - let { toLog, fromStart = true } = options; - if (!toLog){ + const { fromStart = true } = options; + let { toLog } = options; + if (!toLog) { // Get actor name and run id const runData = await this.get(); const runId = runData ? `${runData.id ?? ''}` : ''; const actorId = runData?.actId ?? ''; - const actorData = (await this.apifyClient.actor(actorId).get()) || {name:""}; + const actorData = (await this.apifyClient.actor(actorId).get()) || { name: '' }; - const actorName = runData ? actorData.name ?? '' : ''; + const actorName = runData ? (actorData.name ?? '') : ''; const name = [actorName, `runId:${runId}`].filter(Boolean).join(' '); - - toLog = new Log({level:LEVELS.DEBUG, prefix: `${name} -> `, logger:new LoggerActorRedirect()}) + toLog = new Log({ level: LEVELS.DEBUG, prefix: `${name} -> `, logger: new LoggerActorRedirect() }); } return new StreamedLog(this.log(), toLog, fromStart); diff --git a/test/logs.test.js b/test/logs.test.js index 912c8315f..8e7f02995 100644 --- a/test/logs.test.js +++ b/test/logs.test.js @@ -1,9 +1,8 @@ const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); const { ApifyClient, LoggerActorRedirect } = require('apify-client'); -const {MOCKED_ACTOR_LOGS, MOCKED_ACTOR_LOGS_PROCESSED} = require('./mock_server/consts'); +const { MOCKED_ACTOR_LOGS_PROCESSED } = require('./mock_server/consts'); const mockServer = require('./mock_server/server'); -const c = require("ansi-colors"); -const { Logger } = require("@apify/log"); +const c = require('ansi-colors'); describe('Log methods', () => { let baseUrl; @@ -62,7 +61,6 @@ describe('Log methods', () => { }); }); - describe('Redirect logs', () => { let baseUrl; const browser = new Browser(); @@ -92,30 +90,39 @@ describe('Redirect logs', () => { page.close().catch(() => {}); }); - describe('run log', () => { - test('stream() works', async () => { - const logId = 'redirect-log-id'; - - const res = await client.log(logId).stream(); - const chunks = []; - for await (const chunk of res) { - chunks.push(chunk); - } - const log = Buffer.concat(chunks).toString(); - expect(log).toBe(MOCKED_ACTOR_LOGS.join("")); - }); - - test('StreamedLog', async () => { + describe('run.getStreamedLog', () => { + test('getStreamedLog - fromStart', async () => { const logSpy = jest.spyOn(LoggerActorRedirect.prototype, '_console_log').mockImplementation(() => {}); + // Set fake time in constructor to skip the first redirected log entry, fromStart=True should redirect all logs + jest.useFakeTimers(); + jest.setSystemTime(new Date('2025-05-13T07:24:12.686Z')); const streamedLog = await client.run('redirect-run-id').getStreamedLog(); + jest.useRealTimers(); - await streamedLog.start() - await streamedLog.stop() - let logger_prefix = c.cyan('redirect-actor-name runId:redirect-run-id -> '); - expect(logSpy.mock.calls).toEqual(MOCKED_ACTOR_LOGS_PROCESSED.map(item => [logger_prefix + item])); + await streamedLog.start(); + await streamedLog.stop(); + const loggerPrefix = c.cyan('redirect-actor-name runId:redirect-run-id -> '); + expect(logSpy.mock.calls).toEqual(MOCKED_ACTOR_LOGS_PROCESSED.map((item) => [loggerPrefix + item])); logSpy.mockRestore(); + }); + test('getStreamedLog - not fromStart', async () => { + const logSpy = jest.spyOn(LoggerActorRedirect.prototype, '_console_log').mockImplementation(() => {}); + + // Set fake time in constructor to skip the first redirected log entry, fromStart is redirecting only new logs + jest.useFakeTimers(); + jest.setSystemTime(new Date('2025-05-13T07:24:12.686Z')); + const streamedLog = await client.run('redirect-run-id').getStreamedLog({ fromStart: false }); + jest.useRealTimers(); + + await streamedLog.start(); + await streamedLog.stop(); + const loggerPrefix = c.cyan('redirect-actor-name runId:redirect-run-id -> '); + expect(logSpy.mock.calls).toEqual( + MOCKED_ACTOR_LOGS_PROCESSED.slice(1).map((item) => [loggerPrefix + item]), + ); + logSpy.mockRestore(); }); }); }); diff --git a/test/mock_server/consts.js b/test/mock_server/consts.js index e4913eb7d..b11f7672a 100644 --- a/test/mock_server/consts.js +++ b/test/mock_server/consts.js @@ -1,6 +1,5 @@ -const c = require('ansi-colors'); - -const MOCKED_ACTOR_LOGS = ['2025-05-13T07:24:12.588Z ACTOR: Pulling Docker image of build.\n', +const MOCKED_ACTOR_LOGS = [ + '2025-05-13T07:24:12.588Z ACTOR: Pulling Docker image of build.\n', '2025-05-13T07:24:12.686Z ACTOR: Creating Docker container.\n', '2025-05-13T07:24:12.745Z ACTOR: Starting Docker container.\n', // Several logs merged into one chunk Buffer.from('2025-05-13T07:26:14.132Z [apify] DEBUG \xc3', 'binary'), // Chunked log split in the middle of the multibyte character @@ -12,11 +11,11 @@ const MOCKED_ACTOR_LOGS = ['2025-05-13T07:24:12.588Z ACTOR: Pulling Docker image '7:26:14.132Z [apify] DEBUG d \n', '2025-05-13T07:27:14.132Z [apify] DEB', // Chunked log that got split outside of marker 'UG e\n', - '2025-05-13T07:28:14.132Z [apify.redirect-logger runId:4U1oAnKau6jpzjUuA] -> 2025-05-13T07:27:14.132Z ACTOR:...\n' // Already redirected message + '2025-05-13T07:28:14.132Z [apify.redirect-logger runId:4U1oAnKau6jpzjUuA] -> 2025-05-13T07:27:14.132Z ACTOR:...\n', // Already redirected message ]; - -const MOCKED_ACTOR_LOGS_PROCESSED= ['2025-05-13T07:24:12.588Z ACTOR: Pulling Docker image of build.', +const MOCKED_ACTOR_LOGS_PROCESSED = [ + '2025-05-13T07:24:12.588Z ACTOR: Pulling Docker image of build.', '2025-05-13T07:24:12.686Z ACTOR: Creating Docker container.', '2025-05-13T07:24:12.745Z ACTOR: Starting Docker container.', '2025-05-13T07:26:14.132Z [apify] DEBUG á', @@ -25,7 +24,7 @@ const MOCKED_ACTOR_LOGS_PROCESSED= ['2025-05-13T07:24:12.588Z ACTOR: Pulling Do '2025-05-13T07:26:14.132Z [apify] DEBUG c', '2025-05-13T07:26:14.132Z [apify] DEBUG d', '2025-05-13T07:27:14.132Z [apify] DEBUG e', - '2025-05-13T07:28:14.132Z [apify.redirect-logger runId:4U1oAnKau6jpzjUuA] -> 2025-05-13T07:27:14.132Z ACTOR:...' + '2025-05-13T07:28:14.132Z [apify.redirect-logger runId:4U1oAnKau6jpzjUuA] -> 2025-05-13T07:27:14.132Z ACTOR:...', ]; -module.exports = {MOCKED_ACTOR_LOGS, MOCKED_ACTOR_LOGS_PROCESSED} ; +module.exports = { MOCKED_ACTOR_LOGS, MOCKED_ACTOR_LOGS_PROCESSED }; diff --git a/test/mock_server/server.js b/test/mock_server/server.js index 561ed9920..7224fc0e8 100644 --- a/test/mock_server/server.js +++ b/test/mock_server/server.js @@ -22,7 +22,7 @@ const webhookDispatches = require('./routes/webhook_dispatches'); const webhooks = require('./routes/webhooks'); // Consts -const {MOCKED_ACTOR_LOGS} = require('./consts'); +const { MOCKED_ACTOR_LOGS } = require('./consts'); const app = express(); const v2Router = express.Router(); @@ -77,11 +77,9 @@ app.set('mockServer', mockServer); app.use('/v2', v2Router); app.use('/external', external); - - // Attaching V2 routers v2Router.use('/acts/redirect-actor-id', async (req, res) => { - res.json({'data':{'name':'redirect-actor-name'}}); + res.json({ data: { name: 'redirect-actor-name' } }); }); v2Router.use('/acts', actorRouter); v2Router.use('/actor-builds', buildRouter); @@ -91,14 +89,16 @@ v2Router.use('/actor-runs/redirect-run-id/log', async (req, res) => { res.write(chunk); res.flush(); // Flush the buffer and send the chunk immediately // Wait for a short period to simulate work being done on the server - await new Promise((resolve) => setTimeout(resolve, 10)); + await new Promise((resolve) => { + setTimeout(resolve, 10); + }); } // End the response stream once all chunks have been sent res.end(); }); v2Router.use('/actor-runs/redirect-run-id', async (req, res) => { - res.json({'data':{'id':'redirect-run-id', 'actId':'redirect-actor-id'}}); + res.json({ data: { id: 'redirect-run-id', actId: 'redirect-actor-id' } }); }); v2Router.use('/actor-runs', runRouter); v2Router.use('/actor-tasks', taskRouter); @@ -109,7 +109,9 @@ v2Router.use('/logs/redirect-log-id', async (req, res) => { res.write(chunk); res.flush(); // Flush the buffer and send the chunk immediately // Wait for a short period to simulate work being done on the server - await new Promise((resolve) => setTimeout(resolve, 10)); + await new Promise((resolve) => { + setTimeout(resolve, 10); + }); } // End the response stream once all chunks have been sent From 1c293dae36c57738063c2324b1ca5bc6fd8ae2d6 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Thu, 23 Oct 2025 14:28:14 +0200 Subject: [PATCH 10/29] Add Actor call test, 3 scenarios --- test/actors.test.js | 61 ++++++++++++++++++++++++++++++++- test/logs.test.js | 70 ++------------------------------------ test/mock_server/server.js | 4 +-- test/runs.test.js | 65 ++++++++++++++++++++++++++++++++++- 4 files changed, 128 insertions(+), 72 deletions(-) diff --git a/test/actors.test.js b/test/actors.test.js index 1ab99dc9e..0bba5c4d6 100644 --- a/test/actors.test.js +++ b/test/actors.test.js @@ -1,7 +1,10 @@ const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); -const { ActorListSortBy, ApifyClient } = require('apify-client'); +const { ActorListSortBy, ApifyClient, LoggerActorRedirect } = require('apify-client'); const { stringifyWebhooksToBase64 } = require('../src/utils'); const mockServer = require('./mock_server/server'); +const c = require('ansi-colors'); +const { MOCKED_ACTOR_LOGS_PROCESSED } = require('./mock_server/consts'); +const { Log, LEVELS } = require('@apify/log'); describe('Actor methods', () => { let baseUrl; @@ -669,3 +672,59 @@ describe('Actor methods', () => { }); }); }); + +describe('Run actor with redirected logs', () => { + let baseUrl; + + beforeAll(async () => { + const server = await mockServer.start(); + baseUrl = `http://localhost:${server.address().port}`; + }); + + let client; + beforeEach(async () => { + client = new ApifyClient({ + baseUrl, + maxRetries: 0, + ...DEFAULT_OPTIONS, + }); + }); + afterEach(async () => { + client = null; + }); + + describe('actor.call - redirected logs', () => { + test('default log', async () => { + const logSpy = jest.spyOn(LoggerActorRedirect.prototype, '_console_log').mockImplementation(() => {}); + + const defaultPrefix = 'redirect-actor-name runId:redirect-run-id -> '; + await client.actor('redirect-actor-id').call(); + + const loggerPrefix = c.cyan(defaultPrefix); + expect(logSpy.mock.calls).toEqual(MOCKED_ACTOR_LOGS_PROCESSED.map((item) => [loggerPrefix + item])); + logSpy.mockRestore(); + }); + + test('custom log', async () => { + const logSpy = jest.spyOn(LoggerActorRedirect.prototype, '_console_log').mockImplementation(() => {}); + + const customPrefix = 'custom prefix...'; + await client.actor('redirect-actor-id').call(undefined, { + log: new Log({ level: LEVELS.DEBUG, prefix: customPrefix, logger: new LoggerActorRedirect() }), + }); + + const loggerPrefix = c.cyan(customPrefix); + expect(logSpy.mock.calls).toEqual(MOCKED_ACTOR_LOGS_PROCESSED.map((item) => [loggerPrefix + item])); + logSpy.mockRestore(); + }); + + test('no log', async () => { + const logSpy = jest.spyOn(LoggerActorRedirect.prototype, '_console_log').mockImplementation(() => {}); + + await client.actor('redirect-actor-id').call(undefined, { log: null }); + + expect(logSpy.mock.calls).toEqual([]); + logSpy.mockRestore(); + }); + }); +}); diff --git a/test/logs.test.js b/test/logs.test.js index 8e7f02995..9171449a5 100644 --- a/test/logs.test.js +++ b/test/logs.test.js @@ -39,7 +39,7 @@ describe('Log methods', () => { const res = await client.log(logId).get(); expect(res).toBe('get-log'); - validateRequest({}, { logId }); + validateRequest({raw:0}, { logId }); const browserRes = await page.evaluate((id) => client.log(id).get(), logId); expect(browserRes).toEqual(res); @@ -56,73 +56,7 @@ describe('Log methods', () => { } const id = Buffer.concat(chunks).toString(); expect(id).toBe('get-log'); - validateRequest({ stream: true }, { logId }); - }); - }); -}); - -describe('Redirect logs', () => { - let baseUrl; - const browser = new Browser(); - - beforeAll(async () => { - const server = await mockServer.start(); - await browser.start(); - baseUrl = `http://localhost:${server.address().port}`; - }); - - afterAll(async () => { - await Promise.all([mockServer.close(), browser.cleanUpBrowser()]); - }); - - let client; - let page; - beforeEach(async () => { - page = await browser.getInjectedPage(baseUrl, DEFAULT_OPTIONS); - client = new ApifyClient({ - baseUrl, - maxRetries: 0, - ...DEFAULT_OPTIONS, - }); - }); - afterEach(async () => { - client = null; - page.close().catch(() => {}); - }); - - describe('run.getStreamedLog', () => { - test('getStreamedLog - fromStart', async () => { - const logSpy = jest.spyOn(LoggerActorRedirect.prototype, '_console_log').mockImplementation(() => {}); - - // Set fake time in constructor to skip the first redirected log entry, fromStart=True should redirect all logs - jest.useFakeTimers(); - jest.setSystemTime(new Date('2025-05-13T07:24:12.686Z')); - const streamedLog = await client.run('redirect-run-id').getStreamedLog(); - jest.useRealTimers(); - - await streamedLog.start(); - await streamedLog.stop(); - const loggerPrefix = c.cyan('redirect-actor-name runId:redirect-run-id -> '); - expect(logSpy.mock.calls).toEqual(MOCKED_ACTOR_LOGS_PROCESSED.map((item) => [loggerPrefix + item])); - logSpy.mockRestore(); - }); - - test('getStreamedLog - not fromStart', async () => { - const logSpy = jest.spyOn(LoggerActorRedirect.prototype, '_console_log').mockImplementation(() => {}); - - // Set fake time in constructor to skip the first redirected log entry, fromStart is redirecting only new logs - jest.useFakeTimers(); - jest.setSystemTime(new Date('2025-05-13T07:24:12.686Z')); - const streamedLog = await client.run('redirect-run-id').getStreamedLog({ fromStart: false }); - jest.useRealTimers(); - - await streamedLog.start(); - await streamedLog.stop(); - const loggerPrefix = c.cyan('redirect-actor-name runId:redirect-run-id -> '); - expect(logSpy.mock.calls).toEqual( - MOCKED_ACTOR_LOGS_PROCESSED.slice(1).map((item) => [loggerPrefix + item]), - ); - logSpy.mockRestore(); + validateRequest({ stream: true, raw:0}, { logId }); }); }); }); diff --git a/test/mock_server/server.js b/test/mock_server/server.js index 7224fc0e8..65afd755c 100644 --- a/test/mock_server/server.js +++ b/test/mock_server/server.js @@ -79,7 +79,7 @@ app.use('/external', external); // Attaching V2 routers v2Router.use('/acts/redirect-actor-id', async (req, res) => { - res.json({ data: { name: 'redirect-actor-name' } }); + res.json({ data: { name: 'redirect-actor-name', id: 'redirect-run-id' } }); }); v2Router.use('/acts', actorRouter); v2Router.use('/actor-builds', buildRouter); @@ -98,7 +98,7 @@ v2Router.use('/actor-runs/redirect-run-id/log', async (req, res) => { res.end(); }); v2Router.use('/actor-runs/redirect-run-id', async (req, res) => { - res.json({ data: { id: 'redirect-run-id', actId: 'redirect-actor-id' } }); + res.json({ data: { id: 'redirect-run-id', actId: 'redirect-actor-id', status: 'SUCCEEDED' } }); }); v2Router.use('/actor-runs', runRouter); v2Router.use('/actor-tasks', taskRouter); diff --git a/test/runs.test.js b/test/runs.test.js index f3865bd0d..1bb749da5 100644 --- a/test/runs.test.js +++ b/test/runs.test.js @@ -1,6 +1,8 @@ const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); -const { ApifyClient } = require('apify-client'); +const { ApifyClient, LoggerActorRedirect } = require('apify-client'); const mockServer = require('./mock_server/server'); +const c = require('ansi-colors'); +const { MOCKED_ACTOR_LOGS_PROCESSED } = require('./mock_server/consts'); describe('Run methods', () => { let baseUrl; @@ -377,3 +379,64 @@ describe('Run methods', () => { }); }); }); + +describe('Redirect run logs', () => { + let baseUrl; + + beforeAll(async () => { + const server = await mockServer.start(); + baseUrl = `http://localhost:${server.address().port}`; + }); + + afterAll(async () => { + await Promise.all([mockServer.close()]); + }); + + let client; + beforeEach(async () => { + client = new ApifyClient({ + baseUrl, + maxRetries: 0, + ...DEFAULT_OPTIONS, + }); + }); + afterEach(async () => { + client = null; + }); + + describe('run.getStreamedLog', () => { + test('getStreamedLog - fromStart', async () => { + const logSpy = jest.spyOn(LoggerActorRedirect.prototype, '_console_log').mockImplementation(() => {}); + + // Set fake time in constructor to skip the first redirected log entry, fromStart=True should redirect all logs + jest.useFakeTimers(); + jest.setSystemTime(new Date('2025-05-13T07:24:12.686Z')); + const streamedLog = await client.run('redirect-run-id').getStreamedLog(); + jest.useRealTimers(); + + await streamedLog.start(); + await streamedLog.stop(); + const loggerPrefix = c.cyan('redirect-actor-name runId:redirect-run-id -> '); + expect(logSpy.mock.calls).toEqual(MOCKED_ACTOR_LOGS_PROCESSED.map((item) => [loggerPrefix + item])); + logSpy.mockRestore(); + }); + + test('getStreamedLog - not fromStart', async () => { + const logSpy = jest.spyOn(LoggerActorRedirect.prototype, '_console_log').mockImplementation(() => {}); + + // Set fake time in constructor to skip the first redirected log entry, fromStart is redirecting only new logs + jest.useFakeTimers(); + jest.setSystemTime(new Date('2025-05-13T07:24:12.686Z')); + const streamedLog = await client.run('redirect-run-id').getStreamedLog({ fromStart: false }); + jest.useRealTimers(); + + await streamedLog.start(); + await streamedLog.stop(); + const loggerPrefix = c.cyan('redirect-actor-name runId:redirect-run-id -> '); + expect(logSpy.mock.calls).toEqual( + MOCKED_ACTOR_LOGS_PROCESSED.slice(1).map((item) => [loggerPrefix + item]), + ); + logSpy.mockRestore(); + }); + }); +}); From 0456c2052431b103bb9bf655be0e626972b7498c Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Thu, 23 Oct 2025 15:18:58 +0200 Subject: [PATCH 11/29] Add more comments --- src/resource_clients/log.ts | 69 +++++++++++++++++++++++-------------- test/logs.test.js | 4 +-- 2 files changed, 46 insertions(+), 27 deletions(-) diff --git a/src/resource_clients/log.ts b/src/resource_clients/log.ts index 4402c846a..dae9db5cb 100644 --- a/src/resource_clients/log.ts +++ b/src/resource_clients/log.ts @@ -26,7 +26,7 @@ export class LogClient extends ResourceClient { /** * https://docs.apify.com/api/v2#/reference/logs/log/get-log */ - async get(options = { raw: false }): Promise { + async get(options: LogOptions = {}): Promise { const requestOpts: ApifyRequestConfig = { url: this._url(), method: 'GET', @@ -47,7 +47,7 @@ export class LogClient extends ResourceClient { * Gets the log in a Readable stream format. Only works in Node.js. * https://docs.apify.com/api/v2#/reference/logs/log/get-log */ - async stream(options = { raw: false }): Promise { + async stream(options: LogOptions = {}): Promise { const params = { stream: true, raw: options.raw, @@ -71,13 +71,23 @@ export class LogClient extends ResourceClient { } } +export interface LogOptions { + /** @default false */ + raw?: boolean; +} + // Temp create it here and ask Martin where to put it const DEFAULT_OPTIONS = { + /** Whether to exclude timestamp of log redirection in redirected logs. */ skipTime: true, + /** Level of log redirection */ level: LogLevel.DEBUG, }; +/** + * Logger for redirected actor logs. + */ export class LoggerActorRedirect extends Logger { constructor(options = {}) { super({ ...DEFAULT_OPTIONS, ...options }); @@ -108,6 +118,9 @@ export class LoggerActorRedirect extends Logger { } } +/** + * Helper class for redirecting streamed Actor logs to another log. + */ export class StreamedLog { private toLog: Log; private streamBuffer: Buffer[] = []; @@ -124,6 +137,9 @@ export class StreamedLog { this.relevancyTimeLimit = fromStart ? null : new Date(); } + /** + * Start log redirection. + */ public async start(): Promise { if (this.streamingTask) { throw new Error('Streaming task already active'); @@ -133,6 +149,9 @@ export class StreamedLog { return this.streamingTask; } + /** + * Stop log redirection. + */ public async stop(): Promise { if (!this.streamingTask) { throw new Error('Streaming task is not active'); @@ -149,15 +168,9 @@ export class StreamedLog { } } - public async with(callback: () => Promise): Promise { - await this.start(); - try { - return await callback(); - } finally { - await this.stop(); - } - } - + /** + * Get log stream from response and redirect it to another log. + */ private async _streamLog(): Promise { const logStream = await this.logClient.stream({ raw: true }); if (!logStream) { @@ -165,7 +178,11 @@ export class StreamedLog { } for await (const chunk of logStream) { - this._processNewData(chunk as Buffer); + // Keep processing the new data until stopped + this.streamBuffer.push(chunk); + if (this.splitMarker.test(chunk.toString())) { + this.logBufferContent(false); + } if (this.stopLogging) { break; } @@ -175,18 +192,16 @@ export class StreamedLog { this.logBufferContent(true); } - private _processNewData(data: Buffer): void { - this.streamBuffer.push(data); - if (this.splitMarker.test(data.toString())) { - this.logBufferContent(false); - } - } - + /** + * Parse the buffer and log complete messages. + */ private logBufferContent(includeLastPart = false): void { const allParts = Buffer.concat(this.streamBuffer).toString().split(this.splitMarker).slice(1); let messageMarkers; let messageContents; + // Parse the buffer parts into complete messages if (includeLastPart) { + // This is final call, so log everything. Do not keep anything in the buffer. messageMarkers = allParts.filter((_, i) => i % 2 === 0); messageContents = allParts.filter((_, i) => i % 2 !== 0); this.streamBuffer = []; @@ -194,7 +209,7 @@ export class StreamedLog { messageMarkers = allParts.filter((_, i) => i % 2 === 0).slice(0, -1); messageContents = allParts.filter((_, i) => i % 2 !== 0).slice(0, -1); - // The last two parts (marker and message) are possibly not complete and will be left in the buffer + // The last two parts (marker and message) are possibly not complete and will be left in the buffer. this.streamBuffer = [Buffer.from(allParts.slice(-2).join(''))]; } @@ -202,24 +217,28 @@ export class StreamedLog { const decodedMarker = marker; const decodedContent = messageContents[index]; if (this.relevancyTimeLimit) { + // Log only relevant messages. Ignore too old log messages. const logTime = new Date(decodedMarker); if (logTime < this.relevancyTimeLimit) { return; } } const message = decodedMarker + decodedContent; + + // Log parsed message at guessed level. this.logAtGuessedLevel(message); }); } + /** + * Log messages at appropriate log level guessed from the message content. + * + * Original log level information does not have to be included in the message at all. + * This is methods just guesses, exotic formating or specific keywords can break the guessing logic. + */ private logAtGuessedLevel(message: string) { - // Original log level information does not have to be included in the message at all. - // This is methods just guesses, exotic formating or specific keywords can break the guessing logic. - message = message.trim(); - // TODO: Guessing can use the coloring symbols - if (message.includes('ERROR')) this.toLog.error(message); else if (message.includes('SOFT_FAIL')) this.toLog.softFail(message); else if (message.includes('WARNING')) this.toLog.warning(message); diff --git a/test/logs.test.js b/test/logs.test.js index 9171449a5..560bd0c1c 100644 --- a/test/logs.test.js +++ b/test/logs.test.js @@ -39,7 +39,7 @@ describe('Log methods', () => { const res = await client.log(logId).get(); expect(res).toBe('get-log'); - validateRequest({raw:0}, { logId }); + validateRequest({}, { logId }); const browserRes = await page.evaluate((id) => client.log(id).get(), logId); expect(browserRes).toEqual(res); @@ -56,7 +56,7 @@ describe('Log methods', () => { } const id = Buffer.concat(chunks).toString(); expect(id).toBe('get-log'); - validateRequest({ stream: true, raw:0}, { logId }); + validateRequest({ stream: true }, { logId }); }); }); }); From dc88c2bb5b120761d4187ce2ba3f4355044d4f12 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Thu, 23 Oct 2025 15:30:52 +0200 Subject: [PATCH 12/29] Remove unused import --- test/logs.test.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/logs.test.js b/test/logs.test.js index 560bd0c1c..386f986e5 100644 --- a/test/logs.test.js +++ b/test/logs.test.js @@ -1,8 +1,6 @@ const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); -const { ApifyClient, LoggerActorRedirect } = require('apify-client'); -const { MOCKED_ACTOR_LOGS_PROCESSED } = require('./mock_server/consts'); +const { ApifyClient } = require('apify-client'); const mockServer = require('./mock_server/server'); -const c = require('ansi-colors'); describe('Log methods', () => { let baseUrl; From ec0d60e3f2c0e4e412ebafbee0f8de402f936bfe Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Thu, 23 Oct 2025 15:54:37 +0200 Subject: [PATCH 13/29] Fix failing tests --- test/actors.test.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/actors.test.js b/test/actors.test.js index 0bba5c4d6..137be7cbf 100644 --- a/test/actors.test.js +++ b/test/actors.test.js @@ -254,6 +254,7 @@ describe('Actor methods', () => { timeout, build, waitSecs, + log: null, }); expect(res).toEqual(data); @@ -304,7 +305,7 @@ describe('Actor methods', () => { const maxItems = 100; mockServer.setResponse({ body }); - const res = await client.actor(actorId).call(undefined, { waitSecs, maxItems }); + const res = await client.actor(actorId).call(undefined, { waitSecs, maxItems, log: null }); expect(res).toEqual(data); validateRequest({ waitForFinish: waitSecs }, { runId }); From fc7bb6c03b968dea7e7016703d4bd8b75b3b8d43 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Thu, 23 Oct 2025 17:10:45 +0200 Subject: [PATCH 14/29] Do not run log redirection when in Browser --- src/resource_clients/actor.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/resource_clients/actor.ts b/src/resource_clients/actor.ts index 182b524aa..a1a31cda0 100644 --- a/src/resource_clients/actor.ts +++ b/src/resource_clients/actor.ts @@ -147,23 +147,24 @@ export class ActorClient extends ResourceClient { }), ); - const { waitSecs, log = 'default', ...startOptions } = options; + const { waitSecs, log, ...startOptions } = options; const { id } = await this.start(input, startOptions); // Calling root client because we need access to top level API. // Creating a new instance of RunClient here would only allow // setting it up as a nested route under actor API. const newRunClient = this.apifyClient.run(id); + let streamedLog: StreamedLog; - if (!log) { + // Log redirections is not compatible with browser environment + const isBrowser = typeof window !== 'undefined' && typeof window.document !== 'undefined'; + if (options.log === null || isBrowser) { return newRunClient.waitForFinish({ waitSecs }); } - let streamedLog: StreamedLog; - - if (log === 'default') { + if (options.log === undefined) { streamedLog = await newRunClient.getStreamedLog(); } else { - streamedLog = await newRunClient.getStreamedLog({ toLog: log }); + streamedLog = await newRunClient.getStreamedLog({ toLog: options.log }); } try { From bbeeee4cec6728c82b71f910b35435cb0dd166bb Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Mon, 27 Oct 2025 09:30:07 +0100 Subject: [PATCH 15/29] Review comments --- src/resource_clients/actor.ts | 18 +++----------- src/resource_clients/run.ts | 13 ++++++---- test/mock_server/server.js | 45 +++++++++++++---------------------- 3 files changed, 29 insertions(+), 47 deletions(-) diff --git a/src/resource_clients/actor.ts b/src/resource_clients/actor.ts index a1a31cda0..67b25edfc 100644 --- a/src/resource_clients/actor.ts +++ b/src/resource_clients/actor.ts @@ -14,7 +14,6 @@ import { ActorVersionCollectionClient } from './actor_version_collection'; import type { Build, BuildClientGetOptions } from './build'; import { BuildClient } from './build'; import { BuildCollectionClient } from './build_collection'; -import type { StreamedLog } from './log'; import { RunClient } from './run'; import { RunCollectionClient } from './run_collection'; import type { WebhookUpdateData } from './webhook'; @@ -154,24 +153,13 @@ export class ActorClient extends ResourceClient { // Creating a new instance of RunClient here would only allow // setting it up as a nested route under actor API. const newRunClient = this.apifyClient.run(id); - let streamedLog: StreamedLog; - - // Log redirections is not compatible with browser environment - const isBrowser = typeof window !== 'undefined' && typeof window.document !== 'undefined'; - if (options.log === null || isBrowser) { - return newRunClient.waitForFinish({ waitSecs }); - } - if (options.log === undefined) { - streamedLog = await newRunClient.getStreamedLog(); - } else { - streamedLog = await newRunClient.getStreamedLog({ toLog: options.log }); - } + const streamedLog = await newRunClient.getStreamedLog({ toLog: options?.log }); try { - await streamedLog.start(); + await streamedLog?.start(); return this.apifyClient.run(id).waitForFinish({ waitSecs }); } finally { - await streamedLog.stop(); + await streamedLog?.stop(); } } diff --git a/src/resource_clients/run.ts b/src/resource_clients/run.ts index ff2db2109..4d418e2bd 100644 --- a/src/resource_clients/run.ts +++ b/src/resource_clients/run.ts @@ -7,7 +7,7 @@ import { LEVELS, Log } from '@apify/log'; import type { ApiClientOptionsWithOptionalResourcePath } from '../base/api_client'; import { ResourceClient } from '../base/resource_client'; import type { ApifyResponse } from '../http_client'; -import { cast, parseDateFields, pluckData } from '../utils'; +import { cast, isNode, parseDateFields, pluckData } from '../utils'; import type { ActorRun } from './actor'; import { DatasetClient } from './dataset'; import { KeyValueStoreClient } from './key_value_store'; @@ -271,10 +271,15 @@ export class RunClient extends ResourceClient { /** * Get StreamedLog for convenient streaming of the run log and their redirection. */ - async getStreamedLog(options: GetStreamedLogOptions = {}): Promise { + async getStreamedLog(options: GetStreamedLogOptions = {}): Promise { const { fromStart = true } = options; let { toLog } = options; - if (!toLog) { + if (toLog === null || !isNode()) { + // Explicitly no logging or not in Node.js + return undefined; + } + if (toLog === undefined) { + // Create default StreamedLog // Get actor name and run id const runData = await this.get(); const runId = runData ? `${runData.id ?? ''}` : ''; @@ -293,7 +298,7 @@ export class RunClient extends ResourceClient { } export interface GetStreamedLogOptions { - toLog?: Log; + toLog?: Log | null; fromStart?: boolean; } diff --git a/test/mock_server/server.js b/test/mock_server/server.js index 65afd755c..0d6ecc06d 100644 --- a/test/mock_server/server.js +++ b/test/mock_server/server.js @@ -58,6 +58,21 @@ const mockServer = { }, }; +async function streamLogChunks(req, res) { + // Asynchronously write each chunk to the response stream + for (const chunk of MOCKED_ACTOR_LOGS) { + res.write(chunk); + res.flush(); // Flush the buffer and send the chunk immediately + // Wait for a short period to simulate work being done on the server + await new Promise((resolve) => { + setTimeout(resolve, 10); + }); + } + + // End the response stream once all chunks have been sent + res.end(); +} + // Debugging middleware app.use((req, res, next) => { next(); @@ -83,40 +98,14 @@ v2Router.use('/acts/redirect-actor-id', async (req, res) => { }); v2Router.use('/acts', actorRouter); v2Router.use('/actor-builds', buildRouter); -v2Router.use('/actor-runs/redirect-run-id/log', async (req, res) => { - // Asynchronously write each chunk to the response stream - for (const chunk of MOCKED_ACTOR_LOGS) { - res.write(chunk); - res.flush(); // Flush the buffer and send the chunk immediately - // Wait for a short period to simulate work being done on the server - await new Promise((resolve) => { - setTimeout(resolve, 10); - }); - } - - // End the response stream once all chunks have been sent - res.end(); -}); +v2Router.use('/actor-runs/redirect-run-id/log', streamLogChunks); v2Router.use('/actor-runs/redirect-run-id', async (req, res) => { res.json({ data: { id: 'redirect-run-id', actId: 'redirect-actor-id', status: 'SUCCEEDED' } }); }); v2Router.use('/actor-runs', runRouter); v2Router.use('/actor-tasks', taskRouter); v2Router.use('/users', userRouter); -v2Router.use('/logs/redirect-log-id', async (req, res) => { - // Asynchronously write each chunk to the response stream - for (const chunk of MOCKED_ACTOR_LOGS) { - res.write(chunk); - res.flush(); // Flush the buffer and send the chunk immediately - // Wait for a short period to simulate work being done on the server - await new Promise((resolve) => { - setTimeout(resolve, 10); - }); - } - - // End the response stream once all chunks have been sent - res.end(); -}); +v2Router.use('/logs/redirect-log-id', streamLogChunks); v2Router.use('/logs', logRouter); v2Router.use('/datasets', datasetRouter); v2Router.use('/key-value-stores', keyValueStores); From 00bf7ea391db4aa7ce3d6110d703024f9e9fe184 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Mon, 27 Oct 2025 10:11:47 +0100 Subject: [PATCH 16/29] Update tests to pas on Node 16 --- test/actors.test.js | 2 ++ test/runs.test.js | 2 ++ 2 files changed, 4 insertions(+) diff --git a/test/actors.test.js b/test/actors.test.js index 137be7cbf..f379a1d2e 100644 --- a/test/actors.test.js +++ b/test/actors.test.js @@ -678,6 +678,8 @@ describe('Run actor with redirected logs', () => { let baseUrl; beforeAll(async () => { + // Ensure that the tests that use characters like á are correctly decoded in console. + process.stdout.setDefaultEncoding('utf8'); const server = await mockServer.start(); baseUrl = `http://localhost:${server.address().port}`; }); diff --git a/test/runs.test.js b/test/runs.test.js index 1bb749da5..d590d4777 100644 --- a/test/runs.test.js +++ b/test/runs.test.js @@ -384,6 +384,8 @@ describe('Redirect run logs', () => { let baseUrl; beforeAll(async () => { + // Ensure that the tests that use characters like á are correctly decoded in console. + process.stdout.setDefaultEncoding('utf8'); const server = await mockServer.start(); baseUrl = `http://localhost:${server.address().port}`; }); From 5fd0fc77b3c5d0bb9827c27eed6e1432d4acc85f Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Wed, 29 Oct 2025 09:12:38 +0100 Subject: [PATCH 17/29] Properlyhnadle multibyte characters that are split by chunking --- src/resource_clients/log.ts | 43 +++++++++++++++++++++++++++++++++---- test/mock_server/consts.js | 10 +++++++-- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/src/resource_clients/log.ts b/src/resource_clients/log.ts index dae9db5cb..89502f0f3 100644 --- a/src/resource_clients/log.ts +++ b/src/resource_clients/log.ts @@ -145,7 +145,7 @@ export class StreamedLog { throw new Error('Streaming task already active'); } this.stopLogging = false; - this.streamingTask = this._streamLog(); + this.streamingTask = this.streamLog(); return this.streamingTask; } @@ -171,16 +171,51 @@ export class StreamedLog { /** * Get log stream from response and redirect it to another log. */ - private async _streamLog(): Promise { + private async streamLog(): Promise { + const twoBytesLimit = 0xc0; + const threeBytesLimit = 0xe0; + const fourBytesLimit = 0xf0; + const logStream = await this.logClient.stream({ raw: true }); if (!logStream) { return; } + let incompleteCharacter: Uint8Array = new Uint8Array(); for await (const chunk of logStream) { + // Handle possible leftover incomplete multibyte character from previous chunk + const chunkPrependedWithIncompleteCharacter = new Uint8Array(incompleteCharacter.length + chunk.length); + chunkPrependedWithIncompleteCharacter.set(incompleteCharacter, 0); + chunkPrependedWithIncompleteCharacter.set(chunk, incompleteCharacter.length); + + // Extract possible incomplete multibyte character from the end of this chunk + if ( + chunkPrependedWithIncompleteCharacter.length > 1 && + chunkPrependedWithIncompleteCharacter[chunkPrependedWithIncompleteCharacter.length] >= twoBytesLimit + ) { + incompleteCharacter = chunkPrependedWithIncompleteCharacter.slice( + chunkPrependedWithIncompleteCharacter.length - 1, + ); + } else if ( + chunkPrependedWithIncompleteCharacter.length > 2 && + chunkPrependedWithIncompleteCharacter[chunkPrependedWithIncompleteCharacter.length] >= threeBytesLimit + ) { + incompleteCharacter = chunkPrependedWithIncompleteCharacter.slice( + chunkPrependedWithIncompleteCharacter.length - 2, + ); + } else if ( + chunkPrependedWithIncompleteCharacter.length > 3 && + chunkPrependedWithIncompleteCharacter[chunkPrependedWithIncompleteCharacter.length] >= fourBytesLimit + ) { + incompleteCharacter = chunkPrependedWithIncompleteCharacter.slice( + chunkPrependedWithIncompleteCharacter.length - 3, + ); + } + // Keep processing the new data until stopped - this.streamBuffer.push(chunk); - if (this.splitMarker.test(chunk.toString())) { + this.streamBuffer.push(Buffer.from(chunkPrependedWithIncompleteCharacter)); + // Log data only if the chunk contains the marker as it indicates previous message is complete + if (this.splitMarker.test(chunkPrependedWithIncompleteCharacter.toString())) { this.logBufferContent(false); } if (this.stopLogging) { diff --git a/test/mock_server/consts.js b/test/mock_server/consts.js index b11f7672a..f7f0c6370 100644 --- a/test/mock_server/consts.js +++ b/test/mock_server/consts.js @@ -2,8 +2,12 @@ const MOCKED_ACTOR_LOGS = [ '2025-05-13T07:24:12.588Z ACTOR: Pulling Docker image of build.\n', '2025-05-13T07:24:12.686Z ACTOR: Creating Docker container.\n', '2025-05-13T07:24:12.745Z ACTOR: Starting Docker container.\n', // Several logs merged into one chunk - Buffer.from('2025-05-13T07:26:14.132Z [apify] DEBUG \xc3', 'binary'), // Chunked log split in the middle of the multibyte character - Buffer.from([0xa1, 0x0a]), + Buffer.from('2025-05-13T07:26:14.132Z [apify] DEBUG \xc3', 'binary'), // Chunked log split in the middle of the 2-byte character + Buffer.from('\xa1\x0a', 'binary'), + Buffer.from('2025-05-13T07:26:14.132Z [apify] DEBUG \xE2', 'binary'), // Chunked log split in the middle of the 3-byte character + Buffer.from('\x82\xAC\x0a', 'binary'), + Buffer.from('2025-05-13T07:26:14.132Z [apify] DEBUG \xF0\x9F', 'binary'), // Chunked log split in the middle of the4-byte character + Buffer.from('\x98\x80\x0a', 'binary'), '2025-05-13T07:24:14.132Z [apify] INFO multiline \n log\n', '2025-05-13T07:25:14.132Z [apify] WARNING some warning\n', '2025-05-13T07:26:14.132Z [apify] DEBUG c\n', @@ -19,6 +23,8 @@ const MOCKED_ACTOR_LOGS_PROCESSED = [ '2025-05-13T07:24:12.686Z ACTOR: Creating Docker container.', '2025-05-13T07:24:12.745Z ACTOR: Starting Docker container.', '2025-05-13T07:26:14.132Z [apify] DEBUG á', + '2025-05-13T07:26:14.132Z [apify] DEBUG €', + '2025-05-13T07:26:14.132Z [apify] DEBUG 😀', '2025-05-13T07:24:14.132Z [apify] INFO multiline \n log', '2025-05-13T07:25:14.132Z [apify] WARNING some warning', '2025-05-13T07:26:14.132Z [apify] DEBUG c', From 5d4c1f4631a98c2e5a7d210ba524421abeace620 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Wed, 29 Oct 2025 11:17:07 +0100 Subject: [PATCH 18/29] Fix awaiting logs without calling run --- src/resource_clients/actor.ts | 4 +++- test/actors.test.js | 3 ++- test/mock_server/consts.js | 44 ++++++++++++++++++++++++++++++++++- test/mock_server/server.js | 5 ++-- test/runs.test.js | 14 +++++++++-- 5 files changed, 63 insertions(+), 7 deletions(-) diff --git a/src/resource_clients/actor.ts b/src/resource_clients/actor.ts index 67b25edfc..e81db62db 100644 --- a/src/resource_clients/actor.ts +++ b/src/resource_clients/actor.ts @@ -155,11 +155,13 @@ export class ActorClient extends ResourceClient { const newRunClient = this.apifyClient.run(id); const streamedLog = await newRunClient.getStreamedLog({ toLog: options?.log }); + let streamingPromise: Promise | undefined; try { - await streamedLog?.start(); + streamingPromise = streamedLog?.start(); return this.apifyClient.run(id).waitForFinish({ waitSecs }); } finally { await streamedLog?.stop(); + await streamingPromise; } } diff --git a/test/actors.test.js b/test/actors.test.js index f379a1d2e..a6f61ad5d 100644 --- a/test/actors.test.js +++ b/test/actors.test.js @@ -3,7 +3,7 @@ const { ActorListSortBy, ApifyClient, LoggerActorRedirect } = require('apify-cli const { stringifyWebhooksToBase64 } = require('../src/utils'); const mockServer = require('./mock_server/server'); const c = require('ansi-colors'); -const { MOCKED_ACTOR_LOGS_PROCESSED } = require('./mock_server/consts'); +const { MOCKED_ACTOR_LOGS_PROCESSED, statusGenerator } = require('./mock_server/consts'); const { Log, LEVELS } = require('@apify/log'); describe('Actor methods', () => { @@ -686,6 +686,7 @@ describe('Run actor with redirected logs', () => { let client; beforeEach(async () => { + statusGenerator.reset(); client = new ApifyClient({ baseUrl, maxRetries: 0, diff --git a/test/mock_server/consts.js b/test/mock_server/consts.js index f7f0c6370..e2b50ef90 100644 --- a/test/mock_server/consts.js +++ b/test/mock_server/consts.js @@ -33,4 +33,46 @@ const MOCKED_ACTOR_LOGS_PROCESSED = [ '2025-05-13T07:28:14.132Z [apify.redirect-logger runId:4U1oAnKau6jpzjUuA] -> 2025-05-13T07:27:14.132Z ACTOR:...', ]; -module.exports = { MOCKED_ACTOR_LOGS, MOCKED_ACTOR_LOGS_PROCESSED }; +const MOCKED_ACTOR_STATUSES = [ + ['RUNNING', 'Actor Started'], + ['RUNNING', 'Doing some stuff'], + ['RUNNING', 'Doing some stuff'], + ['RUNNING', 'Doing some stuff'], + ['RUNNING', 'Doing some stuff'], + ['RUNNING', 'Doing some stuff'], + ['RUNNING', 'Doing some stuff'], + ['RUNNING', 'Doing some stuff'], + ['RUNNING', 'Doing some stuff'], + ['SUCCEEDED', 'Actor Finished'], +]; + +/** + * Helper class to allow iterating over defined list of statuses for each test case. + */ +class StatusGenerator { + constructor() { + this.reset(); + } + + reset() { + this.generator = (() => { + // Iterate over MOCKED_ACTOR_STATUSES and keep returning the last status when exhausted + let i = 0; + return () => { + if (i >= MOCKED_ACTOR_STATUSES.length) { + return MOCKED_ACTOR_STATUSES[MOCKED_ACTOR_STATUSES.length - 1]; + } + return MOCKED_ACTOR_STATUSES[i++]; + }; + })(); + } + + next() { + return this.generator(); + } +} + +// Test can call statusGenerator.reset() to receive the statuses from the start +const statusGenerator = new StatusGenerator(); + +module.exports = { MOCKED_ACTOR_LOGS, MOCKED_ACTOR_LOGS_PROCESSED, MOCKED_ACTOR_STATUSES, statusGenerator }; diff --git a/test/mock_server/server.js b/test/mock_server/server.js index 0d6ecc06d..ff6b35c70 100644 --- a/test/mock_server/server.js +++ b/test/mock_server/server.js @@ -22,7 +22,7 @@ const webhookDispatches = require('./routes/webhook_dispatches'); const webhooks = require('./routes/webhooks'); // Consts -const { MOCKED_ACTOR_LOGS } = require('./consts'); +const { MOCKED_ACTOR_LOGS, statusGenerator } = require('./consts'); const app = express(); const v2Router = express.Router(); @@ -100,7 +100,8 @@ v2Router.use('/acts', actorRouter); v2Router.use('/actor-builds', buildRouter); v2Router.use('/actor-runs/redirect-run-id/log', streamLogChunks); v2Router.use('/actor-runs/redirect-run-id', async (req, res) => { - res.json({ data: { id: 'redirect-run-id', actId: 'redirect-actor-id', status: 'SUCCEEDED' } }); + const [status, statusMessage] = statusGenerator.next(); + res.json({ data: { id: 'redirect-run-id', actId: 'redirect-actor-id', status, statusMessage } }); }); v2Router.use('/actor-runs', runRouter); v2Router.use('/actor-tasks', taskRouter); diff --git a/test/runs.test.js b/test/runs.test.js index d590d4777..d44daba70 100644 --- a/test/runs.test.js +++ b/test/runs.test.js @@ -416,8 +416,13 @@ describe('Redirect run logs', () => { const streamedLog = await client.run('redirect-run-id').getStreamedLog(); jest.useRealTimers(); - await streamedLog.start(); + streamedLog.start(); + // Wait some time to accumulate logs + await new Promise((resolve) => { + setTimeout(resolve, 10); + }); await streamedLog.stop(); + const loggerPrefix = c.cyan('redirect-actor-name runId:redirect-run-id -> '); expect(logSpy.mock.calls).toEqual(MOCKED_ACTOR_LOGS_PROCESSED.map((item) => [loggerPrefix + item])); logSpy.mockRestore(); @@ -432,8 +437,13 @@ describe('Redirect run logs', () => { const streamedLog = await client.run('redirect-run-id').getStreamedLog({ fromStart: false }); jest.useRealTimers(); - await streamedLog.start(); + streamedLog.start(); + // Wait some time to accumulate logs + await new Promise((resolve) => { + setTimeout(resolve, 10); + }); await streamedLog.stop(); + const loggerPrefix = c.cyan('redirect-actor-name runId:redirect-run-id -> '); expect(logSpy.mock.calls).toEqual( MOCKED_ACTOR_LOGS_PROCESSED.slice(1).map((item) => [loggerPrefix + item]), From 51dd246f4acaec011b2b42074423e95b1a2f8038 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Wed, 29 Oct 2025 15:07:27 +0100 Subject: [PATCH 19/29] Make it possible to set response sequence per each test if needed --- src/resource_clients/actor.ts | 5 +- test/_helper.js | 2 +- test/actors.test.js | 31 +++++-- test/apify_api_error.test.js | 2 +- test/builds.test.js | 2 +- test/datasets.test.js | 2 +- test/http_client.test.js | 2 +- test/key_value_stores.test.js | 2 +- test/logs.test.js | 2 +- test/mock_server/consts.js | 27 +++--- test/mock_server/server.js | 142 ++++++++++++++++---------------- test/request_queues.test.js | 2 +- test/runs.test.js | 6 +- test/schedules.test.js | 2 +- test/store.test.ts | 2 +- test/tasks.test.js | 2 +- test/users.test.js | 2 +- test/webhook_dispatches.test.js | 2 +- test/webhooks.test.js | 2 +- 19 files changed, 129 insertions(+), 110 deletions(-) diff --git a/src/resource_clients/actor.ts b/src/resource_clients/actor.ts index e81db62db..dfa9e8965 100644 --- a/src/resource_clients/actor.ts +++ b/src/resource_clients/actor.ts @@ -156,10 +156,13 @@ export class ActorClient extends ResourceClient { const streamedLog = await newRunClient.getStreamedLog({ toLog: options?.log }); let streamingPromise: Promise | undefined; + let actorRun: Promise | undefined; try { streamingPromise = streamedLog?.start(); - return this.apifyClient.run(id).waitForFinish({ waitSecs }); + actorRun = this.apifyClient.run(id).waitForFinish({ waitSecs }); + return actorRun; } finally { + await actorRun; await streamedLog?.stop(); await streamingPromise; } diff --git a/test/_helper.js b/test/_helper.js index 61f68e9cb..951ecb38f 100644 --- a/test/_helper.js +++ b/test/_helper.js @@ -1,6 +1,6 @@ const { launchPuppeteer, puppeteerUtils } = require('@crawlee/puppeteer'); -const mockServer = require('./mock_server/server'); +const { mockServer } = require('./mock_server/server'); class Browser { async start() { diff --git a/test/actors.test.js b/test/actors.test.js index a6f61ad5d..5a57c609f 100644 --- a/test/actors.test.js +++ b/test/actors.test.js @@ -1,10 +1,11 @@ const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); const { ActorListSortBy, ApifyClient, LoggerActorRedirect } = require('apify-client'); const { stringifyWebhooksToBase64 } = require('../src/utils'); -const mockServer = require('./mock_server/server'); +const { mockServer, createDefaultApp } = require('./mock_server/server'); const c = require('ansi-colors'); -const { MOCKED_ACTOR_LOGS_PROCESSED, statusGenerator } = require('./mock_server/consts'); +const { MOCKED_ACTOR_LOGS_PROCESSED, StatusGenerator } = require('./mock_server/consts'); const { Log, LEVELS } = require('@apify/log'); +const express = require('express'); describe('Actor methods', () => { let baseUrl; @@ -676,17 +677,31 @@ describe('Actor methods', () => { describe('Run actor with redirected logs', () => { let baseUrl; + let client; + const statusGenerator = new StatusGenerator(); beforeAll(async () => { - // Ensure that the tests that use characters like á are correctly decoded in console. - process.stdout.setDefaultEncoding('utf8'); - const server = await mockServer.start(); + // Use custom router for the tests + const router = express.Router(); + // Set up a status generator to simulate run status changes. It will be reset for each test. + router.get('/actor-runs/redirect-run-id', async (req, res) => { + // Delay the response to give the actor time to run and produce expected logs + await new Promise((resolve) => { + setTimeout(resolve, 10); + }); + const [status, statusMessage] = statusGenerator.next().value; + res.json({ data: { id: 'redirect-run-id', actId: 'redirect-actor-id', status, statusMessage } }); + }); + const app = createDefaultApp(router); + const server = await mockServer.start(undefined, app); baseUrl = `http://localhost:${server.address().port}`; }); - let client; + afterAll(async () => { + await Promise.all([mockServer.close()]); + }); + beforeEach(async () => { - statusGenerator.reset(); client = new ApifyClient({ baseUrl, maxRetries: 0, @@ -694,6 +709,8 @@ describe('Run actor with redirected logs', () => { }); }); afterEach(async () => { + // Reset the generator to so that the next test starts fresh + statusGenerator.reset(); client = null; }); diff --git a/test/apify_api_error.test.js b/test/apify_api_error.test.js index 2d086bb74..7f60e813a 100644 --- a/test/apify_api_error.test.js +++ b/test/apify_api_error.test.js @@ -1,5 +1,5 @@ const { Browser, DEFAULT_OPTIONS } = require('./_helper'); -const mockServer = require('./mock_server/server'); +const { mockServer } = require('./mock_server/server'); const { ApifyClient } = require('apify-client'); describe('ApifyApiError', () => { diff --git a/test/builds.test.js b/test/builds.test.js index fa9627fb5..44310231e 100644 --- a/test/builds.test.js +++ b/test/builds.test.js @@ -1,6 +1,6 @@ const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); const { ApifyClient } = require('apify-client'); -const mockServer = require('./mock_server/server'); +const { mockServer } = require('./mock_server/server'); describe('Build methods', () => { let baseUrl; diff --git a/test/datasets.test.js b/test/datasets.test.js index e9c1d160e..e698f47a9 100644 --- a/test/datasets.test.js +++ b/test/datasets.test.js @@ -1,6 +1,6 @@ const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); const { ApifyClient } = require('apify-client'); -const mockServer = require('./mock_server/server'); +const { mockServer } = require('./mock_server/server'); describe('Dataset methods', () => { let baseUrl; diff --git a/test/http_client.test.js b/test/http_client.test.js index db7c5e0d6..c9c70bff0 100644 --- a/test/http_client.test.js +++ b/test/http_client.test.js @@ -1,5 +1,5 @@ const { Browser } = require('./_helper'); -const mockServer = require('./mock_server/server'); +const { mockServer } = require('./mock_server/server'); const { ApifyClient } = require('apify-client'); describe('HttpClient', () => { diff --git a/test/key_value_stores.test.js b/test/key_value_stores.test.js index 7a4ba98dc..201da1af6 100644 --- a/test/key_value_stores.test.js +++ b/test/key_value_stores.test.js @@ -2,7 +2,7 @@ const { Readable } = require('node:stream'); const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); const { ApifyClient } = require('apify-client'); -const mockServer = require('./mock_server/server'); +const { mockServer } = require('./mock_server/server'); describe('Key-Value Store methods', () => { let baseUrl; diff --git a/test/logs.test.js b/test/logs.test.js index 386f986e5..03a60b34d 100644 --- a/test/logs.test.js +++ b/test/logs.test.js @@ -1,6 +1,6 @@ const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); const { ApifyClient } = require('apify-client'); -const mockServer = require('./mock_server/server'); +const { mockServer } = require('./mock_server/server'); describe('Log methods', () => { let baseUrl; diff --git a/test/mock_server/consts.js b/test/mock_server/consts.js index e2b50ef90..6296059eb 100644 --- a/test/mock_server/consts.js +++ b/test/mock_server/consts.js @@ -55,24 +55,21 @@ class StatusGenerator { } reset() { - this.generator = (() => { - // Iterate over MOCKED_ACTOR_STATUSES and keep returning the last status when exhausted - let i = 0; - return () => { - if (i >= MOCKED_ACTOR_STATUSES.length) { - return MOCKED_ACTOR_STATUSES[MOCKED_ACTOR_STATUSES.length - 1]; - } - return MOCKED_ACTOR_STATUSES[i++]; - }; - })(); + function* getStatusGenerator() { + for (const status of MOCKED_ACTOR_STATUSES) { + yield status; + } + // After exhausting, keep yielding the last status + while (true) { + yield MOCKED_ACTOR_STATUSES[MOCKED_ACTOR_STATUSES.length - 1]; + } + } + this.generator = getStatusGenerator(); } next() { - return this.generator(); + return this.generator.next(); } } -// Test can call statusGenerator.reset() to receive the statuses from the start -const statusGenerator = new StatusGenerator(); - -module.exports = { MOCKED_ACTOR_LOGS, MOCKED_ACTOR_LOGS_PROCESSED, MOCKED_ACTOR_STATUSES, statusGenerator }; +module.exports = { MOCKED_ACTOR_LOGS, MOCKED_ACTOR_LOGS_PROCESSED, MOCKED_ACTOR_STATUSES, StatusGenerator }; diff --git a/test/mock_server/server.js b/test/mock_server/server.js index ff6b35c70..f934321c6 100644 --- a/test/mock_server/server.js +++ b/test/mock_server/server.js @@ -22,14 +22,15 @@ const webhookDispatches = require('./routes/webhook_dispatches'); const webhooks = require('./routes/webhooks'); // Consts -const { MOCKED_ACTOR_LOGS, statusGenerator } = require('./consts'); +const { MOCKED_ACTOR_LOGS } = require('./consts'); + +const defaultApp = createDefaultApp(); -const app = express(); -const v2Router = express.Router(); const mockServer = { requests: [], response: null, - async start(port = 0) { + async start(port = 0, app = defaultApp) { + app.set('mockServer', this); this.server = http.createServer(app); return new Promise((resolve, reject) => { this.server.on('error', reject); @@ -58,76 +59,77 @@ const mockServer = { }, }; -async function streamLogChunks(req, res) { - // Asynchronously write each chunk to the response stream - for (const chunk of MOCKED_ACTOR_LOGS) { - res.write(chunk); - res.flush(); // Flush the buffer and send the chunk immediately - // Wait for a short period to simulate work being done on the server - await new Promise((resolve) => { - setTimeout(resolve, 10); - }); - } +function createDefaultApp(v2Router = express.Router()) { + async function streamLogChunks(req, res) { + // Asynchronously write each chunk to the response stream + for (const chunk of MOCKED_ACTOR_LOGS) { + res.write(chunk); + res.flush(); // Flush the buffer and send the chunk immediately + // Wait for a short period to simulate work being done on the server + await new Promise((resolve) => { + setTimeout(resolve, 1); + }); + } - // End the response stream once all chunks have been sent - res.end(); -} - -// Debugging middleware -app.use((req, res, next) => { - next(); -}); -app.use(express.text()); -app.use(express.json({ limit: '9mb' })); -app.use(express.urlencoded({ extended: false })); -app.use(bodyParser.raw()); -app.use(express.static(path.join(__dirname, 'public'))); -app.use(compression()); + // End the response stream once all chunks have been sent + res.end(); + } + const app = express(); + // Debugging middleware + app.use((req, res, next) => { + next(); + }); + app.use(express.text()); + app.use(express.json({ limit: '9mb' })); + app.use(express.urlencoded({ extended: false })); + app.use(bodyParser.raw()); + app.use(express.static(path.join(__dirname, 'public'))); + app.use(compression()); + app.use('/', (req, res, next) => { + mockServer.requests.push(req); + next(); + }); + app.use('/v2', v2Router); + app.use('/external', external); -app.use('/', (req, res, next) => { - mockServer.requests.push(req); - next(); -}); -app.set('mockServer', mockServer); -app.use('/v2', v2Router); -app.use('/external', external); + // Attaching V2 routers + v2Router.use('/acts/redirect-actor-id', async (req, res) => { + res.json({ data: { name: 'redirect-actor-name', id: 'redirect-run-id' } }); + }); + v2Router.use('/acts', actorRouter); + v2Router.use('/actor-builds', buildRouter); + v2Router.use('/actor-runs/redirect-run-id/log', streamLogChunks); + v2Router.use('/actor-runs/redirect-run-id', async (req, res) => { + res.json({ data: { id: 'redirect-run-id', actId: 'redirect-actor-id', status: 'SUCCEEDED' } }); + }); -// Attaching V2 routers -v2Router.use('/acts/redirect-actor-id', async (req, res) => { - res.json({ data: { name: 'redirect-actor-name', id: 'redirect-run-id' } }); -}); -v2Router.use('/acts', actorRouter); -v2Router.use('/actor-builds', buildRouter); -v2Router.use('/actor-runs/redirect-run-id/log', streamLogChunks); -v2Router.use('/actor-runs/redirect-run-id', async (req, res) => { - const [status, statusMessage] = statusGenerator.next(); - res.json({ data: { id: 'redirect-run-id', actId: 'redirect-actor-id', status, statusMessage } }); -}); -v2Router.use('/actor-runs', runRouter); -v2Router.use('/actor-tasks', taskRouter); -v2Router.use('/users', userRouter); -v2Router.use('/logs/redirect-log-id', streamLogChunks); -v2Router.use('/logs', logRouter); -v2Router.use('/datasets', datasetRouter); -v2Router.use('/key-value-stores', keyValueStores); -v2Router.use('/request-queues', requestQueues); -v2Router.use('/webhooks', webhooks); -v2Router.use('/schedules', schedules); -v2Router.use('/webhook-dispatches', webhookDispatches); -v2Router.use('/store', store); + v2Router.use('/actor-runs', runRouter); + v2Router.use('/actor-tasks', taskRouter); + v2Router.use('/users', userRouter); + v2Router.use('/logs/redirect-log-id', streamLogChunks); + v2Router.use('/logs', logRouter); + v2Router.use('/datasets', datasetRouter); + v2Router.use('/key-value-stores', keyValueStores); + v2Router.use('/request-queues', requestQueues); + v2Router.use('/webhooks', webhooks); + v2Router.use('/schedules', schedules); + v2Router.use('/webhook-dispatches', webhookDispatches); + v2Router.use('/store', store); -// Debugging middleware -app.use((err, req, res, _next) => { - res.status(500).json({ error: { message: err.message } }); -}); + // Debugging middleware + app.use((err, req, res, _next) => { + res.status(500).json({ error: { message: err.message } }); + }); -app.use((req, res) => { - res.status(404).json({ - error: { - type: 'page-not-found', - message: 'Nothing to do here.', - }, + app.use((req, res) => { + res.status(404).json({ + error: { + type: 'page-not-found', + message: 'Nothing to do here.', + }, + }); }); -}); + return app; +} -module.exports = mockServer; +module.exports = { mockServer, createDefaultApp }; diff --git a/test/request_queues.test.js b/test/request_queues.test.js index 06739f0d9..2834a5059 100644 --- a/test/request_queues.test.js +++ b/test/request_queues.test.js @@ -1,6 +1,6 @@ const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); const { ApifyClient } = require('apify-client'); -const mockServer = require('./mock_server/server'); +const { mockServer } = require('./mock_server/server'); describe('Request Queue methods', () => { let baseUrl; diff --git a/test/runs.test.js b/test/runs.test.js index d44daba70..fddf3d925 100644 --- a/test/runs.test.js +++ b/test/runs.test.js @@ -1,6 +1,6 @@ const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); const { ApifyClient, LoggerActorRedirect } = require('apify-client'); -const mockServer = require('./mock_server/server'); +const { mockServer } = require('./mock_server/server'); const c = require('ansi-colors'); const { MOCKED_ACTOR_LOGS_PROCESSED } = require('./mock_server/consts'); @@ -419,7 +419,7 @@ describe('Redirect run logs', () => { streamedLog.start(); // Wait some time to accumulate logs await new Promise((resolve) => { - setTimeout(resolve, 10); + setTimeout(resolve, 1000); }); await streamedLog.stop(); @@ -440,7 +440,7 @@ describe('Redirect run logs', () => { streamedLog.start(); // Wait some time to accumulate logs await new Promise((resolve) => { - setTimeout(resolve, 10); + setTimeout(resolve, 1000); }); await streamedLog.stop(); diff --git a/test/schedules.test.js b/test/schedules.test.js index 498c6869f..8427a564c 100644 --- a/test/schedules.test.js +++ b/test/schedules.test.js @@ -1,6 +1,6 @@ const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); const { ApifyClient } = require('apify-client'); -const mockServer = require('./mock_server/server'); +const { mockServer } = require('./mock_server/server'); describe('Schedule methods', () => { let baseUrl; diff --git a/test/store.test.ts b/test/store.test.ts index 5e5592a8c..daa07da96 100644 --- a/test/store.test.ts +++ b/test/store.test.ts @@ -2,7 +2,7 @@ import type { StoreCollectionListOptions } from 'apify-client'; import { ApifyClient } from 'apify-client'; import { Browser, DEFAULT_OPTIONS, validateRequest } from './_helper'; -import mockServer from './mock_server/server'; +import { mockServer } from './mock_server/server'; describe('Store', () => { let baseUrl: string | undefined; diff --git a/test/tasks.test.js b/test/tasks.test.js index 75c808cec..f151cce33 100644 --- a/test/tasks.test.js +++ b/test/tasks.test.js @@ -1,6 +1,6 @@ const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); const { ApifyClient } = require('apify-client'); -const mockServer = require('./mock_server/server'); +const { mockServer } = require('./mock_server/server'); const { stringifyWebhooksToBase64 } = require('../src/utils'); describe('Task methods', () => { diff --git a/test/users.test.js b/test/users.test.js index 7fb18f0c9..42c5a25ee 100644 --- a/test/users.test.js +++ b/test/users.test.js @@ -2,7 +2,7 @@ const { ME_USER_NAME_PLACEHOLDER } = require('@apify/consts'); const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); const { ApifyClient } = require('apify-client'); -const mockServer = require('./mock_server/server'); +const { mockServer } = require('./mock_server/server'); describe('User methods', () => { let baseUrl; diff --git a/test/webhook_dispatches.test.js b/test/webhook_dispatches.test.js index 7b735b204..fb5c2dc25 100644 --- a/test/webhook_dispatches.test.js +++ b/test/webhook_dispatches.test.js @@ -1,6 +1,6 @@ const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); const { ApifyClient } = require('apify-client'); -const mockServer = require('./mock_server/server'); +const { mockServer } = require('./mock_server/server'); describe('Webhook Dispatch methods', () => { let baseUrl; diff --git a/test/webhooks.test.js b/test/webhooks.test.js index bf2c33cae..0ec5486d6 100644 --- a/test/webhooks.test.js +++ b/test/webhooks.test.js @@ -1,6 +1,6 @@ const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); const { ApifyClient } = require('apify-client'); -const mockServer = require('./mock_server/server'); +const { mockServer } = require('./mock_server/server'); describe('Webhook methods', () => { let baseUrl; From 71d7fb541eeb2f9f895152dc18317309b9d0f5e3 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Mon, 3 Nov 2025 15:43:50 +0100 Subject: [PATCH 20/29] Review comments --- src/resource_clients/actor.ts | 4 +--- src/resource_clients/log.ts | 12 +++++----- test/actors.test.js | 6 ++--- test/runs.test.js | 41 ++++++++++------------------------- 4 files changed, 20 insertions(+), 43 deletions(-) diff --git a/src/resource_clients/actor.ts b/src/resource_clients/actor.ts index dfa9e8965..15214b92c 100644 --- a/src/resource_clients/actor.ts +++ b/src/resource_clients/actor.ts @@ -155,16 +155,14 @@ export class ActorClient extends ResourceClient { const newRunClient = this.apifyClient.run(id); const streamedLog = await newRunClient.getStreamedLog({ toLog: options?.log }); - let streamingPromise: Promise | undefined; let actorRun: Promise | undefined; try { - streamingPromise = streamedLog?.start(); + streamedLog?.start(); actorRun = this.apifyClient.run(id).waitForFinish({ waitSecs }); return actorRun; } finally { await actorRun; await streamedLog?.stop(); - await streamingPromise; } } diff --git a/src/resource_clients/log.ts b/src/resource_clients/log.ts index 89502f0f3..f0113bba3 100644 --- a/src/resource_clients/log.ts +++ b/src/resource_clients/log.ts @@ -93,10 +93,6 @@ export class LoggerActorRedirect extends Logger { super({ ...DEFAULT_OPTIONS, ...options }); } - _console_log(line: string) { - console.log(line); // eslint-disable-line no-console - } - override _log(level: LogLevel, message: string, data?: any, exception?: unknown, opts: Record = {}) { if (level > this.options.level) { return; @@ -113,7 +109,10 @@ export class LoggerActorRedirect extends Logger { } const line = `${c.gray(maybeDate)}${c.cyan(prefix)}${message || ''}`; - this._console_log(line); + + // All redirected logs are logged at info level to avid any console specific formating for non-info levels, + // which have already been applied once to the original log. (For example error stack traces etc.) + this._outputWithConsole(LogLevel.INFO, line); return line; } } @@ -140,13 +139,12 @@ export class StreamedLog { /** * Start log redirection. */ - public async start(): Promise { + public start(): void { if (this.streamingTask) { throw new Error('Streaming task already active'); } this.stopLogging = false; this.streamingTask = this.streamLog(); - return this.streamingTask; } /** diff --git a/test/actors.test.js b/test/actors.test.js index 5a57c609f..e5d125c37 100644 --- a/test/actors.test.js +++ b/test/actors.test.js @@ -716,7 +716,7 @@ describe('Run actor with redirected logs', () => { describe('actor.call - redirected logs', () => { test('default log', async () => { - const logSpy = jest.spyOn(LoggerActorRedirect.prototype, '_console_log').mockImplementation(() => {}); + const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); const defaultPrefix = 'redirect-actor-name runId:redirect-run-id -> '; await client.actor('redirect-actor-id').call(); @@ -727,7 +727,7 @@ describe('Run actor with redirected logs', () => { }); test('custom log', async () => { - const logSpy = jest.spyOn(LoggerActorRedirect.prototype, '_console_log').mockImplementation(() => {}); + const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); const customPrefix = 'custom prefix...'; await client.actor('redirect-actor-id').call(undefined, { @@ -740,7 +740,7 @@ describe('Run actor with redirected logs', () => { }); test('no log', async () => { - const logSpy = jest.spyOn(LoggerActorRedirect.prototype, '_console_log').mockImplementation(() => {}); + const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); await client.actor('redirect-actor-id').call(undefined, { log: null }); diff --git a/test/runs.test.js b/test/runs.test.js index fddf3d925..0896e8951 100644 --- a/test/runs.test.js +++ b/test/runs.test.js @@ -1,5 +1,5 @@ const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); -const { ApifyClient, LoggerActorRedirect } = require('apify-client'); +const { ApifyClient } = require('apify-client'); const { mockServer } = require('./mock_server/server'); const c = require('ansi-colors'); const { MOCKED_ACTOR_LOGS_PROCESSED } = require('./mock_server/consts'); @@ -406,35 +406,18 @@ describe('Redirect run logs', () => { client = null; }); - describe('run.getStreamedLog', () => { - test('getStreamedLog - fromStart', async () => { - const logSpy = jest.spyOn(LoggerActorRedirect.prototype, '_console_log').mockImplementation(() => {}); - - // Set fake time in constructor to skip the first redirected log entry, fromStart=True should redirect all logs - jest.useFakeTimers(); - jest.setSystemTime(new Date('2025-05-13T07:24:12.686Z')); - const streamedLog = await client.run('redirect-run-id').getStreamedLog(); - jest.useRealTimers(); + const testCases = [ + { fromStart: true, expected: MOCKED_ACTOR_LOGS_PROCESSED }, + { fromStart: false, expected: MOCKED_ACTOR_LOGS_PROCESSED.slice(1) }, + ]; - streamedLog.start(); - // Wait some time to accumulate logs - await new Promise((resolve) => { - setTimeout(resolve, 1000); - }); - await streamedLog.stop(); - - const loggerPrefix = c.cyan('redirect-actor-name runId:redirect-run-id -> '); - expect(logSpy.mock.calls).toEqual(MOCKED_ACTOR_LOGS_PROCESSED.map((item) => [loggerPrefix + item])); - logSpy.mockRestore(); - }); - - test('getStreamedLog - not fromStart', async () => { - const logSpy = jest.spyOn(LoggerActorRedirect.prototype, '_console_log').mockImplementation(() => {}); - - // Set fake time in constructor to skip the first redirected log entry, fromStart is redirecting only new logs + describe('run.getStreamedLog', () => { + test.each(testCases)('getStreamedLog $fromStart', async ({ fromStart, expected }) => { + const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); + // Set fake time in constructor to skip the first redirected log entry// fromStart=True should redirect all logs jest.useFakeTimers(); jest.setSystemTime(new Date('2025-05-13T07:24:12.686Z')); - const streamedLog = await client.run('redirect-run-id').getStreamedLog({ fromStart: false }); + const streamedLog = await client.run('redirect-run-id').getStreamedLog({ fromStart }); jest.useRealTimers(); streamedLog.start(); @@ -445,9 +428,7 @@ describe('Redirect run logs', () => { await streamedLog.stop(); const loggerPrefix = c.cyan('redirect-actor-name runId:redirect-run-id -> '); - expect(logSpy.mock.calls).toEqual( - MOCKED_ACTOR_LOGS_PROCESSED.slice(1).map((item) => [loggerPrefix + item]), - ); + expect(logSpy.mock.calls).toEqual(expected.map((item) => [loggerPrefix + item])); logSpy.mockRestore(); }); }); From 0b2e6e6cabde4b65818c09e633bb79b5d4f689b9 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Tue, 4 Nov 2025 16:19:32 +0100 Subject: [PATCH 21/29] Simplify handling of multibyte chars --- src/resource_clients/log.ts | 75 ++++++++----------------------------- 1 file changed, 16 insertions(+), 59 deletions(-) diff --git a/src/resource_clients/log.ts b/src/resource_clients/log.ts index f0113bba3..92a2647d2 100644 --- a/src/resource_clients/log.ts +++ b/src/resource_clients/log.ts @@ -170,52 +170,27 @@ export class StreamedLog { * Get log stream from response and redirect it to another log. */ private async streamLog(): Promise { - const twoBytesLimit = 0xc0; - const threeBytesLimit = 0xe0; - const fourBytesLimit = 0xf0; - const logStream = await this.logClient.stream({ raw: true }); if (!logStream) { return; } - let incompleteCharacter: Uint8Array = new Uint8Array(); + let previousChunkRemainder: Uint8Array = new Uint8Array(); for await (const chunk of logStream) { - // Handle possible leftover incomplete multibyte character from previous chunk - const chunkPrependedWithIncompleteCharacter = new Uint8Array(incompleteCharacter.length + chunk.length); - chunkPrependedWithIncompleteCharacter.set(incompleteCharacter, 0); - chunkPrependedWithIncompleteCharacter.set(chunk, incompleteCharacter.length); - - // Extract possible incomplete multibyte character from the end of this chunk - if ( - chunkPrependedWithIncompleteCharacter.length > 1 && - chunkPrependedWithIncompleteCharacter[chunkPrependedWithIncompleteCharacter.length] >= twoBytesLimit - ) { - incompleteCharacter = chunkPrependedWithIncompleteCharacter.slice( - chunkPrependedWithIncompleteCharacter.length - 1, - ); - } else if ( - chunkPrependedWithIncompleteCharacter.length > 2 && - chunkPrependedWithIncompleteCharacter[chunkPrependedWithIncompleteCharacter.length] >= threeBytesLimit - ) { - incompleteCharacter = chunkPrependedWithIncompleteCharacter.slice( - chunkPrependedWithIncompleteCharacter.length - 2, - ); - } else if ( - chunkPrependedWithIncompleteCharacter.length > 3 && - chunkPrependedWithIncompleteCharacter[chunkPrependedWithIncompleteCharacter.length] >= fourBytesLimit - ) { - incompleteCharacter = chunkPrependedWithIncompleteCharacter.slice( - chunkPrependedWithIncompleteCharacter.length - 3, - ); - } + // Handle possible leftover incomplete line from previous chunk. + // Everything before last end of line is complete. + const chunkWithPreviousRemainder = new Uint8Array(previousChunkRemainder.length + chunk.length); + chunkWithPreviousRemainder.set(previousChunkRemainder, 0); + chunkWithPreviousRemainder.set(chunk, previousChunkRemainder.length); + + const lastCompleteMessageIndex = chunkWithPreviousRemainder.lastIndexOf(0x0a); + previousChunkRemainder = chunkWithPreviousRemainder.slice(lastCompleteMessageIndex); + + // Push complete part of the chunk to the buffer + this.streamBuffer.push(Buffer.from(chunkWithPreviousRemainder.slice(0, lastCompleteMessageIndex))); + this.logBufferContent(false); // Keep processing the new data until stopped - this.streamBuffer.push(Buffer.from(chunkPrependedWithIncompleteCharacter)); - // Log data only if the chunk contains the marker as it indicates previous message is complete - if (this.splitMarker.test(chunkPrependedWithIncompleteCharacter.toString())) { - this.logBufferContent(false); - } if (this.stopLogging) { break; } @@ -258,27 +233,9 @@ export class StreamedLog { } const message = decodedMarker + decodedContent; - // Log parsed message at guessed level. - this.logAtGuessedLevel(message); + // Original log level information is not available. Log all on info level. Log level could be guessed for + // some logs, but for any multiline logs such guess would be probably correct only for the first line. + this.toLog.info(message.trim()); }); } - - /** - * Log messages at appropriate log level guessed from the message content. - * - * Original log level information does not have to be included in the message at all. - * This is methods just guesses, exotic formating or specific keywords can break the guessing logic. - */ - private logAtGuessedLevel(message: string) { - message = message.trim(); - - if (message.includes('ERROR')) this.toLog.error(message); - else if (message.includes('SOFT_FAIL')) this.toLog.softFail(message); - else if (message.includes('WARNING')) this.toLog.warning(message); - else if (message.includes('INFO')) this.toLog.info(message); - else if (message.includes('DEBUG')) this.toLog.debug(message); - else if (message.includes('PERF')) this.toLog.perf(message); - // Fallback in case original log message does not indicate known log level. - else this.toLog.info(message); - } } From 0440e39397692991688992c2a52bab2eebed9960 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Wed, 5 Nov 2025 15:25:24 +0100 Subject: [PATCH 22/29] Basic review feedback --- src/resource_clients/run.ts | 4 ++-- test/actors.test.js | 2 +- test/mock_server/server.js | 2 +- test/mock_server/{consts.js => test_utils.js} | 0 test/runs.test.js | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) rename test/mock_server/{consts.js => test_utils.js} (100%) diff --git a/src/resource_clients/run.ts b/src/resource_clients/run.ts index 4d418e2bd..cd3a78374 100644 --- a/src/resource_clients/run.ts +++ b/src/resource_clients/run.ts @@ -282,12 +282,12 @@ export class RunClient extends ResourceClient { // Create default StreamedLog // Get actor name and run id const runData = await this.get(); - const runId = runData ? `${runData.id ?? ''}` : ''; + const runId = runData?.id ?? ''; const actorId = runData?.actId ?? ''; const actorData = (await this.apifyClient.actor(actorId).get()) || { name: '' }; - const actorName = runData ? (actorData.name ?? '') : ''; + const actorName = actorData?.name ?? ''; const name = [actorName, `runId:${runId}`].filter(Boolean).join(' '); toLog = new Log({ level: LEVELS.DEBUG, prefix: `${name} -> `, logger: new LoggerActorRedirect() }); diff --git a/test/actors.test.js b/test/actors.test.js index e5d125c37..698145735 100644 --- a/test/actors.test.js +++ b/test/actors.test.js @@ -3,7 +3,7 @@ const { ActorListSortBy, ApifyClient, LoggerActorRedirect } = require('apify-cli const { stringifyWebhooksToBase64 } = require('../src/utils'); const { mockServer, createDefaultApp } = require('./mock_server/server'); const c = require('ansi-colors'); -const { MOCKED_ACTOR_LOGS_PROCESSED, StatusGenerator } = require('./mock_server/consts'); +const { MOCKED_ACTOR_LOGS_PROCESSED, StatusGenerator } = require('./mock_server/test_utils'); const { Log, LEVELS } = require('@apify/log'); const express = require('express'); diff --git a/test/mock_server/server.js b/test/mock_server/server.js index f934321c6..f9d1b60e0 100644 --- a/test/mock_server/server.js +++ b/test/mock_server/server.js @@ -22,7 +22,7 @@ const webhookDispatches = require('./routes/webhook_dispatches'); const webhooks = require('./routes/webhooks'); // Consts -const { MOCKED_ACTOR_LOGS } = require('./consts'); +const { MOCKED_ACTOR_LOGS } = require('./test_utils'); const defaultApp = createDefaultApp(); diff --git a/test/mock_server/consts.js b/test/mock_server/test_utils.js similarity index 100% rename from test/mock_server/consts.js rename to test/mock_server/test_utils.js diff --git a/test/runs.test.js b/test/runs.test.js index 0896e8951..6c3276877 100644 --- a/test/runs.test.js +++ b/test/runs.test.js @@ -2,7 +2,7 @@ const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); const { ApifyClient } = require('apify-client'); const { mockServer } = require('./mock_server/server'); const c = require('ansi-colors'); -const { MOCKED_ACTOR_LOGS_PROCESSED } = require('./mock_server/consts'); +const { MOCKED_ACTOR_LOGS_PROCESSED } = require('./mock_server/test_utils'); describe('Run methods', () => { let baseUrl; @@ -412,7 +412,7 @@ describe('Redirect run logs', () => { ]; describe('run.getStreamedLog', () => { - test.each(testCases)('getStreamedLog $fromStart', async ({ fromStart, expected }) => { + test.each(testCases)('getStreamedLog fromStart:$fromStart', async ({ fromStart, expected }) => { const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); // Set fake time in constructor to skip the first redirected log entry// fromStart=True should redirect all logs jest.useFakeTimers(); From 0b84a326eba1eb04cad28490e26fdb668259fca7 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Wed, 5 Nov 2025 15:54:48 +0100 Subject: [PATCH 23/29] Simplify loging and add more docs --- src/resource_clients/actor.ts | 8 ++++++++ src/resource_clients/log.ts | 29 ++++++++++------------------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/src/resource_clients/actor.ts b/src/resource_clients/actor.ts index 15214b92c..9ec86207d 100644 --- a/src/resource_clients/actor.ts +++ b/src/resource_clients/actor.ts @@ -438,7 +438,15 @@ export interface ActorStartOptions { } export interface ActorCallOptions extends Omit { + /** + * Wait time in seconds for the actor run to finish. + */ waitSecs?: number; + /** + * `Log` instance that should be used to redirect actor run logs to. + * If `undefined` default `Log` will be created and used. + * If `null`, no log redirection will occur. + */ log?: Log | null; } diff --git a/src/resource_clients/log.ts b/src/resource_clients/log.ts index 92a2647d2..67bdd57ab 100644 --- a/src/resource_clients/log.ts +++ b/src/resource_clients/log.ts @@ -188,38 +188,29 @@ export class StreamedLog { // Push complete part of the chunk to the buffer this.streamBuffer.push(Buffer.from(chunkWithPreviousRemainder.slice(0, lastCompleteMessageIndex))); - this.logBufferContent(false); + this.logBufferContent(); // Keep processing the new data until stopped if (this.stopLogging) { break; } } - - // Process the remaining buffer - this.logBufferContent(true); + // Process whatever is left when exiting. Maybe it is incomplete, maybe it is last line without EOL. + const lastMessage = Buffer.from(previousChunkRemainder).toString().trim(); + if (lastMessage.length) { + this.toLog.info(lastMessage); + } } /** * Parse the buffer and log complete messages. */ - private logBufferContent(includeLastPart = false): void { + private logBufferContent(): void { const allParts = Buffer.concat(this.streamBuffer).toString().split(this.splitMarker).slice(1); - let messageMarkers; - let messageContents; // Parse the buffer parts into complete messages - if (includeLastPart) { - // This is final call, so log everything. Do not keep anything in the buffer. - messageMarkers = allParts.filter((_, i) => i % 2 === 0); - messageContents = allParts.filter((_, i) => i % 2 !== 0); - this.streamBuffer = []; - } else { - messageMarkers = allParts.filter((_, i) => i % 2 === 0).slice(0, -1); - messageContents = allParts.filter((_, i) => i % 2 !== 0).slice(0, -1); - - // The last two parts (marker and message) are possibly not complete and will be left in the buffer. - this.streamBuffer = [Buffer.from(allParts.slice(-2).join(''))]; - } + const messageMarkers = allParts.filter((_, i) => i % 2 === 0); + const messageContents = allParts.filter((_, i) => i % 2 !== 0); + this.streamBuffer = []; messageMarkers.forEach((marker, index) => { const decodedMarker = marker; From bca27b23b83fdf95b59cf5b2acfed00496eb476b Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Wed, 5 Nov 2025 16:10:52 +0100 Subject: [PATCH 24/29] Cleanup using finally chain --- src/resource_clients/actor.ts | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/resource_clients/actor.ts b/src/resource_clients/actor.ts index 9ec86207d..c84db32df 100644 --- a/src/resource_clients/actor.ts +++ b/src/resource_clients/actor.ts @@ -155,15 +155,13 @@ export class ActorClient extends ResourceClient { const newRunClient = this.apifyClient.run(id); const streamedLog = await newRunClient.getStreamedLog({ toLog: options?.log }); - let actorRun: Promise | undefined; - try { - streamedLog?.start(); - actorRun = this.apifyClient.run(id).waitForFinish({ waitSecs }); - return actorRun; - } finally { - await actorRun; - await streamedLog?.stop(); - } + streamedLog?.start(); + return this.apifyClient + .run(id) + .waitForFinish({ waitSecs }) + .finally(async () => { + await streamedLog?.stop(); + }); } /** From 8eb6e7159b40fdcb1b9bf0f7e693905ce249c2ee Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Thu, 6 Nov 2025 09:19:57 +0100 Subject: [PATCH 25/29] Use setTimeout from node --- test/actors.test.js | 6 +++--- test/runs.test.js | 5 ++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/test/actors.test.js b/test/actors.test.js index 698145735..eea9f6191 100644 --- a/test/actors.test.js +++ b/test/actors.test.js @@ -6,6 +6,7 @@ const c = require('ansi-colors'); const { MOCKED_ACTOR_LOGS_PROCESSED, StatusGenerator } = require('./mock_server/test_utils'); const { Log, LEVELS } = require('@apify/log'); const express = require('express'); +const { setTimeout } = require('node:timers/promises'); describe('Actor methods', () => { let baseUrl; @@ -686,9 +687,8 @@ describe('Run actor with redirected logs', () => { // Set up a status generator to simulate run status changes. It will be reset for each test. router.get('/actor-runs/redirect-run-id', async (req, res) => { // Delay the response to give the actor time to run and produce expected logs - await new Promise((resolve) => { - setTimeout(resolve, 10); - }); + await setTimeout(10); + const [status, statusMessage] = statusGenerator.next().value; res.json({ data: { id: 'redirect-run-id', actId: 'redirect-actor-id', status, statusMessage } }); }); diff --git a/test/runs.test.js b/test/runs.test.js index 6c3276877..c8a050323 100644 --- a/test/runs.test.js +++ b/test/runs.test.js @@ -3,6 +3,7 @@ const { ApifyClient } = require('apify-client'); const { mockServer } = require('./mock_server/server'); const c = require('ansi-colors'); const { MOCKED_ACTOR_LOGS_PROCESSED } = require('./mock_server/test_utils'); +const { setTimeout: setTimeoutNode } = require('node:timers/promises'); describe('Run methods', () => { let baseUrl; @@ -422,9 +423,7 @@ describe('Redirect run logs', () => { streamedLog.start(); // Wait some time to accumulate logs - await new Promise((resolve) => { - setTimeout(resolve, 1000); - }); + await setTimeoutNode(1000); await streamedLog.stop(); const loggerPrefix = c.cyan('redirect-actor-name runId:redirect-run-id -> '); From dfa5dcd2df8f26d6f4e3d9f8b658961c771e571e Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Tue, 11 Nov 2025 10:38:22 +0100 Subject: [PATCH 26/29] Review comments --- src/resource_clients/log.ts | 45 +++++++++++++++++++++---------------- src/resource_clients/run.ts | 2 +- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/src/resource_clients/log.ts b/src/resource_clients/log.ts index 67bdd57ab..c77b43bb5 100644 --- a/src/resource_clients/log.ts +++ b/src/resource_clients/log.ts @@ -76,21 +76,12 @@ export interface LogOptions { raw?: boolean; } -// Temp create it here and ask Martin where to put it - -const DEFAULT_OPTIONS = { - /** Whether to exclude timestamp of log redirection in redirected logs. */ - skipTime: true, - /** Level of log redirection */ - level: LogLevel.DEBUG, -}; - /** * Logger for redirected actor logs. */ export class LoggerActorRedirect extends Logger { constructor(options = {}) { - super({ ...DEFAULT_OPTIONS, ...options }); + super({ skipTime: true, level: LogLevel.DEBUG, ...options }); } override _log(level: LogLevel, message: string, data?: any, exception?: unknown, opts: Record = {}) { @@ -121,7 +112,7 @@ export class LoggerActorRedirect extends Logger { * Helper class for redirecting streamed Actor logs to another log. */ export class StreamedLog { - private toLog: Log; + private destinationLog: Log; private streamBuffer: Buffer[] = []; private splitMarker = /(?:\n|^)(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z)/g; private relevancyTimeLimit: Date | null; @@ -130,8 +121,9 @@ export class StreamedLog { private streamingTask: Promise | null = null; private stopLogging = false; - constructor(logClient: LogClient, toLog: Log, fromStart = true) { - this.toLog = toLog; + constructor(options: StreamedLogOptions) { + const { toLog, logClient, fromStart = true } = options; + this.destinationLog = toLog; this.logClient = logClient; this.relevancyTimeLimit = fromStart ? null : new Date(); } @@ -174,6 +166,16 @@ export class StreamedLog { if (!logStream) { return; } + const lastChunkRemainder = await this.logStreamChunks(logStream); + // Process whatever is left when exiting. Maybe it is incomplete, maybe it is last log without EOL. + const lastMessage = Buffer.from(lastChunkRemainder).toString().trim(); + if (lastMessage.length) { + this.destinationLog.info(lastMessage); + } + } + + private async logStreamChunks(logStream: Readable): Promise { + // Chunk may be incomplete. Keep remainder for next chunk. let previousChunkRemainder: Uint8Array = new Uint8Array(); for await (const chunk of logStream) { @@ -195,11 +197,7 @@ export class StreamedLog { break; } } - // Process whatever is left when exiting. Maybe it is incomplete, maybe it is last line without EOL. - const lastMessage = Buffer.from(previousChunkRemainder).toString().trim(); - if (lastMessage.length) { - this.toLog.info(lastMessage); - } + return previousChunkRemainder; } /** @@ -226,7 +224,16 @@ export class StreamedLog { // Original log level information is not available. Log all on info level. Log level could be guessed for // some logs, but for any multiline logs such guess would be probably correct only for the first line. - this.toLog.info(message.trim()); + this.destinationLog.info(message.trim()); }); } } + +export interface StreamedLogOptions { + /** Log client used to communicate with the Apify API. */ + logClient: LogClient; + /** Log to which the Actor run logs will be redirected. */ + toLog: Log; + /** Whether to redirect all logs from Actor run start (even logs from the past). */ + fromStart?: boolean; +} diff --git a/src/resource_clients/run.ts b/src/resource_clients/run.ts index cd3a78374..f077ce026 100644 --- a/src/resource_clients/run.ts +++ b/src/resource_clients/run.ts @@ -293,7 +293,7 @@ export class RunClient extends ResourceClient { toLog = new Log({ level: LEVELS.DEBUG, prefix: `${name} -> `, logger: new LoggerActorRedirect() }); } - return new StreamedLog(this.log(), toLog, fromStart); + return new StreamedLog({ logClient: this.log(), toLog, fromStart }); } } From 3e48c600dee90550bbdbc501b490ef9cd1925ce1 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Tue, 11 Nov 2025 11:14:48 +0100 Subject: [PATCH 27/29] Add `default` alternative to `undefined` --- src/resource_clients/actor.ts | 6 +++--- src/resource_clients/run.ts | 4 ++-- test/actors.test.js | 11 +++++++++++ 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/resource_clients/actor.ts b/src/resource_clients/actor.ts index c84db32df..9650ae4ea 100644 --- a/src/resource_clients/actor.ts +++ b/src/resource_clients/actor.ts @@ -140,7 +140,7 @@ export class ActorClient extends ResourceClient { webhooks: ow.optional.array.ofType(ow.object), maxItems: ow.optional.number.not.negative, maxTotalChargeUsd: ow.optional.number.not.negative, - log: ow.optional.any(ow.null, ow.object.instanceOf(Log)), + log: ow.optional.any(ow.null, ow.object.instanceOf(Log), ow.string.equals('default')), restartOnError: ow.optional.boolean, forcePermissionLevel: ow.optional.string.oneOf(Object.values(ACTOR_PERMISSION_LEVEL)), }), @@ -442,10 +442,10 @@ export interface ActorCallOptions extends Omit { logSpy.mockRestore(); }); + test('explicit default log', async () => { + const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); + + const defaultPrefix = 'redirect-actor-name runId:redirect-run-id -> '; + await client.actor('redirect-actor-id').call(undefined, { log: 'default' }); + + const loggerPrefix = c.cyan(defaultPrefix); + expect(logSpy.mock.calls).toEqual(MOCKED_ACTOR_LOGS_PROCESSED.map((item) => [loggerPrefix + item])); + logSpy.mockRestore(); + }); + test('custom log', async () => { const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); From f560b3b73f94c8d10c67acb975bf42b598389b64 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Wed, 19 Nov 2025 12:55:31 +0100 Subject: [PATCH 28/29] Test review comments --- test/actors.test.js | 47 +++++++++++++++------------------------------ 1 file changed, 16 insertions(+), 31 deletions(-) diff --git a/test/actors.test.js b/test/actors.test.js index 407241091..ada2468b7 100644 --- a/test/actors.test.js +++ b/test/actors.test.js @@ -698,7 +698,7 @@ describe('Run actor with redirected logs', () => { }); afterAll(async () => { - await Promise.all([mockServer.close()]); + await mockServer.close(); }); beforeEach(async () => { @@ -714,43 +714,28 @@ describe('Run actor with redirected logs', () => { client = null; }); - describe('actor.call - redirected logs', () => { - test('default log', async () => { - const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); - - const defaultPrefix = 'redirect-actor-name runId:redirect-run-id -> '; - await client.actor('redirect-actor-id').call(); - - const loggerPrefix = c.cyan(defaultPrefix); - expect(logSpy.mock.calls).toEqual(MOCKED_ACTOR_LOGS_PROCESSED.map((item) => [loggerPrefix + item])); - logSpy.mockRestore(); - }); - - test('explicit default log', async () => { - const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); - - const defaultPrefix = 'redirect-actor-name runId:redirect-run-id -> '; - await client.actor('redirect-actor-id').call(undefined, { log: 'default' }); + const testCases = [ + { expectedPrefix: c.cyan('redirect-actor-name runId:redirect-run-id -> '), logOptions: {} }, + { expectedPrefix: c.cyan('redirect-actor-name runId:redirect-run-id -> '), logOptions: { log: 'default' } }, + { + expectedPrefix: c.cyan('custom prefix...'), + logOptions: { + log: new Log({ level: LEVELS.DEBUG, prefix: 'custom prefix...', logger: new LoggerActorRedirect() }), + }, + }, + ]; - const loggerPrefix = c.cyan(defaultPrefix); - expect(logSpy.mock.calls).toEqual(MOCKED_ACTOR_LOGS_PROCESSED.map((item) => [loggerPrefix + item])); - logSpy.mockRestore(); - }); - - test('custom log', async () => { + describe('actor.call - redirected logs', () => { + test.each(testCases)('logOptions:$logOptions', async ({ expectedPrefix, logOptions }) => { const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); - const customPrefix = 'custom prefix...'; - await client.actor('redirect-actor-id').call(undefined, { - log: new Log({ level: LEVELS.DEBUG, prefix: customPrefix, logger: new LoggerActorRedirect() }), - }); + await client.actor('redirect-actor-id').call(undefined, logOptions); - const loggerPrefix = c.cyan(customPrefix); - expect(logSpy.mock.calls).toEqual(MOCKED_ACTOR_LOGS_PROCESSED.map((item) => [loggerPrefix + item])); + expect(logSpy.mock.calls).toEqual(MOCKED_ACTOR_LOGS_PROCESSED.map((item) => [expectedPrefix + item])); logSpy.mockRestore(); }); - test('no log', async () => { + test('logOptions:{ "log": null }', async () => { const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); await client.actor('redirect-actor-id').call(undefined, { log: null }); From bc2f1e2837b2c2c2fc265c53024cdd781ca87679 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Wed, 19 Nov 2025 13:07:29 +0100 Subject: [PATCH 29/29] Remove test related tmp edit --- package.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/package.json b/package.json index 4d98d92cf..1b5df2d2c 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,10 @@ "url": "https://github.com/apify/apify-client-js/issues" }, "homepage": "https://docs.apify.com/api/client/js/", + "files": [ + "dist", + "!dist/*.tsbuildinfo" + ], "scripts": { "build": "npm run clean && npm run build:node && npm run build:browser", "postbuild": "gen-esm-wrapper dist/index.js dist/index.mjs",