diff --git a/apps/render-farm/src/worker.test.ts b/apps/render-farm/src/worker.test.ts index a9e9e99c025..39d144ece20 100644 --- a/apps/render-farm/src/worker.test.ts +++ b/apps/render-farm/src/worker.test.ts @@ -2,7 +2,12 @@ import { describe, expect, test } from "bun:test"; import { randomUUID } from "node:crypto"; import { readFileSync } from "node:fs"; import { join } from "node:path"; -import { MIN_PART, type TranscodeTask, type WorkItem } from "./protocol"; +import { + MIN_PART, + type TranscodeTask, + type VideoTask, + type WorkItem, +} from "./protocol"; import * as stitch from "./stitch"; import * as transcode from "./transcode"; @@ -26,6 +31,8 @@ function harness( randomUUID, join, ...transcode, + processCpuSeconds: async () => 0, + sampleThreads: async () => ({}), availableParallelism: () => 1, hostname: () => "worker-test", mkdirSync: () => {}, @@ -34,6 +41,13 @@ function harness( s3ConfigFromEnv: () => ({}), ...stitch, MIN_PART, + ProjectCache: class { + root = "/work/job"; + async materialize() { + return { bytes: 0, ms: 0, requests: 0 }; + } + close() {} + }, S3: class { async presignFresh() { await options.presignGate; @@ -97,8 +111,18 @@ function harness( const compiled = new Bun.Transpiler({ loader: "ts" }).transformSync(source); const worker = new Function( ...Object.keys(deps), - `${compiled}\nreturn {runTranscode, cancel, busy, progress, transcoders, engineEnv, uploadLayout};`, + `${compiled}\nreturn {runVideo, runTranscode, cancel, busy, progress, transcoders, engineEnv, uploadLayout};`, )(...Object.values(deps)) as { + runVideo: ( + task: VideoTask, + engine: { + pid: number; + onEvent: ((event: Record) => void) | null; + onProgress: ((frames: number) => void) | null; + request: () => Promise; + }, + queuedMs: number, + ) => Promise; runTranscode: (task: TranscodeTask, slot: number) => Promise; cancel: (taskIds: string[]) => void; busy: Map; @@ -227,3 +251,41 @@ describe("chunk upload", () => { }); }); }); + +test("a failed video render removes its temporary output", async () => { + const worker = harness(); + const task: VideoTask = { + kind: "video", + taskId: "job:v0", + jobId: "job", + chunk: 0, + fps: 30, + resolution: [1920, 1080], + bpp: 0.1, + frames: [0, 30], + threads: 1, + files: [], + upload: { + key: "out/job.mp4", + uploadId: "upload", + firstPart: 2, + partLimit: 3, + partTarget: MIN_PART, + stashKey: "stash/job/0", + }, + audio: null, + hls: null, + }; + const engine = { + pid: 1, + onEvent: null, + onProgress: null, + request: () => Promise.reject(new Error("render failed")), + }; + await expect(worker.runVideo(task, engine, 0)).rejects.toThrow( + "render failed", + ); + expect(worker.removed).toHaveLength(1); + expect(worker.removed[0]).toStartWith("/work/job/v0-"); + expect(worker.removed[0]).toEndWith(".h264"); +}); diff --git a/apps/render-farm/src/worker.ts b/apps/render-farm/src/worker.ts index 4de9dfdbf24..e6e129f212b 100644 --- a/apps/render-farm/src/worker.ts +++ b/apps/render-farm/src/worker.ts @@ -505,110 +505,116 @@ async function runVideo( cache.root, `v${task.chunk}-${randomUUID().slice(0, 6)}.h264`, ); - const engineStarted = performance.now(); - const slot = engines.indexOf(engine); - const entry = { - taskId: task.taskId, - kind: "video" as const, - frames: 0, - total: task.frames[1] - task.frames[0], - startedAt: Date.now(), - lastProgressAt: Date.now(), - }; - progress.set(slot, entry); - const stream = task.hls - ? new SegmentStream({ ...task, hls: task.hls }, out, engineStarted) - : null; - engine.onEvent = stream ? (event) => stream.onEvent(event) : null; - // Ask for the next task once this one is ~PREFETCH_LEAD_MS from done, so - // its sources download while this one finishes instead of after. - engine.onProgress = (frames) => { - entry.frames = frames; - entry.lastProgressAt = Date.now(); - const elapsed = performance.now() - engineStarted; - const remainingMs = - ((entry.total - frames) * elapsed) / Math.max(1, frames); - if (frames > 0 && remainingMs <= PREFETCH_LEAD_MS) nearEnd(); - }; - const request = engine.request<{ - width: number; - height: number; - sizes: number[]; - keyframes: number[]; - extradata: string; - bytes: number; - timings: Record; - }>("video", { - project: cache.root, - fps: task.fps, - resolution: task.resolution, - bpp: task.bpp, - frames: task.frames, - threads: task.threads, - out, - }); - const threadsPromise: Promise> = - process.env.RF_PROFILE_THREADS === "1" - ? sampleThreads(engine.pid, request) - : Promise.resolve({}); - const result = await request.finally(() => { - engine.onProgress = null; - engine.onEvent = null; - progress.delete(slot); - }); - nearEnd(); - const segmentsDone = stream?.finish(result.sizes, result.extradata); - segmentsDone?.catch(() => {}); - const engineMs = performance.now() - engineStarted; - const threads = await threadsPromise; - const waitStarted = performance.now(); - const audio = await audioPromise; - const audioWaitMs = performance.now() - waitStarted; - const uploadStarted = performance.now(); - const plan = layout(task, out, result.sizes, audio); - const [{ parts, stash }, segments] = await Promise.all([ - uploadLayout(task, plan.segments, plan.bytes), - segmentsDone ?? Promise.resolve(0), - ]); - const uploadMs = performance.now() - uploadStarted; - rmSync(out, { force: true }); - const timings: TaskTimings = { - queuedMs, - fetch: fetchStats, - engine: { - ...result.timings, - ...Object.fromEntries( - Object.entries(threads).map(([name, value]) => [ - `thread:${name}`, - value, - ]), - ), - ...(stream - ? { segments, firstSegmentMs: Math.round(stream.firstUploadedMs ?? -1) } - : {}), - }, - engineMs, - audioWaitMs, - uploadMs, - totalMs: performance.now() - started, - cpuSeconds: (await processCpuSeconds(engine.pid)) - cpuBefore, - }; - return { - taskId: task.taskId, - worker: WORKER_ID, - sizes: result.sizes, - keyframes: result.keyframes, - extradata: result.extradata, - width: result.width, - height: result.height, - videoRuns: plan.videoRuns, - audioRuns: plan.audioRuns, - firstPart: task.upload.firstPart, - stash, - parts, - bytes: plan.bytes, - timings, - }; + try { + const engineStarted = performance.now(); + const slot = engines.indexOf(engine); + const entry = { + taskId: task.taskId, + kind: "video" as const, + frames: 0, + total: task.frames[1] - task.frames[0], + startedAt: Date.now(), + lastProgressAt: Date.now(), + }; + progress.set(slot, entry); + const stream = task.hls + ? new SegmentStream({ ...task, hls: task.hls }, out, engineStarted) + : null; + engine.onEvent = stream ? (event) => stream.onEvent(event) : null; + // Ask for the next task once this one is ~PREFETCH_LEAD_MS from done, so + // its sources download while this one finishes instead of after. + engine.onProgress = (frames) => { + entry.frames = frames; + entry.lastProgressAt = Date.now(); + const elapsed = performance.now() - engineStarted; + const remainingMs = + ((entry.total - frames) * elapsed) / Math.max(1, frames); + if (frames > 0 && remainingMs <= PREFETCH_LEAD_MS) nearEnd(); + }; + const request = engine.request<{ + width: number; + height: number; + sizes: number[]; + keyframes: number[]; + extradata: string; + bytes: number; + timings: Record; + }>("video", { + project: cache.root, + fps: task.fps, + resolution: task.resolution, + bpp: task.bpp, + frames: task.frames, + threads: task.threads, + out, + }); + const threadsPromise: Promise> = + process.env.RF_PROFILE_THREADS === "1" + ? sampleThreads(engine.pid, request) + : Promise.resolve({}); + const result = await request.finally(() => { + engine.onProgress = null; + engine.onEvent = null; + progress.delete(slot); + }); + nearEnd(); + const segmentsDone = stream?.finish(result.sizes, result.extradata); + segmentsDone?.catch(() => {}); + const engineMs = performance.now() - engineStarted; + const threads = await threadsPromise; + const waitStarted = performance.now(); + const audio = await audioPromise; + const audioWaitMs = performance.now() - waitStarted; + const uploadStarted = performance.now(); + const plan = layout(task, out, result.sizes, audio); + const [{ parts, stash }, segments] = await Promise.all([ + uploadLayout(task, plan.segments, plan.bytes), + segmentsDone ?? Promise.resolve(0), + ]); + const uploadMs = performance.now() - uploadStarted; + const timings: TaskTimings = { + queuedMs, + fetch: fetchStats, + engine: { + ...result.timings, + ...Object.fromEntries( + Object.entries(threads).map(([name, value]) => [ + `thread:${name}`, + value, + ]), + ), + ...(stream + ? { + segments, + firstSegmentMs: Math.round(stream.firstUploadedMs ?? -1), + } + : {}), + }, + engineMs, + audioWaitMs, + uploadMs, + totalMs: performance.now() - started, + cpuSeconds: (await processCpuSeconds(engine.pid)) - cpuBefore, + }; + return { + taskId: task.taskId, + worker: WORKER_ID, + sizes: result.sizes, + keyframes: result.keyframes, + extradata: result.extradata, + width: result.width, + height: result.height, + videoRuns: plan.videoRuns, + audioRuns: plan.audioRuns, + firstPart: task.upload.firstPart, + stash, + parts, + bytes: plan.bytes, + timings, + }; + } finally { + rmSync(out, { force: true }); + } } async function runAudio(task: AudioTask, engine: Engine, queuedMs: number) {