diff --git a/packages/plugins/live-debugger/CONTRIBUTING.md b/packages/plugins/live-debugger/CONTRIBUTING.md index ec8cdc040..db35af52d 100644 --- a/packages/plugins/live-debugger/CONTRIBUTING.md +++ b/packages/plugins/live-debugger/CONTRIBUTING.md @@ -6,6 +6,7 @@ Developer notes for the Live Debugger plugin. - [Development workflow](#development-workflow) +- [Consumer build canary](#consumer-build-canary) - [Runtime benchmark](#runtime-benchmark) - [Running it](#running-it) - [What it measures](#what-it-measures) @@ -40,6 +41,59 @@ Generated code should keep the dormant runtime path small: call `$dd_probes(func When adding or changing `liveDebugger` configuration, update [`src/types.ts`](./src/types.ts), [`src/validate.ts`](./src/validate.ts), [`src/validate.test.ts`](./src/validate.test.ts), and the consumer-facing [`README.md`](./README.md). +## Consumer build canary + +Canary targets build real consumer projects with local Datadog build plugins. +Each target defines its own build phases, package-linking strategy, output +locations, and validation checks. + +To see the currently registered targets and the available command options, run: + +```bash +yarn cli canary --help +``` + +Run every phase defined by a target with: + +```bash +yarn cli canary +``` + +Install the dependencies of both repositories first. A target may refuse to run +when either checkout already has a local package-link setup, so it cannot +overwrite an existing development environment. + +The default run compares control and Live Debugger-instrumented variants. Only +Live Debugger is toggled between a target's paired builds. To select one +target-defined phase or use a non-default checkout, run: + +```bash +yarn cli canary --phase --root /path/to/checkout +``` + +For each phase, the result reports: + +- wall-clock build duration, excluding the separate syntax-validation pass; +- total raw bytes across all emitted JavaScript files; +- total gzip bytes when each emitted JavaScript file is compressed separately; +- absolute and percentage differences between control and instrumented builds. + +Build order alternates across runs to reduce systematic warm-cache bias. A +single duration difference is still noisy and should be interpreted as part of +a trend, not as a hard regression threshold. + +The command writes a versioned JSON report to a timestamped temporary file and +prints its path. Use `--report /path/to/report.json` to choose an artifact path. +Generated assets are retained at target-defined locations for diagnosis. +Temporary package links and package export changes are reverted even when a +build fails. + +The report records each repository's HEAD commit and whether its worktree was +dirty when the run started. The canary fails when a build or validation command +exits unsuccessfully, no JavaScript is emitted, the instrumented output lacks +Live Debugger markers, or the plugin reports an instrumentation or source-map +error. + ## Runtime benchmark The opt-in browser benchmark measures the dormant runtime overhead added by Live Debugger instrumentation. It compares instrumented code against equivalent uninstrumented code, back-to-back in the same browser session, with the real Browser Debugger SDK loaded but dormant (no active probes). diff --git a/packages/tools/src/commands/canary/assertions/live-debugger.test.ts b/packages/tools/src/commands/canary/assertions/live-debugger.test.ts new file mode 100644 index 000000000..328fe3154 --- /dev/null +++ b/packages/tools/src/commands/canary/assertions/live-debugger.test.ts @@ -0,0 +1,79 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { outputFile, rm } from '@dd/core/helpers/fs'; +import os from 'os'; +import path from 'path'; + +import type { CommandResult } from '../types'; + +import { assertLiveDebuggerArtifacts, assertLiveDebuggerBuildOutput } from './live-debugger'; + +const TEST_ROOT = path.resolve(os.tmpdir(), `build-plugins-canary-live-debugger-${process.pid}`); + +const resultWithOutput = (output: string): CommandResult => { + return { + durationMs: 1, + exitCode: 0, + output, + signal: null, + }; +}; + +describe('Live Debugger canary assertions', () => { + afterAll(async () => { + await rm(TEST_ROOT); + }); + + test('should require positive instrumentation only from the instrumented build', () => { + expect(() => { + assertLiveDebuggerBuildOutput(resultWithOutput('build complete'), 'control'); + }).not.toThrow(); + expect(() => { + assertLiveDebuggerBuildOutput( + resultWithOutput('Live Debugger: 25/30 functions instrumented across 5/6 files'), + 'instrumented', + ); + }).not.toThrow(); + expect(() => { + assertLiveDebuggerBuildOutput( + resultWithOutput('Live Debugger: 0/30 functions instrumented across 0/6 files'), + 'instrumented', + ); + }).toThrow('did not report any Live Debugger instrumentation'); + expect(() => { + assertLiveDebuggerBuildOutput( + resultWithOutput('Live Debugger: 1/1 functions instrumented across 1/1 files'), + 'control', + ); + }).toThrow('Control build unexpectedly ran Live Debugger'); + }); + + test('should fail when instrumentation reports a recoverable transform error', () => { + expect(() => { + assertLiveDebuggerBuildOutput( + resultWithOutput('Instrumentation Error in /consumer/app.ts: parse failed'), + 'instrumented', + ); + }).toThrow('reported a Live Debugger error'); + }); + + test('should require an emitted Live Debugger runtime marker', async () => { + await rm(TEST_ROOT); + const plainPath = path.resolve(TEST_ROOT, 'plain.js'); + const instrumentedPath = path.resolve(TEST_ROOT, 'instrumented.js'); + await outputFile(plainPath, 'function plain() {}'); + await outputFile( + instrumentedPath, + 'globalThis.$dd_probes = globalThis.$dd_probes || (() => []);', + ); + + await expect(assertLiveDebuggerArtifacts([plainPath], 'instrumented')).rejects.toThrow( + 'does not contain the Live Debugger runtime marker', + ); + await expect( + assertLiveDebuggerArtifacts([plainPath, instrumentedPath], 'instrumented'), + ).resolves.toBeUndefined(); + }); +}); diff --git a/packages/tools/src/commands/canary/assertions/live-debugger.ts b/packages/tools/src/commands/canary/assertions/live-debugger.ts new file mode 100644 index 000000000..bf649281f --- /dev/null +++ b/packages/tools/src/commands/canary/assertions/live-debugger.ts @@ -0,0 +1,71 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { readFile } from '@dd/core/helpers/fs'; + +import { CanaryCommandError } from '../runner'; +import type { CanaryVariant, CommandResult } from '../types'; + +const ERROR_MARKERS = [ + 'Instrumentation Error in ', + 'Failed to compose source map for ', + 'Invalid configuration for datadog-live-debugger-plugin', +]; + +export const assertLiveDebuggerBuildOutput = ( + result: CommandResult, + variant: CanaryVariant, +): void => { + for (const marker of ERROR_MARKERS) { + if (result.output.includes(marker)) { + throw new CanaryCommandError( + `${variant} build reported a Live Debugger error containing "${marker}".`, + ); + } + } + + const summaryPattern = + /Live Debugger: (\d+)\/(\d+) functions instrumented across (\d+)\/(\d+) files/g; + const summaries = Array.from(result.output.matchAll(summaryPattern)); + if (variant === 'control') { + if (summaries.length > 0) { + throw new CanaryCommandError( + 'Control build unexpectedly ran Live Debugger instrumentation.', + ); + } + return; + } + + const hasInstrumentation = summaries.some((summary) => { + const instrumentedFunctions = Number(summary[1] ?? 0); + const totalFunctions = Number(summary[2] ?? 0); + const transformedFiles = Number(summary[3] ?? 0); + return instrumentedFunctions > 0 && totalFunctions > 0 && transformedFiles > 0; + }); + if (!hasInstrumentation) { + throw new CanaryCommandError( + 'Instrumented build did not report any Live Debugger instrumentation.', + ); + } +}; + +export const assertLiveDebuggerArtifacts = async ( + filePaths: string[], + variant: CanaryVariant, +): Promise => { + if (variant === 'control') { + return; + } + + for (const filePath of filePaths) { + const content = await readFile(filePath); + if (content.includes('$dd_probes')) { + return; + } + } + + throw new CanaryCommandError( + 'Instrumented build output does not contain the Live Debugger runtime marker.', + ); +}; diff --git a/packages/tools/src/commands/canary/index.ts b/packages/tools/src/commands/canary/index.ts new file mode 100644 index 000000000..ede7dd4b6 --- /dev/null +++ b/packages/tools/src/commands/canary/index.ts @@ -0,0 +1,154 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { formatDuration as formatMilliseconds } from '@dd/core/helpers/strings'; +import { ROOT } from '@dd/tools/constants'; +import { Command, Option, UsageError } from 'clipanion'; +import os from 'os'; +import path from 'path'; + +import { runCanary, StreamingCommandRunner } from './runner'; +import { getCanaryTarget, getCanaryTargetNames } from './targets'; +import type { CanaryReport, InterruptSignal, MetricComparison } from './types'; + +const SUPPORTED_TARGETS = getCanaryTargetNames().join(', '); + +const getRunId = (): string => { + return process.env.CI_PIPELINE_ID ?? process.env.GITHUB_RUN_ID ?? new Date().toISOString(); +}; + +const getDefaultReportPath = (): string => { + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + return path.resolve(os.tmpdir(), `build-plugins-canary-${timestamp}.json`); +}; + +const BYTE_UNITS = ['B', 'kB', 'MB', 'GB', 'TB'] as const; +const BYTES_PER_UNIT = 1_000; + +const formatDuration = (durationMs: number): string => { + const roundedDurationMs = Math.round(durationMs / 1_000) * 1_000; + return roundedDurationMs === 0 ? '0s' : formatMilliseconds(roundedDurationMs); +}; + +export const formatBytes = (bytes: number): string => { + let value = bytes; + let unitIndex = 0; + + while (Math.abs(value) >= BYTES_PER_UNIT && unitIndex < BYTE_UNITS.length - 1) { + value /= BYTES_PER_UNIT; + unitIndex++; + } + + const maximumFractionDigits = unitIndex === 0 ? 0 : 2; + const formatted = value.toLocaleString('en-US', { maximumFractionDigits }); + return `${formatted} ${BYTE_UNITS[unitIndex]}`; +}; + +export const formatDelta = ( + comparison: MetricComparison, + formatter: (value: number) => string, +): string => { + const sign = comparison.delta > 0 ? '+' : comparison.delta < 0 ? '-' : ''; + const absoluteDelta = Math.abs(comparison.delta); + const percentage = + comparison.deltaPercent === null + ? 'n/a' + : `${sign}${Math.abs(comparison.deltaPercent).toFixed(2)}%`; + return `${formatter(comparison.control)} -> ${formatter( + comparison.instrumented, + )} (${sign}${formatter(absoluteDelta)}, ${percentage})`; +}; + +const printReport = (report: CanaryReport): void => { + console.log('\n[Canary] Live Debugger comparison'); + for (const phase of report.phases) { + console.log(`\n${phase.id} (${phase.buildTool})`); + console.log(` build time: ${formatDelta(phase.comparison.durationMs, formatDuration)}`); + console.log(` raw JS: ${formatDelta(phase.comparison.rawBytes, formatBytes)}`); + console.log(` gzip JS: ${formatDelta(phase.comparison.gzipBytes, formatBytes)}`); + } + console.log(`\n[Canary] JSON report: ${report.reportPath}`); +}; + +class Canary extends Command { + static paths = [['canary']]; + + static usage = Command.Usage({ + category: 'Verification', + description: 'Run an external-project canary with control and Live Debugger builds.', + details: `Registered targets: ${SUPPORTED_TARGETS}.`, + examples: [ + ['Run every phase for a target', '$0 canary '], + ['Run one target-defined phase', '$0 canary --phase '], + ], + }); + + targetName = Option.String(); + + root = Option.String('--root', { + description: 'Path to the target repository checkout.', + }); + + phase = Option.String('--phase', 'all', { + description: 'Target phase to run, or "all".', + }); + + report = Option.String('--report', { + description: 'Path for the versioned JSON result.', + }); + + async execute(): Promise { + const target = getCanaryTarget(this.targetName); + if (!target) { + const targetNames = getCanaryTargetNames().join(', '); + throw new UsageError( + `Unknown canary target "${this.targetName}". Available targets: ${targetNames}.`, + ); + } + + const root = path.resolve(this.root ?? target.getDefaultRoot()); + const reportPath = this.report ? path.resolve(this.report) : getDefaultReportPath(); + const runId = getRunId(); + const commandRunner = new StreamingCommandRunner(); + let interruption: InterruptSignal | undefined; + const handleSignal = (signal: InterruptSignal): void => { + interruption = signal; + commandRunner.cancel(signal); + }; + const handleInterrupt = (): void => handleSignal('SIGINT'); + const handleTermination = (): void => handleSignal('SIGTERM'); + process.once('SIGINT', handleInterrupt); + process.once('SIGTERM', handleTermination); + + try { + const report = await runCanary({ + buildPluginsRoot: ROOT, + phaseSelection: this.phase, + reportPath, + root, + runCommand: commandRunner.run, + runId, + target, + }); + printReport(report); + return 0; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + this.context.stderr.write(`\n[Canary] ${message}\n`); + this.context.stderr.write(`[Canary] JSON report: ${reportPath}\n`); + if (interruption === 'SIGINT') { + return 130; + } + if (interruption === 'SIGTERM') { + return 143; + } + return 1; + } finally { + process.removeListener('SIGINT', handleInterrupt); + process.removeListener('SIGTERM', handleTermination); + } + } +} + +export default [Canary]; diff --git a/packages/tools/src/commands/canary/runner.test.ts b/packages/tools/src/commands/canary/runner.test.ts new file mode 100644 index 000000000..f5e37252f --- /dev/null +++ b/packages/tools/src/commands/canary/runner.test.ts @@ -0,0 +1,207 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { outputFile, readFile, rm } from '@dd/core/helpers/fs'; +import os from 'os'; +import path from 'path'; +import { gzipSync } from 'zlib'; + +import { createMetricComparison, getVariantOrder, measureArtifacts, runCanary } from './runner'; +import type { CanaryPhase, CanaryTarget, CommandResult, CommandSpec, RunCommand } from './types'; + +const TEST_ROOT = path.resolve(os.tmpdir(), `build-plugins-canary-runner-${process.pid}`); + +const successfulResult = (durationMs: number, output = ''): CommandResult => { + return { + durationMs, + exitCode: 0, + output, + signal: null, + }; +}; + +const createCommand = (root: string, label: string): CommandSpec => { + return { + command: 'synthetic-build', + args: [], + cwd: root, + label, + }; +}; + +const createSyntheticPhase = (root: string): CanaryPhase => { + return { + id: 'webpack-production', + buildTool: 'webpack', + localPackages: ['@datadog/webpack-plugin'], + getArtifactSpec: (_targetRoot, variant) => ({ + roots: [path.resolve(root, variant)], + patterns: ['**/*.js'], + }), + getBuildCommand: (_targetRoot, variant) => createCommand(root, `build:${variant}`), + getValidationCommand: (_targetRoot, variant) => createCommand(root, `validate:${variant}`), + assertBuildOutput: () => undefined, + }; +}; + +const createSyntheticTarget = (root: string, cleanupOrder: string[]): CanaryTarget => { + return { + id: 'synthetic', + getDefaultRoot: () => root, + getPhases: () => [createSyntheticPhase(root)], + preflight: async () => undefined, + setup: async ({ registerCleanup }) => { + registerCleanup({ + name: 'first', + run: async () => { + cleanupOrder.push('first'); + }, + }); + registerCleanup({ + name: 'second', + run: async () => { + cleanupOrder.push('second'); + }, + }); + }, + }; +}; + +describe('canary runner', () => { + beforeEach(async () => { + await rm(TEST_ROOT); + }); + + afterAll(async () => { + await rm(TEST_ROOT); + }); + + test('should calculate absolute and percentage deltas', () => { + expect(createMetricComparison(100, 125)).toEqual({ + control: 100, + instrumented: 125, + delta: 25, + deltaPercent: 25, + }); + expect(createMetricComparison(0, 10)).toEqual({ + control: 0, + instrumented: 10, + delta: 10, + deltaPercent: null, + }); + }); + + test('should choose a stable variant order from the run and phase IDs', () => { + const first = getVariantOrder('run-1', 'main'); + const second = getVariantOrder('run-1', 'main'); + const sorted = [...first].sort(); + + expect(second).toEqual(first); + expect(sorted).toEqual(['control', 'instrumented']); + }); + + test('should measure raw and gzip bytes without double-counting overlapping roots', async () => { + const nestedRoot = path.resolve(TEST_ROOT, 'output/nested'); + const firstPath = path.resolve(TEST_ROOT, 'output/first.js'); + const secondPath = path.resolve(nestedRoot, 'second.js'); + const firstContent = 'console.log("first");'; + const secondContent = 'console.log("second");'; + await outputFile(firstPath, firstContent); + await outputFile(secondPath, secondContent); + + const measurement = await measureArtifacts({ + roots: [path.resolve(TEST_ROOT, 'output'), nestedRoot], + patterns: ['**/*.js'], + }); + const rawBytes = Buffer.byteLength(firstContent) + Buffer.byteLength(secondContent); + const firstGzip = gzipSync(firstContent); + const secondGzip = gzipSync(secondContent); + const gzipBytes = firstGzip.length + secondGzip.length; + + expect(measurement).toEqual({ + fileCount: 2, + filePaths: [firstPath, secondPath], + gzipBytes, + rawBytes, + }); + }); + + test('should run a non-Rspack phase, compare variants, write a report, and clean up', async () => { + const controlPath = path.resolve(TEST_ROOT, 'control/app.js'); + const instrumentedPath = path.resolve(TEST_ROOT, 'instrumented/app.js'); + await outputFile(controlPath, 'function app() { return 1; }'); + await outputFile(instrumentedPath, 'function app() { $dd_probes("app"); return 1; }'); + + const commands: string[] = []; + const runCommand: RunCommand = async (spec) => { + commands.push(spec.label); + if (spec.label === 'build:control') { + return successfulResult(100); + } + if (spec.label === 'build:instrumented') { + return successfulResult(125); + } + return successfulResult(1); + }; + const cleanupOrder: string[] = []; + const target = createSyntheticTarget(TEST_ROOT, cleanupOrder); + const reportPath = path.resolve(TEST_ROOT, 'report.json'); + const report = await runCanary({ + buildPluginsRoot: process.cwd(), + phaseSelection: 'all', + reportPath, + root: TEST_ROOT, + runCommand, + runId: 'synthetic-run', + target, + }); + + expect(report.status).toBe('passed'); + expect(report.phases[0]?.buildTool).toBe('webpack'); + expect(report.phases[0]?.comparison.durationMs).toEqual({ + control: 100, + instrumented: 125, + delta: 25, + deltaPercent: 25, + }); + expect(commands).toHaveLength(4); + expect(cleanupOrder).toEqual(['second', 'first']); + const serialized = await readFile(reportPath); + expect(serialized).toContain('"schemaVersion": 1'); + expect(serialized).toContain('"buildTool": "webpack"'); + expect(serialized).toContain('"dirty": true'); + }); + + test('should write failure details and run cleanup in reverse order', async () => { + const controlPath = path.resolve(TEST_ROOT, 'control/app.js'); + const instrumentedPath = path.resolve(TEST_ROOT, 'instrumented/app.js'); + await outputFile(controlPath, 'control'); + await outputFile(instrumentedPath, 'instrumented'); + + const runCommand: RunCommand = async () => ({ + durationMs: 10, + exitCode: 1, + output: 'build failed', + signal: null, + }); + const cleanupOrder: string[] = []; + const target = createSyntheticTarget(TEST_ROOT, cleanupOrder); + const reportPath = path.resolve(TEST_ROOT, 'failed-report.json'); + const promise = runCanary({ + buildPluginsRoot: process.cwd(), + phaseSelection: 'all', + reportPath, + root: TEST_ROOT, + runCommand, + runId: 'failed-run', + target, + }); + + await expect(promise).rejects.toThrow('failed with exit code 1'); + expect(cleanupOrder).toEqual(['second', 'first']); + const serialized = await readFile(reportPath); + expect(serialized).toContain('"status": "failed"'); + expect(serialized).toContain('"stage": "phase:webpack-production"'); + }); +}); diff --git a/packages/tools/src/commands/canary/runner.ts b/packages/tools/src/commands/canary/runner.ts new file mode 100644 index 000000000..fc32d796c --- /dev/null +++ b/packages/tools/src/commands/canary/runner.ts @@ -0,0 +1,436 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { outputFile, readFile } from '@dd/core/helpers/fs'; +import { execute } from '@dd/tools/helpers'; +import type { ChildProcess } from 'child_process'; +import { spawn } from 'child_process'; +import { glob } from 'glob'; +import { performance } from 'perf_hooks'; +import type { Readable, Writable } from 'stream'; +import { gzipSync } from 'zlib'; + +import type { + ArtifactMeasurement, + ArtifactSpec, + CanaryFailure, + CanaryPhase, + CanaryReport, + CanaryTarget, + CanaryVariant, + Cleanup, + CommandResult, + CommandSpec, + InterruptSignal, + MetricComparison, + PhaseComparison, + PhaseReport, + RunCommand, + VariantMeasurement, + VariantReport, +} from './types'; + +const normalizeError = (error: unknown): Error => { + return error instanceof Error ? error : new Error(String(error)); +}; + +export class CanaryCommandError extends Error {} + +export class StreamingCommandRunner { + private activeChild: ChildProcess | undefined; + + cancel(signal: InterruptSignal): void { + this.activeChild?.kill(signal); + } + + run: RunCommand = async (spec) => { + const startedAt = performance.now(); + const env = { + ...process.env, + ...spec.env, + }; + + return new Promise((resolve, reject) => { + const child = spawn(spec.command, spec.args, { + cwd: spec.cwd, + env, + stdio: ['ignore', 'pipe', 'pipe'], + }); + this.activeChild = child; + + let output = ''; + let settled = false; + + const capture = (stream: Readable, destination: Writable): void => { + stream.setEncoding('utf8'); + stream.on('data', (chunk: string) => { + output += chunk; + destination.write(chunk); + }); + }; + + capture(child.stdout, process.stdout); + capture(child.stderr, process.stderr); + + child.once('error', (error) => { + if (settled) { + return; + } + settled = true; + this.activeChild = undefined; + reject(error); + }); + + child.once('close', (exitCode, signal) => { + if (settled) { + return; + } + settled = true; + this.activeChild = undefined; + resolve({ + durationMs: performance.now() - startedAt, + exitCode, + output, + signal, + }); + }); + }); + }; +} + +export const runCheckedCommand = async ( + runCommand: RunCommand, + spec: CommandSpec, +): Promise => { + const result = await runCommand(spec); + if (result.exitCode === 0) { + return result; + } + + const exitDescription = result.signal + ? `signal ${result.signal}` + : `exit code ${result.exitCode ?? 'unknown'}`; + throw new CanaryCommandError(`${spec.label} failed with ${exitDescription}.`); +}; + +const getArtifactFiles = async (spec: ArtifactSpec): Promise => { + const filePaths = new Set(); + + for (const root of spec.roots) { + for (const pattern of spec.patterns) { + const matches = await glob(pattern, { + absolute: true, + cwd: root, + nodir: true, + }); + for (const match of matches) { + filePaths.add(match); + } + } + } + + return Array.from(filePaths).sort(); +}; + +export const measureArtifacts = async ( + spec: ArtifactSpec, +): Promise => { + const filePaths = await getArtifactFiles(spec); + let gzipBytes = 0; + let rawBytes = 0; + + for (const filePath of filePaths) { + const content = await readFile(filePath); + rawBytes += Buffer.byteLength(content); + const compressed = gzipSync(content); + gzipBytes += compressed.length; + } + + return { + fileCount: filePaths.length, + filePaths, + gzipBytes, + rawBytes, + }; +}; + +export const createMetricComparison = (control: number, instrumented: number): MetricComparison => { + const delta = instrumented - control; + const deltaPercent = control === 0 ? null : (delta / control) * 100; + + return { + control, + instrumented, + delta, + deltaPercent, + }; +}; + +const createPhaseComparison = ( + control: VariantMeasurement, + instrumented: VariantMeasurement, +): PhaseComparison => { + return { + durationMs: createMetricComparison(control.durationMs, instrumented.durationMs), + gzipBytes: createMetricComparison(control.gzipBytes, instrumented.gzipBytes), + rawBytes: createMetricComparison(control.rawBytes, instrumented.rawBytes), + }; +}; + +const hash = (value: string): number => { + let result = 0; + for (const character of value) { + result = (result * 31 + character.charCodeAt(0)) % Number.MAX_SAFE_INTEGER; + } + return result; +}; + +export const getVariantOrder = (runId: string, phaseId: string): CanaryVariant[] => { + const key = `${runId}:${phaseId}`; + return hash(key) % 2 === 0 ? ['control', 'instrumented'] : ['instrumented', 'control']; +}; + +const measureVariant = async ({ + phase, + root, + runCommand, + variant, +}: { + phase: CanaryPhase; + root: string; + runCommand: RunCommand; + variant: CanaryVariant; +}): Promise => { + const buildCommand = phase.getBuildCommand(root, variant); + const buildResult = await runCheckedCommand(runCommand, buildCommand); + phase.assertBuildOutput(buildResult, variant); + + const artifactSpec = phase.getArtifactSpec(root, variant); + const artifacts = await measureArtifacts(artifactSpec); + if (artifacts.fileCount === 0) { + throw new CanaryCommandError( + `${phase.id} ${variant} build emitted no matching JavaScript files.`, + ); + } + if (phase.assertArtifacts) { + await phase.assertArtifacts(artifacts.filePaths, variant); + } + + const validationCommand = phase.getValidationCommand(root, variant); + await runCheckedCommand(runCommand, validationCommand); + + return { + durationMs: buildResult.durationMs, + fileCount: artifacts.fileCount, + gzipBytes: artifacts.gzipBytes, + rawBytes: artifacts.rawBytes, + outputRoots: artifactSpec.roots, + }; +}; + +const runPhase = async ({ + phase, + root, + runCommand, + runId, +}: { + phase: CanaryPhase; + root: string; + runCommand: RunCommand; + runId: string; +}): Promise => { + const variantOrder = getVariantOrder(runId, phase.id); + let control: VariantReport | undefined; + let instrumented: VariantReport | undefined; + + for (const variant of variantOrder) { + console.log(`\n[Canary] Running ${phase.id} ${variant} build.`); + const result = await measureVariant({ + phase, + root, + runCommand, + variant, + }); + if (variant === 'control') { + control = result; + } else { + instrumented = result; + } + } + + if (!control || !instrumented) { + throw new CanaryCommandError(`${phase.id} did not produce both canary variants.`); + } + + return { + id: phase.id, + buildTool: phase.buildTool, + localPackages: phase.localPackages, + variantOrder, + variants: { + control, + instrumented, + }, + comparison: createPhaseComparison(control, instrumented), + }; +}; + +const getGitState = async (root: string): Promise<{ commit: string; dirty: boolean }> => { + try { + const commitResult = await execute('git', ['rev-parse', 'HEAD'], root); + const statusResult = await execute( + 'git', + ['status', '--porcelain', '--untracked-files=normal'], + root, + ); + return { + commit: commitResult.stdout.trim(), + dirty: statusResult.stdout.trim().length > 0, + }; + } catch { + return { + commit: 'unknown', + dirty: true, + }; + } +}; + +const runCleanups = async ( + cleanups: Cleanup[], + failures: CanaryFailure[], +): Promise => { + let firstError: Error | undefined; + + for (const cleanup of [...cleanups].reverse()) { + try { + await cleanup.run(); + } catch (error) { + const normalized = normalizeError(error); + failures.push({ + message: normalized.message, + stage: `cleanup:${cleanup.name}`, + }); + firstError ??= normalized; + } + } + + return firstError; +}; + +const writeReport = async (report: CanaryReport): Promise => { + const serialized = JSON.stringify(report, null, 2); + await outputFile(report.reportPath, `${serialized}\n`); +}; + +export const runCanary = async ({ + buildPluginsRoot, + phaseSelection, + reportPath, + root, + runCommand, + runId, + target, +}: { + buildPluginsRoot: string; + phaseSelection: string; + reportPath: string; + root: string; + runCommand: RunCommand; + runId: string; + target: CanaryTarget; +}): Promise => { + const startedAt = new Date().toISOString(); + const buildPluginsGit = await getGitState(buildPluginsRoot); + const targetGit = await getGitState(root); + const report: CanaryReport = { + schemaVersion: 1, + status: 'passed', + target: target.id, + phaseSelection, + runId, + startedAt, + generatedAt: startedAt, + reportPath, + environment: { + node: process.version, + platform: process.platform, + arch: process.arch, + }, + repositories: { + buildPlugins: { + root: buildPluginsRoot, + ...buildPluginsGit, + }, + target: { + root, + ...targetGit, + }, + }, + phases: [], + failures: [], + }; + const cleanups: Cleanup[] = []; + const registerCleanup = (cleanup: Cleanup): void => { + cleanups.push(cleanup); + }; + const setupContext = { + buildPluginsRoot, + registerCleanup, + root, + runCommand, + }; + let primaryError: Error | undefined; + let stage = 'preflight'; + + try { + await target.preflight(setupContext); + stage = 'setup'; + await target.setup(setupContext); + + const allPhases = target.getPhases(root); + const selectedPhases = + phaseSelection === 'all' + ? allPhases + : allPhases.filter((phase) => phase.id === phaseSelection); + if (selectedPhases.length === 0) { + throw new CanaryCommandError( + `Target ${target.id} does not define phase "${phaseSelection}".`, + ); + } + + for (const phase of selectedPhases) { + stage = `phase:${phase.id}`; + const phaseReport = await runPhase({ + phase, + root, + runCommand, + runId, + }); + report.phases.push(phaseReport); + } + } catch (error) { + primaryError = normalizeError(error); + report.failures.push({ + message: primaryError.message, + stage, + }); + } finally { + const cleanupError = await runCleanups(cleanups, report.failures); + primaryError ??= cleanupError; + report.status = report.failures.length === 0 ? 'passed' : 'failed'; + report.generatedAt = new Date().toISOString(); + + try { + await writeReport(report); + } catch (error) { + primaryError ??= normalizeError(error); + } + } + + if (primaryError) { + throw primaryError; + } + + return report; +}; diff --git a/packages/tools/src/commands/canary/targets/index.ts b/packages/tools/src/commands/canary/targets/index.ts new file mode 100644 index 000000000..ad782da44 --- /dev/null +++ b/packages/tools/src/commands/canary/targets/index.ts @@ -0,0 +1,17 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import type { CanaryTarget } from '../types'; + +import { webUiTarget } from './web-ui'; + +const targets = new Map([[webUiTarget.id, webUiTarget]]); + +export const getCanaryTarget = (name: string): CanaryTarget | undefined => { + return targets.get(name); +}; + +export const getCanaryTargetNames = (): string[] => { + return Array.from(targets.keys()).sort(); +}; diff --git a/packages/tools/src/commands/canary/targets/web-ui.test.ts b/packages/tools/src/commands/canary/targets/web-ui.test.ts new file mode 100644 index 000000000..35bd4d36f --- /dev/null +++ b/packages/tools/src/commands/canary/targets/web-ui.test.ts @@ -0,0 +1,61 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { containsLinkReference, createWebUiPhase } from './web-ui'; + +describe('web-ui canary target', () => { + test('should detect nested Yarn link and portal references', () => { + expect( + containsLinkReference({ + resolutions: { + package: 'link:/tmp/package', + }, + }), + ).toBe(true); + expect( + containsLinkReference({ + resolutions: ['portal:/tmp/package'], + }), + ).toBe(true); + expect( + containsLinkReference({ + resolutions: { + package: 'npm:1.0.0', + }, + }), + ).toBe(false); + }); + + test('should vary only Live Debugger enablement between paired builds', () => { + const phase = createWebUiPhase('main'); + const control = phase.getBuildCommand('/web-ui', 'control'); + const instrumented = phase.getBuildCommand('/web-ui', 'instrumented'); + + expect(control.env).toEqual({ + ...instrumented.env, + BUILD_PLUGIN_LIVE_DEBUGGER: 'false', + }); + expect(instrumented.env?.BUILD_PLUGIN_LIVE_DEBUGGER).toBe('true'); + expect(instrumented.env).not.toHaveProperty('BUILD_PLUGIN_LIVE_DEBUGGER_INCLUDE'); + expect(instrumented.env).not.toHaveProperty('BUILD_PLUGIN_LIVE_DEBUGGER_EXCLUDE'); + }); + + test('should keep validation outside the timed build', () => { + const phase = createWebUiPhase('main'); + const build = phase.getBuildCommand('/web-ui', 'control'); + const validation = phase.getValidationCommand('/web-ui', 'control'); + + expect(build.args).toEqual(expect.arrayContaining(['--clean', '--no-validate'])); + expect(validation.args).toEqual(expect.arrayContaining(['--no-build', '--validate'])); + }); + + test('should use the dynamic split-deploys preset for the federated phase', () => { + const phase = createWebUiPhase('federated'); + const build = phase.getBuildCommand('/web-ui', 'instrumented'); + + expect(build.args).toEqual( + expect.arrayContaining(['--split-deploys', '--entry-preset=split-deploys']), + ); + }); +}); diff --git a/packages/tools/src/commands/canary/targets/web-ui.ts b/packages/tools/src/commands/canary/targets/web-ui.ts new file mode 100644 index 000000000..d258c7c8c --- /dev/null +++ b/packages/tools/src/commands/canary/targets/web-ui.ts @@ -0,0 +1,324 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { existsSync, outputFile, readFile } from '@dd/core/helpers/fs'; +import { ROOT } from '@dd/tools/constants'; +import { glob } from 'glob'; +import path from 'path'; + +import { + assertLiveDebuggerArtifacts, + assertLiveDebuggerBuildOutput, +} from '../assertions/live-debugger'; +import { CanaryCommandError, runCheckedCommand } from '../runner'; +import type { + ArtifactSpec, + CanaryPhase, + CanaryTarget, + CanaryVariant, + CommandSpec, + TargetSetupContext, +} from '../types'; + +const RSPACK_PACKAGE = '@datadog/rspack-plugin'; +const RSPACK_PACKAGE_PATH = path.resolve(ROOT, 'packages/published/rspack-plugin'); +const WEB_UI_PACKAGE_JSON = 'package.json'; +const WEB_UI_LOCKFILE = 'yarn.lock'; +const WEB_UI_BUILD_COMMAND = 'packages/apps/devx/commands/build-spa/build-spa.ts'; +const CANARY_OUTPUT_PREFIX = 'live-debugger-canary'; +const JAVASCRIPT_PATTERNS = ['**/*.js']; +const DEFAULT_MAX_OLD_SPACE_SIZE_MB = 16_384; + +const MAIN_ENTRIES = [ + 'spa-rspack', + 'react-core-rspack', + 'dd-login-rspack', + 'embed-rspack', + 'polyfills-rspack', + 'spa-internal-rspack', + 'snapshot-print-rspack', +]; + +type JsonObject = Record; + +const isJsonObject = (value: unknown): value is JsonObject => { + return typeof value === 'object' && value !== null && !Array.isArray(value); +}; + +const parseJsonObject = (content: string, filePath: string): JsonObject => { + const value: unknown = JSON.parse(content); + if (!isJsonObject(value)) { + throw new CanaryCommandError(`${filePath} does not contain a JSON object.`); + } + return value; +}; + +export const containsLinkReference = (value: unknown): boolean => { + if (typeof value === 'string') { + return /^(?:link|portal):/.test(value); + } + if (Array.isArray(value)) { + return value.some((entry) => containsLinkReference(entry)); + } + if (!isJsonObject(value)) { + return false; + } + return Object.values(value).some((entry) => containsLinkReference(entry)); +}; + +const getPublishedPackageJsonFiles = async (buildPluginsRoot: string): Promise => { + return glob('packages/published/*-plugin/package.json', { + absolute: true, + cwd: buildPluginsRoot, + nodir: true, + }); +}; + +const assertPublishedPackagesAreNotPrepared = async (buildPluginsRoot: string): Promise => { + const packageFiles = await getPublishedPackageJsonFiles(buildPluginsRoot); + for (const packageFile of packageFiles) { + const content = await readFile(packageFile); + const packageJson = parseJsonObject(content, packageFile); + if ('previousExports' in packageJson) { + throw new CanaryCommandError( + `Published packages are already prepared for linking. Revert them before running the canary.`, + ); + } + } +}; + +const assertPublishedPackagesArePrepared = async (buildPluginsRoot: string): Promise => { + const packageFiles = await getPublishedPackageJsonFiles(buildPluginsRoot); + for (const packageFile of packageFiles) { + const content = await readFile(packageFile); + const packageJson = parseJsonObject(content, packageFile); + if (!('previousExports' in packageJson)) { + throw new CanaryCommandError( + `The prepare-link command did not prepare ${packageFile}.`, + ); + } + } +}; + +const readFiles = async (filePaths: string[]): Promise> => { + const files = new Map(); + for (const filePath of filePaths) { + const content = await readFile(filePath); + files.set(filePath, content); + } + return files; +}; + +const restoreSnapshots = async (snapshots: Map): Promise => { + for (const [filePath, expected] of snapshots) { + const actual = await readFile(filePath); + if (actual !== expected) { + await outputFile(filePath, expected); + } + } +}; + +const getOutputSubdirectories = ( + phaseId: string, + variant: CanaryVariant, +): { chunk: string; entry: string } => { + return { + entry: `v/${CANARY_OUTPUT_PREFIX}/${phaseId}/${variant}/js`, + chunk: `${CANARY_OUTPUT_PREFIX}/${phaseId}/${variant}/chunks`, + }; +}; + +const getArtifactSpec = (root: string, phaseId: string, variant: CanaryVariant): ArtifactSpec => { + const subdirectories = getOutputSubdirectories(phaseId, variant); + const staticRoot = path.resolve(root, 'public/static'); + + return { + roots: [ + path.resolve(staticRoot, subdirectories.entry), + path.resolve(staticRoot, subdirectories.chunk), + ], + patterns: JAVASCRIPT_PATTERNS, + }; +}; + +const getBuildEnvironment = (variant: CanaryVariant): Record => { + const existingNodeOptions = process.env.NODE_OPTIONS?.trim() ?? ''; + const hasHeapLimit = /--max[-_]old[-_]space[-_]size(?:=|\s)/.test(existingNodeOptions); + const heapOption = `--max-old-space-size=${DEFAULT_MAX_OLD_SPACE_SIZE_MB}`; + const nodeOptions = hasHeapLimit + ? existingNodeOptions + : `${existingNodeOptions} ${heapOption}`.trim(); + + return { + BUILD_PLUGIN_DISABLE_METRICS: 'true', + BUILD_PLUGIN_DISABLE_SOURCEMAPS: 'true', + BUILD_PLUGIN_LIVE_DEBUGGER: variant === 'instrumented' ? 'true' : 'false', + BUILD_PLUGIN_RUM_PRIVACY: 'true', + BUILD_PLUGIN_UPLOAD_SOURCEMAPS: 'false', + NODE_OPTIONS: nodeOptions, + }; +}; + +const getOutputArguments = (phaseId: string, variant: CanaryVariant): string[] => { + const subdirectories = getOutputSubdirectories(phaseId, variant); + return [`--entry-subdir=${subdirectories.entry}`, `--chunk-subdir=${subdirectories.chunk}`]; +}; + +const getPhaseArguments = (phaseId: string): string[] => { + if (phaseId === 'main') { + return ['--bundler=rspack', `--entries=${MAIN_ENTRIES.join(',')}`]; + } + return ['--bundler=rspack', '--split-deploys', '--entry-preset=split-deploys']; +}; + +const getBuildCommand = (root: string, phaseId: string, variant: CanaryVariant): CommandSpec => { + const phaseArguments = getPhaseArguments(phaseId); + const outputArguments = getOutputArguments(phaseId, variant); + return { + command: 'yarn', + args: [ + 'cli', + 'build-spa', + ...phaseArguments, + ...outputArguments, + '--clean', + '--no-validate', + ], + cwd: root, + env: getBuildEnvironment(variant), + label: `${phaseId} ${variant} build`, + }; +}; + +const getValidationCommand = ( + root: string, + phaseId: string, + variant: CanaryVariant, +): CommandSpec => { + const outputArguments = getOutputArguments(phaseId, variant); + return { + command: 'yarn', + args: ['cli', 'build-spa', ...outputArguments, '--no-build', '--validate'], + cwd: root, + label: `${phaseId} ${variant} JavaScript validation`, + }; +}; + +export const createWebUiPhase = (id: 'main' | 'federated'): CanaryPhase => { + return { + id, + buildTool: 'rspack', + localPackages: [RSPACK_PACKAGE], + getArtifactSpec: (root, variant) => getArtifactSpec(root, id, variant), + getBuildCommand: (root, variant) => getBuildCommand(root, id, variant), + getValidationCommand: (root, variant) => getValidationCommand(root, id, variant), + assertBuildOutput: assertLiveDebuggerBuildOutput, + assertArtifacts: assertLiveDebuggerArtifacts, + }; +}; + +const preflightWebUi = async ({ buildPluginsRoot, root }: TargetSetupContext): Promise => { + const packageJsonPath = path.resolve(root, WEB_UI_PACKAGE_JSON); + const buildCommandPath = path.resolve(root, WEB_UI_BUILD_COMMAND); + if (!existsSync(packageJsonPath) || !existsSync(buildCommandPath)) { + throw new CanaryCommandError( + `${root} does not look like a web-ui checkout. Pass its path with --root.`, + ); + } + + await assertPublishedPackagesAreNotPrepared(buildPluginsRoot); + + const packageJsonContent = await readFile(packageJsonPath); + const packageJson = parseJsonObject(packageJsonContent, packageJsonPath); + if (containsLinkReference(packageJson)) { + throw new CanaryCommandError( + 'web-ui already contains a link or portal resolution. Unlink it before running the canary.', + ); + } +}; + +const setupWebUi = async ({ + buildPluginsRoot, + registerCleanup, + root, + runCommand, +}: TargetSetupContext): Promise => { + const packageFiles = await getPublishedPackageJsonFiles(buildPluginsRoot); + const publishedSnapshots = await readFiles(packageFiles); + const webUiPackageJsonPath = path.resolve(root, WEB_UI_PACKAGE_JSON); + const webUiLockfilePath = path.resolve(root, WEB_UI_LOCKFILE); + const webUiSnapshots = await readFiles([webUiPackageJsonPath, webUiLockfilePath]); + + const buildSpec: CommandSpec = { + command: 'yarn', + args: ['workspace', RSPACK_PACKAGE, 'build'], + cwd: buildPluginsRoot, + label: `build ${RSPACK_PACKAGE}`, + }; + await runCheckedCommand(runCommand, buildSpec); + + registerCleanup({ + name: 'published-package-exports', + run: async () => { + const revertSpec: CommandSpec = { + command: 'yarn', + args: ['cli', 'prepare-link', '--revert'], + cwd: buildPluginsRoot, + label: 'revert prepared package exports', + }; + await runCheckedCommand(runCommand, revertSpec); + await restoreSnapshots(publishedSnapshots); + }, + }); + + const prepareSpec: CommandSpec = { + command: 'yarn', + args: ['cli', 'prepare-link'], + cwd: buildPluginsRoot, + label: 'prepare published package exports', + }; + await runCheckedCommand(runCommand, prepareSpec); + await assertPublishedPackagesArePrepared(buildPluginsRoot); + + registerCleanup({ + name: 'web-ui-link', + run: async () => { + const unlinkSpec: CommandSpec = { + command: 'yarn', + args: ['unlink', '--all'], + cwd: root, + label: 'unlink build plugins from web-ui', + }; + await runCheckedCommand(runCommand, unlinkSpec); + await restoreSnapshots(webUiSnapshots); + + const installSpec: CommandSpec = { + command: 'yarn', + args: ['install', '--immutable', '--mode=skip-build'], + cwd: root, + label: 'restore web-ui dependency state', + }; + await runCheckedCommand(runCommand, installSpec); + }, + }); + + const linkSpec: CommandSpec = { + command: 'yarn', + args: ['link', '-Ap', RSPACK_PACKAGE_PATH], + cwd: root, + label: 'link local Rspack plugin into web-ui', + }; + await runCheckedCommand(runCommand, linkSpec); +}; + +export const webUiTarget: CanaryTarget = { + id: 'web-ui', + getDefaultRoot: () => { + const datadogRoot = process.env.DATADOG_ROOT; + return datadogRoot ? path.resolve(datadogRoot, 'web-ui') : path.resolve(ROOT, '../web-ui'); + }, + getPhases: () => [createWebUiPhase('main'), createWebUiPhase('federated')], + preflight: preflightWebUi, + setup: setupWebUi, +}; diff --git a/packages/tools/src/commands/canary/types.ts b/packages/tools/src/commands/canary/types.ts new file mode 100644 index 000000000..839ce409c --- /dev/null +++ b/packages/tools/src/commands/canary/types.ts @@ -0,0 +1,133 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +export const CANARY_VARIANTS = ['control', 'instrumented'] as const; +export type CanaryVariant = (typeof CANARY_VARIANTS)[number]; +export type InterruptSignal = 'SIGINT' | 'SIGTERM'; + +export type CommandSpec = { + command: string; + args: string[]; + cwd: string; + env?: Record; + label: string; +}; + +export type CommandResult = { + durationMs: number; + exitCode: number | null; + output: string; + signal: string | null; +}; + +export type RunCommand = (spec: CommandSpec) => Promise; + +export type ArtifactSpec = { + roots: string[]; + patterns: string[]; +}; + +export type ArtifactMeasurement = { + fileCount: number; + gzipBytes: number; + rawBytes: number; +}; + +export type VariantMeasurement = ArtifactMeasurement & { + durationMs: number; +}; + +export type MetricComparison = { + control: number; + instrumented: number; + delta: number; + deltaPercent: number | null; +}; + +export type PhaseComparison = { + durationMs: MetricComparison; + gzipBytes: MetricComparison; + rawBytes: MetricComparison; +}; + +export type CanaryPhase = { + id: string; + buildTool: string; + localPackages: string[]; + getArtifactSpec: (root: string, variant: CanaryVariant) => ArtifactSpec; + getBuildCommand: (root: string, variant: CanaryVariant) => CommandSpec; + getValidationCommand: (root: string, variant: CanaryVariant) => CommandSpec; + assertBuildOutput: (result: CommandResult, variant: CanaryVariant) => void; + assertArtifacts?: (filePaths: string[], variant: CanaryVariant) => Promise; +}; + +export type Cleanup = { + name: string; + run: () => Promise; +}; + +export type RegisterCleanup = (cleanup: Cleanup) => void; + +export type TargetSetupContext = { + buildPluginsRoot: string; + registerCleanup: RegisterCleanup; + root: string; + runCommand: RunCommand; +}; + +export type CanaryTarget = { + id: string; + getDefaultRoot: () => string; + getPhases: (root: string) => CanaryPhase[]; + preflight: (context: TargetSetupContext) => Promise; + setup: (context: TargetSetupContext) => Promise; +}; + +export type VariantReport = VariantMeasurement & { + outputRoots: string[]; +}; + +export type PhaseReport = { + id: string; + buildTool: string; + localPackages: string[]; + variantOrder: CanaryVariant[]; + variants: Record; + comparison: PhaseComparison; +}; + +export type CanaryFailure = { + message: string; + stage: string; +}; + +export type CanaryReport = { + schemaVersion: 1; + status: 'passed' | 'failed'; + target: string; + phaseSelection: string; + runId: string; + startedAt: string; + generatedAt: string; + reportPath: string; + environment: { + node: string; + platform: string; + arch: string; + }; + repositories: { + buildPlugins: { + root: string; + commit: string; + dirty: boolean; + }; + target: { + root: string; + commit: string; + dirty: boolean; + }; + }; + phases: PhaseReport[]; + failures: CanaryFailure[]; +};