Add Live Debugger consumer build canary

packages/plugins/live-debugger/CONTRIBUTING.md

@@ -6,6 +6,7 @@ Developer notes for the Live Debugger plugin.
 
 <!-- #toc -->
 -   [Development workflow](#development-workflow)
+-   [Consumer build canary](#consumer-build-canary)
 -   [Runtime benchmark](#runtime-benchmark)
     -   [Running it](#running-it)
     -   [What it measures](#what-it-measures)
@@ -40,6 +41,59 @@ Generated code should keep the dormant runtime path small: call `$dd_probes(func
 
 When adding or changing `liveDebugger` configuration, update [`src/types.ts`](./src/types.ts), [`src/validate.ts`](./src/validate.ts), [`src/validate.test.ts`](./src/validate.test.ts), and the consumer-facing [`README.md`](./README.md).
 
+## Consumer build canary
+
+Canary targets build real consumer projects with local Datadog build plugins.
+Each target defines its own build phases, package-linking strategy, output
+locations, and validation checks.
+
+To see the currently registered targets and the available command options, run:
+
+```bash
+yarn cli canary --help
+```
+
+Run every phase defined by a target with:
+
+```bash
+yarn cli canary <target>
+```
+
+Install the dependencies of both repositories first. A target may refuse to run
+when either checkout already has a local package-link setup, so it cannot
+overwrite an existing development environment.
+
+The default run compares control and Live Debugger-instrumented variants. Only
+Live Debugger is toggled between a target's paired builds. To select one
+target-defined phase or use a non-default checkout, run:
+
+```bash
+yarn cli canary <target> --phase <phase> --root /path/to/checkout
+```
+
+For each phase, the result reports:
+
+- wall-clock build duration, excluding the separate syntax-validation pass;
+- total raw bytes across all emitted JavaScript files;
+- total gzip bytes when each emitted JavaScript file is compressed separately;
+- absolute and percentage differences between control and instrumented builds.
+
+Build order alternates across runs to reduce systematic warm-cache bias. A
+single duration difference is still noisy and should be interpreted as part of
+a trend, not as a hard regression threshold.
+
+The command writes a versioned JSON report to a timestamped temporary file and
+prints its path. Use `--report /path/to/report.json` to choose an artifact path.
+Generated assets are retained at target-defined locations for diagnosis.
+Temporary package links and package export changes are reverted even when a
+build fails.
+
+The report records each repository's HEAD commit and whether its worktree was
+dirty when the run started. The canary fails when a build or validation command
+exits unsuccessfully, no JavaScript is emitted, the instrumented output lacks
+Live Debugger markers, or the plugin reports an instrumentation or source-map
+error.
+
 ## Runtime benchmark
 
 The opt-in browser benchmark measures the dormant runtime overhead added by Live Debugger instrumentation. It compares instrumented code against equivalent uninstrumented code, back-to-back in the same browser session, with the real Browser Debugger SDK loaded but dormant (no active probes).

packages/tools/src/commands/canary/assertions/live-debugger.test.ts

@@ -0,0 +1,79 @@
+// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2019-Present Datadog, Inc.
+
+import { outputFile, rm } from '@dd/core/helpers/fs';
+import os from 'os';
+import path from 'path';
+
+import type { CommandResult } from '../types';
+
+import { assertLiveDebuggerArtifacts, assertLiveDebuggerBuildOutput } from './live-debugger';
+
+const TEST_ROOT = path.resolve(os.tmpdir(), `build-plugins-canary-live-debugger-${process.pid}`);
+
+const resultWithOutput = (output: string): CommandResult => {
+    return {
+        durationMs: 1,
+        exitCode: 0,
+        output,
+        signal: null,
+    };
+};
+
+describe('Live Debugger canary assertions', () => {
+    afterAll(async () => {
+        await rm(TEST_ROOT);
+    });
+
+    test('should require positive instrumentation only from the instrumented build', () => {
+        expect(() => {
+            assertLiveDebuggerBuildOutput(resultWithOutput('build complete'), 'control');
+        }).not.toThrow();
+        expect(() => {
+            assertLiveDebuggerBuildOutput(
+                resultWithOutput('Live Debugger: 25/30 functions instrumented across 5/6 files'),
+                'instrumented',
+            );
+        }).not.toThrow();
+        expect(() => {
+            assertLiveDebuggerBuildOutput(
+                resultWithOutput('Live Debugger: 0/30 functions instrumented across 0/6 files'),
+                'instrumented',
+            );
+        }).toThrow('did not report any Live Debugger instrumentation');
+        expect(() => {
+            assertLiveDebuggerBuildOutput(
+                resultWithOutput('Live Debugger: 1/1 functions instrumented across 1/1 files'),
+                'control',
+            );
+        }).toThrow('Control build unexpectedly ran Live Debugger');
+    });
+
+    test('should fail when instrumentation reports a recoverable transform error', () => {
+        expect(() => {
+            assertLiveDebuggerBuildOutput(
+                resultWithOutput('Instrumentation Error in /consumer/app.ts: parse failed'),
+                'instrumented',
+            );
+        }).toThrow('reported a Live Debugger error');
+    });
+
+    test('should require an emitted Live Debugger runtime marker', async () => {
+        await rm(TEST_ROOT);
+        const plainPath = path.resolve(TEST_ROOT, 'plain.js');
+        const instrumentedPath = path.resolve(TEST_ROOT, 'instrumented.js');
+        await outputFile(plainPath, 'function plain() {}');
+        await outputFile(
+            instrumentedPath,
+            'globalThis.$dd_probes = globalThis.$dd_probes || (() => []);',
+        );
+
+        await expect(assertLiveDebuggerArtifacts([plainPath], 'instrumented')).rejects.toThrow(
+            'does not contain the Live Debugger runtime marker',
+        );
+        await expect(
+            assertLiveDebuggerArtifacts([plainPath, instrumentedPath], 'instrumented'),
+        ).resolves.toBeUndefined();
+    });
+});

packages/tools/src/commands/canary/assertions/live-debugger.ts

@@ -0,0 +1,71 @@
+// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2019-Present Datadog, Inc.
+
+import { readFile } from '@dd/core/helpers/fs';
+
+import { CanaryCommandError } from '../runner';
+import type { CanaryVariant, CommandResult } from '../types';
+
+const ERROR_MARKERS = [
+    'Instrumentation Error in ',
+    'Failed to compose source map for ',
+    'Invalid configuration for datadog-live-debugger-plugin',
+];
+
+export const assertLiveDebuggerBuildOutput = (
+    result: CommandResult,
+    variant: CanaryVariant,
+): void => {
+    for (const marker of ERROR_MARKERS) {
+        if (result.output.includes(marker)) {
+            throw new CanaryCommandError(
+                `${variant} build reported a Live Debugger error containing "${marker}".`,
+            );
+        }
+    }
+
+    const summaryPattern =
+        /Live Debugger: (\d+)\/(\d+) functions instrumented across (\d+)\/(\d+) files/g;
+    const summaries = Array.from(result.output.matchAll(summaryPattern));
+    if (variant === 'control') {
+        if (summaries.length > 0) {
+            throw new CanaryCommandError(
+                'Control build unexpectedly ran Live Debugger instrumentation.',
+            );
+        }
+        return;
+    }
+
+    const hasInstrumentation = summaries.some((summary) => {
+        const instrumentedFunctions = Number(summary[1] ?? 0);
+        const totalFunctions = Number(summary[2] ?? 0);
+        const transformedFiles = Number(summary[3] ?? 0);
+        return instrumentedFunctions > 0 && totalFunctions > 0 && transformedFiles > 0;
+    });
+    if (!hasInstrumentation) {
+        throw new CanaryCommandError(
+            'Instrumented build did not report any Live Debugger instrumentation.',
+        );
+    }
+};
+
+export const assertLiveDebuggerArtifacts = async (
+    filePaths: string[],
+    variant: CanaryVariant,
+): Promise<void> => {
+    if (variant === 'control') {
+        return;
+    }
+
+    for (const filePath of filePaths) {
+        const content = await readFile(filePath);
+        if (content.includes('$dd_probes')) {
+            return;
+        }
+    }
+
+    throw new CanaryCommandError(
+        'Instrumented build output does not contain the Live Debugger runtime marker.',
+    );
+};

packages/tools/src/commands/canary/index.ts

@@ -0,0 +1,154 @@
+// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2019-Present Datadog, Inc.
+
+import { formatDuration as formatMilliseconds } from '@dd/core/helpers/strings';
+import { ROOT } from '@dd/tools/constants';
+import { Command, Option, UsageError } from 'clipanion';
+import os from 'os';
+import path from 'path';
+
+import { runCanary, StreamingCommandRunner } from './runner';
+import { getCanaryTarget, getCanaryTargetNames } from './targets';
+import type { CanaryReport, InterruptSignal, MetricComparison } from './types';
+
+const SUPPORTED_TARGETS = getCanaryTargetNames().join(', ');
+
+const getRunId = (): string => {
+    return process.env.CI_PIPELINE_ID ?? process.env.GITHUB_RUN_ID ?? new Date().toISOString();
+};
+
+const getDefaultReportPath = (): string => {
+    const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+    return path.resolve(os.tmpdir(), `build-plugins-canary-${timestamp}.json`);
+};
+
+const BYTE_UNITS = ['B', 'kB', 'MB', 'GB', 'TB'] as const;
+const BYTES_PER_UNIT = 1_000;
+
+const formatDuration = (durationMs: number): string => {
+    const roundedDurationMs = Math.round(durationMs / 1_000) * 1_000;
+    return roundedDurationMs === 0 ? '0s' : formatMilliseconds(roundedDurationMs);
+};
+
+export const formatBytes = (bytes: number): string => {
+    let value = bytes;
+    let unitIndex = 0;
+
+    while (Math.abs(value) >= BYTES_PER_UNIT && unitIndex < BYTE_UNITS.length - 1) {
+        value /= BYTES_PER_UNIT;
+        unitIndex++;
+    }
+
+    const maximumFractionDigits = unitIndex === 0 ? 0 : 2;
+    const formatted = value.toLocaleString('en-US', { maximumFractionDigits });
+    return `${formatted} ${BYTE_UNITS[unitIndex]}`;
+};
+
+export const formatDelta = (
+    comparison: MetricComparison,
+    formatter: (value: number) => string,
+): string => {
+    const sign = comparison.delta > 0 ? '+' : comparison.delta < 0 ? '-' : '';
+    const absoluteDelta = Math.abs(comparison.delta);
+    const percentage =
+        comparison.deltaPercent === null
+            ? 'n/a'
+            : `${sign}${Math.abs(comparison.deltaPercent).toFixed(2)}%`;
+    return `${formatter(comparison.control)} -> ${formatter(
+        comparison.instrumented,
+    )} (${sign}${formatter(absoluteDelta)}, ${percentage})`;
+};
+
+const printReport = (report: CanaryReport): void => {
+    console.log('\n[Canary] Live Debugger comparison');
+    for (const phase of report.phases) {
+        console.log(`\n${phase.id} (${phase.buildTool})`);
+        console.log(`  build time: ${formatDelta(phase.comparison.durationMs, formatDuration)}`);
+        console.log(`  raw JS:     ${formatDelta(phase.comparison.rawBytes, formatBytes)}`);
+        console.log(`  gzip JS:    ${formatDelta(phase.comparison.gzipBytes, formatBytes)}`);
+    }
+    console.log(`\n[Canary] JSON report: ${report.reportPath}`);
+};
+
+class Canary extends Command {
+    static paths = [['canary']];
+
+    static usage = Command.Usage({
+        category: 'Verification',
+        description: 'Run an external-project canary with control and Live Debugger builds.',
+        details: `Registered targets: ${SUPPORTED_TARGETS}.`,
+        examples: [
+            ['Run every phase for a target', '$0 canary <target>'],
+            ['Run one target-defined phase', '$0 canary <target> --phase <phase>'],
+        ],
+    });
+
+    targetName = Option.String();
+
+    root = Option.String('--root', {
+        description: 'Path to the target repository checkout.',
+    });
+
+    phase = Option.String('--phase', 'all', {
+        description: 'Target phase to run, or "all".',
+    });
+
+    report = Option.String('--report', {
+        description: 'Path for the versioned JSON result.',
+    });
+
+    async execute(): Promise<number> {
+        const target = getCanaryTarget(this.targetName);
+        if (!target) {
+            const targetNames = getCanaryTargetNames().join(', ');
+            throw new UsageError(
+                `Unknown canary target "${this.targetName}". Available targets: ${targetNames}.`,
+            );
+        }
+
+        const root = path.resolve(this.root ?? target.getDefaultRoot());
+        const reportPath = this.report ? path.resolve(this.report) : getDefaultReportPath();
+        const runId = getRunId();
+        const commandRunner = new StreamingCommandRunner();
+        let interruption: InterruptSignal | undefined;
+        const handleSignal = (signal: InterruptSignal): void => {
+            interruption = signal;
+            commandRunner.cancel(signal);
+        };
+        const handleInterrupt = (): void => handleSignal('SIGINT');
+        const handleTermination = (): void => handleSignal('SIGTERM');
+        process.once('SIGINT', handleInterrupt);
+        process.once('SIGTERM', handleTermination);
+
+        try {
+            const report = await runCanary({
+                buildPluginsRoot: ROOT,
+                phaseSelection: this.phase,
+                reportPath,
+                root,
+                runCommand: commandRunner.run,
+                runId,
+                target,
+            });
+            printReport(report);
+            return 0;
+        } catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            this.context.stderr.write(`\n[Canary] ${message}\n`);
+            this.context.stderr.write(`[Canary] JSON report: ${reportPath}\n`);
+            if (interruption === 'SIGINT') {
+                return 130;
+            }
+            if (interruption === 'SIGTERM') {
+                return 143;
+            }
+            return 1;
+        } finally {
+            process.removeListener('SIGINT', handleInterrupt);
+            process.removeListener('SIGTERM', handleTermination);
+        }
+    }
+}
+
+export default [Canary];