diff --git a/README.md b/README.md index 759dd97..ae66e14 100644 --- a/README.md +++ b/README.md @@ -110,4 +110,4 @@ bun run build ## Telemetry -Published builds may send anonymous usage telemetry. It never includes project names, file paths, or database URLs. Disable it with `DO_NOT_TRACK`, `CREATE_PRISMA_DISABLE_TELEMETRY`, or `CREATE_PRISMA_TELEMETRY_DISABLED`. +Published builds may send anonymous usage telemetry. It never includes project names, file paths, or database URLs. Failure events carry only stable identifiers such as exit codes and tool error codes, never messages or command output. Disable it with `DO_NOT_TRACK`, `CREATE_PRISMA_DISABLE_TELEMETRY`, or `CREATE_PRISMA_TELEMETRY_DISABLED`. diff --git a/package.json b/package.json index b5b7350..371b01f 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "dev": "tsdown --watch", "start": "bun run ./dist/cli.mjs", "test": "bun run test:unit && bun run test:e2e", - "test:unit": "bun test ./tests/dependencies.test.ts ./tests/deploy-with-composer.test.ts ./tests/initialize-git.test.ts ./tests/install.test.ts ./tests/json-output.test.ts ./tests/node-version.test.ts ./tests/setup-prisma.test.ts ./tests/telemetry-client.test.ts ./tests/telemetry.test.ts", + "test:unit": "bun test ./tests/dependencies.test.ts ./tests/deploy-with-composer.test.ts ./tests/initialize-git.test.ts ./tests/install.test.ts ./tests/json-output.test.ts ./tests/node-version.test.ts ./tests/package-manager-error-code.test.ts ./tests/setup-prisma.test.ts ./tests/telemetry-client.test.ts ./tests/telemetry.test.ts", "test:e2e": "bun test --timeout 180000 ./tests/e2e/create-prisma.e2e.test.ts", "check": "bun run format:check && bun run lint", "lint": "oxlint . --deny-warnings", diff --git a/src/create-outcome.ts b/src/create-outcome.ts index 8535799..998d5e4 100644 --- a/src/create-outcome.ts +++ b/src/create-outcome.ts @@ -88,12 +88,20 @@ export class CreateFailure extends Schema.TaggedError()("CreateFa errorReported: Schema.optionalKey(Schema.Boolean), }) {} +const STRUCTURED_ERROR_CODE_PATTERN = /^[A-Z][A-Z0-9_]*(\.[A-Z0-9_]+)+$/; + +export function isStructuredErrorCode(value: unknown): value is string { + return typeof value === "string" && STRUCTURED_ERROR_CODE_PATTERN.test(value); +} + export class PrismaCliCommandError extends Schema.TaggedError()( "PrismaCliCommandError", { message: Schema.String, command: Schema.optionalKey(Schema.String), code: Schema.optionalKey(Schema.String), + // The delegated tool's own failure code. + causeCode: Schema.optionalKey(Schema.String), stderr: Schema.optionalKey(Schema.String), exitCode: Schema.optionalKey(Schema.Number), childProcessFailure: Schema.optional(ChildProcessFailureSchema), @@ -101,6 +109,7 @@ export class PrismaCliCommandError extends Schema.TaggedError void; + }) { + const fs = yield* FileSystem.FileSystem; + // The report is diagnostic, so its absence never blocks the deploy. + const reportPath = yield* fs.makeTempDirectoryScoped({ prefix: "create-prisma-deploy-" }).pipe( + Effect.map((directory) => path.join(directory, "report.json")), + Effect.option, + Effect.map(Option.getOrUndefined), + ); + + return yield* runPrismaJsonCommandEffect({ + packageManager: options.packageManager, + projectDir: options.projectDir, + args: ["deploy", "module.ts", ...(reportPath ? ["--report", reportPath] : [])], + ...(options.onStderrLine ? { onStderrLine: options.onStderrLine } : {}), + }).pipe( + Effect.catchTag("PrismaCliCommandError", (error) => + Effect.gen(function* () { + const causeCode = reportPath + ? yield* readComposerDeployFailureCode(reportPath) + : undefined; + return yield* Effect.fail(causeCode ? withCauseCode(error, causeCode) : error); + }), + ), + ); + }, + Effect.scoped, +); diff --git a/src/tasks/deploy-with-composer.ts b/src/tasks/deploy-with-composer.ts index f332e80..7202286 100644 --- a/src/tasks/deploy-with-composer.ts +++ b/src/tasks/deploy-with-composer.ts @@ -21,7 +21,8 @@ import { ComposerDeployCommandResultSchema, parseComposerDeployResult, } from "./composer/deployment-result"; -import { decodePrismaCommandResult, runPrismaJsonCommandEffect } from "./prisma-cli"; +import { runComposerDeployEffect } from "./composer/deploy-report"; +import { decodePrismaCommandResult } from "./prisma-cli"; import { ensureProjectNameAvailable, getProjectDetails } from "./composer/projects"; export type ComposerDeployExecutionResult = @@ -150,10 +151,9 @@ export const deployNewProjectWithComposerEffect = Effect.fn("Deployment.deploy") } }); const rawDeployment = yield* atCreateStage( - runPrismaJsonCommandEffect({ + runComposerDeployEffect({ packageManager: options.packageManager, projectDir: options.projectDir, - args: ["deploy", "module.ts"], onStderrLine: (line) => { const redacted = redactSecrets(line); if (options.verbose) output.write(`${redacted}\n`); diff --git a/src/telemetry/create.ts b/src/telemetry/create.ts index a00c689..ae79f24 100644 --- a/src/telemetry/create.ts +++ b/src/telemetry/create.ts @@ -2,6 +2,7 @@ import { Effect } from "effect"; import type { CreatePromptContext } from "../commands/create"; import { + isStructuredErrorCode, PrismaCliCommandError, type CreateCancellationStage, type CreateFailureReason, @@ -10,6 +11,7 @@ import { import type { CreateCommandInput } from "../types"; import { applicationRuntime } from "../runtime"; import { CommandExecutionError } from "../services/command-runner"; +import { getPackageManagerErrorCode } from "../utils/package-manager-error-code"; import { TELEMETRY_TIMEOUT_MS, trackCliTelemetryEffect } from "./client"; @@ -100,9 +102,15 @@ function getChildProcessFailureProperty(error: unknown): string | null { : null; } +function getPackageManagerErrorCodeProperty(error: unknown): string | null { + return error instanceof CommandExecutionError + ? (getPackageManagerErrorCode(error.command, error) ?? null) + : null; +} + function getPrismaCliFailureProperty( error: unknown, - property: "prismaCliCommand" | "prismaCliErrorCode", + property: "prismaCliCommand" | "prismaCliErrorCode" | "prismaCliCauseCode", ): string | null { if (typeof error !== "object" || error === null) { return null; @@ -112,6 +120,11 @@ function getPrismaCliFailureProperty( return typeof value === "string" && value.length > 0 ? value : null; } +function getPrismaCliCauseCodeProperty(error: unknown): string | null { + const causeCode = getPrismaCliFailureProperty(error, "prismaCliCauseCode"); + return isStructuredErrorCode(causeCode) ? causeCode : null; +} + export const trackCreateCompletedEffect = Effect.fn("Telemetry.createCompleted")( function* (params: { input: CreateCommandInput; @@ -148,6 +161,11 @@ export const trackCreateFailedEffect = Effect.fn("Telemetry.createFailed")(funct "child-process-failure": getChildProcessFailureProperty(params.error), "prisma-cli-command": getPrismaCliFailureProperty(params.error, "prismaCliCommand"), "prisma-cli-error-code": getPrismaCliFailureProperty(params.error, "prismaCliErrorCode"), + "prisma-cli-cause-code": getPrismaCliCauseCodeProperty(params.error), + "package-manager-error-code": + params.stage === "install_dependencies" + ? getPackageManagerErrorCodeProperty(params.error) + : null, }).pipe( Effect.scoped, Effect.timeout(TELEMETRY_TIMEOUT_MS), diff --git a/src/utils/package-manager-error-code.ts b/src/utils/package-manager-error-code.ts new file mode 100644 index 0000000..95f389a --- /dev/null +++ b/src/utils/package-manager-error-code.ts @@ -0,0 +1,46 @@ +import { stripVTControlCharacters } from "node:util"; + +type ErrorCodeGrammar = { line: RegExp; allow: RegExp }; + +const grammars: Record = { + // npm error code E404 (npm <= 9: npm ERR! code E404) + npm: { + line: /^npm (?:error|ERR!) code (\S+)$/, + allow: /^E[A-Z0-9_]{2,40}$/, + }, + // [ERR_PNPM_FETCH_404] ... (pnpm <= 10 pads with thin spaces) + pnpm: { + line: /^(?:\[([A-Z0-9_]+)\] |\u2009([A-Z0-9_]+)\u2009 )/, + allow: /^(?:ERR_PNPM_[A-Z0-9_]{1,60}|E[A-Z0-9_]{2,40})$/, + }, + // ➤ YN0035: ...; YN0000 is the unnamed one + yarn: { + line: /^(?:➤ )?(YN\d{4}): /, + allow: /^YN(?!0000)\d{4}$/, + }, +}; + +function getGrammar(command: string): ErrorCodeGrammar | undefined { + const name = command + .replace(/^.*[\\/]/, "") + .replace(/\.(?:cmd|exe)$/i, "") + .toLowerCase(); + return Object.hasOwn(grammars, name) ? grammars[name] : undefined; +} + +// The result always matches the manager's allow-pattern, so it cannot carry free text. +export function getPackageManagerErrorCode( + command: string, + output: { stdout: string; stderr: string }, +): string | undefined { + const grammar = getGrammar(command); + if (!grammar) return undefined; + + const lines = stripVTControlCharacters(`${output.stdout}\n${output.stderr}`).split(/\r?\n/); + for (const line of lines.reverse()) { + const match = grammar.line.exec(line); + const candidate = match?.[1] ?? match?.[2]; + if (candidate !== undefined && grammar.allow.test(candidate)) return candidate; + } + return undefined; +} diff --git a/tests/package-manager-error-code.test.ts b/tests/package-manager-error-code.test.ts new file mode 100644 index 0000000..ee82acf --- /dev/null +++ b/tests/package-manager-error-code.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, test } from "bun:test"; + +import { getPackageManagerErrorCode } from "../src/utils/package-manager-error-code"; + +describe("getPackageManagerErrorCode", () => { + test.each([ + [ + "npm", + "npm error code ETARGET\nnpm error notarget No matching version found for left-pad@99.99.99.", + "ETARGET", + ], + [ + "npm", + "npm ERR! code ETARGET\nnpm ERR! notarget No matching version found for left-pad@99.99.99.", + "ETARGET", + ], + [ + "npm", + "npm error code 3\nnpm error path /Users/jane/my-app\nnpm error command failed", + undefined, + ], + [ + "pnpm", + "[ERR_PNPM_FETCH_404] GET https://registry.npmjs.org/nope: Not Found - 404", + "ERR_PNPM_FETCH_404", + ], + [ + "pnpm", + "\u2009ERR_PNPM_NO_MATCHING_VERSION\u2009 No matching version found for left-pad@99.99.99", + "ERR_PNPM_NO_MATCHING_VERSION", + ], + [ + "yarn", + "➤ YN0000: ┌ Resolution step\n➤ YN0082: │ left-pad@npm:99.99.99: No candidates found\n➤ YN0000: · Failed with errors in 0s 403ms", + "YN0082", + ], + [ + "bun", + 'error: No version matching "99.99.99" found for specifier "left-pad" (but package exists)', + undefined, + ], + ])("%s: %s", (command, output, expected) => { + expect(getPackageManagerErrorCode(command, { stdout: output, stderr: "" })).toBe(expected); + expect(getPackageManagerErrorCode(command, { stdout: "", stderr: output })).toBe(expected); + }); + + test("never returns free text from the identifier position", () => { + for (const secret of [ + "/Users/jane/my-app", + "https://jane:hunter2@registry.example.com/", + "npm_AbCdEf0123456789", + ]) { + for (const [command, output] of [ + ["npm", `npm error code ${secret}`], + ["pnpm", `[${secret}] failed`], + ["yarn", `➤ ${secret}: │ failed`], + ] as const) { + expect( + getPackageManagerErrorCode(command, { stdout: output, stderr: output }), + ).toBeUndefined(); + } + } + }); +}); diff --git a/tests/telemetry.test.ts b/tests/telemetry.test.ts index 341677b..492af33 100644 --- a/tests/telemetry.test.ts +++ b/tests/telemetry.test.ts @@ -1,5 +1,7 @@ import { beforeEach, describe, expect, mock, test } from "bun:test"; +import { NodeFileSystem } from "@effect/platform-node-shared"; import { Effect } from "effect"; +import { existsSync, writeFileSync } from "node:fs"; import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; @@ -7,6 +9,7 @@ import path from "node:path"; import type { CreatePromptContext } from "../src/commands/create"; import type { CreateCommandInput } from "../src/types"; import { CommandExecutionError, CommandRunner } from "../src/services/command-runner"; +import { runComposerDeployEffect } from "../src/tasks/composer/deploy-report"; import { runPrismaJsonCommandEffect } from "../src/tasks/prisma-cli"; import { getChildProcessFailure, @@ -51,6 +54,40 @@ const createContext: CreatePromptContext = { beforeEach(() => trackCliTelemetry.mockClear()); +async function failComposerDeploy(report?: string) { + let reportPath = ""; + const run = (spec: { args: readonly string[] }) => + Effect.sync(() => { + reportPath = spec.args[spec.args.indexOf("--report") + 1]!; + if (report !== undefined) writeFileSync(reportPath, report); + return { + exitCode: 1, + stdout: '{"ok":false,"commandId":"deploy","error":{"code":"CLI.CHILD_PROCESS_FAILED"}}', + stderr: "", + }; + }); + const error = await Effect.runPromise( + runComposerDeployEffect({ packageManager: "npm", projectDir: process.cwd() }).pipe( + Effect.provideService(CommandRunner, { run, runChecked: run }), + Effect.provide(NodeFileSystem.layer), + Effect.flip, + ), + ); + return { error, reportPath }; +} + +async function trackFailure(error: unknown, stage: "install_dependencies" | "composer_deploy") { + await trackCreateFailed({ + input: createInput, + context: createContext, + durationMs: 10, + error, + stage, + reason: stage === "composer_deploy" ? "composer_deploy_failed" : "dependency_install_failed", + }); + return (trackCliTelemetry.mock.calls.at(-1) as unknown as [string, Record])[1]; +} + describe("create telemetry", () => { test("tracks Composer deployment intent on completion", async () => { await trackCreateCompleted({ input: createInput, context: createContext, durationMs: 123 }); @@ -264,6 +301,45 @@ describe("create telemetry", () => { } }); + test("tracks the package manager's error code for install failures", async () => { + const properties = await trackFailure( + new CommandExecutionError({ + command: "npm", + args: ["install"], + exitCode: 1, + stdout: "", + stderr: + "npm error code E404\nnpm error 404 Not Found - GET https://registry.npmjs.org/nope", + }), + "install_dependencies", + ); + expect(properties["package-manager-error-code"]).toBe("E404"); + expect(JSON.stringify(properties)).not.toContain("registry"); + }); + + test("tracks Composer's failure code from the deploy report", async () => { + const { error, reportPath } = await failComposerDeploy( + JSON.stringify({ + version: 1, + failure: { code: "DEPLOY.ENGINE_FAILED", message: "failed in /Users/jane/my-app" }, + }), + ); + const properties = await trackFailure(error, "composer_deploy"); + expect(properties["prisma-cli-error-code"]).toBe("CLI.CHILD_PROCESS_FAILED"); + expect(properties["prisma-cli-cause-code"]).toBe("DEPLOY.ENGINE_FAILED"); + expect(JSON.stringify(properties)).not.toContain("jane"); + expect(existsSync(path.dirname(reportPath))).toBe(false); + }); + + test("keeps the original deploy error when the report is missing or invalid", async () => { + for (const report of [undefined, "not json"]) { + const { error } = await failComposerDeploy(report); + expect(error).toMatchObject({ code: "CLI.CHILD_PROCESS_FAILED", exitCode: 1 }); + expect(error).not.toHaveProperty("causeCode"); + expect((await trackFailure(error, "composer_deploy"))["prisma-cli-cause-code"]).toBeNull(); + } + }); + test.each(["select_workspace", "authenticate"] as const)( "tracks %s cancellation as a separate outcome", async (stage) => {