From 66078198c3b7c8010f37c14112f728cf599a7785 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Halber?= Date: Wed, 26 Aug 2026 14:58:42 -0700 Subject: [PATCH 1/3] chore(ci): use real bt in ci, not a mock up --- .github/workflows/ci.yml | 5 + packages/spark/test/braintrust-cli.test.ts | 208 ++++++++------------- 2 files changed, 87 insertions(+), 126 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d2ff98c..714485c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,6 +81,11 @@ jobs: - uses: ./.github/actions/setup + - name: Install Braintrust CLI + run: | + curl --proto '=https' --tlsv1.2 -LsSf https://github.com/braintrustdata/bt/releases/latest/download/bt-installer.sh | sh + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + - name: Run tests run: pnpm test diff --git a/packages/spark/test/braintrust-cli.test.ts b/packages/spark/test/braintrust-cli.test.ts index 5b20c7a..137b9f4 100644 --- a/packages/spark/test/braintrust-cli.test.ts +++ b/packages/spark/test/braintrust-cli.test.ts @@ -1,143 +1,99 @@ +import { createServer } from "node:http"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + import { describe, expect, it } from "vitest"; import { createBraintrustCliRuntime } from "../src/braintrust-cli"; describe("Braintrust CLI runtime", () => { - it("builds the Unix installer command", async () => { - const calls: Array<{ - readonly command: string; - readonly args: readonly string[]; - readonly env?: NodeJS.ProcessEnv; - }> = []; - const runtime = createBraintrustCliRuntime({ - platform: "darwin", - env: { PATH: "/usr/bin" }, - exec: (spec) => { - calls.push(spec); - return Promise.resolve({ - exitCode: 0, - signal: null, - stdout: "", - stderr: "", - }); - }, - }); + it("configures and reads context using the real bt CLI", async () => { + const home = await mkdtemp(join(tmpdir(), "braintrust-cli-test-")); + const server = createServer((request, response) => { + response.setHeader("content-type", "application/json"); - await runtime.install(); + if (request.method === "POST" && request.url === "/api/apikey/login") { + response.end( + JSON.stringify({ + org_info: [ + { + id: "org-id", + name: "acme", + api_url: serverUrl(server), + }, + ], + }), + ); + return; + } - expect(calls).toEqual([ - { - command: "sh", - args: [ - "-c", - "curl -fsSL https://bt.dev/cli/install.sh | bash -s -- --quiet", - ], - env: { PATH: "/usr/bin" }, - }, - ]); - }); + if ( + request.method === "GET" && + request.url === "/v1/project?org_name=acme&project_name=demo" + ) { + response.end( + JSON.stringify({ + objects: [{ id: "project-id", name: "demo", org_id: "org-id" }], + }), + ); + return; + } - it("builds the update command", async () => { - const calls: Array<{ - readonly command: string; - readonly args: readonly string[]; - readonly env?: NodeJS.ProcessEnv; - }> = []; - const runtime = createBraintrustCliRuntime({ - env: { PATH: "/usr/bin" }, - exec: (spec) => { - calls.push(spec); - return Promise.resolve({ - exitCode: 0, - signal: null, - stdout: "", - stderr: "", - }); - }, + response.statusCode = 404; + response.end(JSON.stringify({ error: "not found" })); }); - await runtime.update("/usr/local/bin/bt"); - - expect(calls).toEqual([ - { - command: "/usr/local/bin/bt", - args: ["self", "update"], - env: { PATH: "/usr/bin" }, - }, - ]); - }); - - it("passes the API key only through env when configuring auth and context", async () => { - const calls: Array<{ - readonly command: string; - readonly args: readonly string[]; - readonly env?: NodeJS.ProcessEnv; - }> = []; - const runtime = createBraintrustCliRuntime({ - env: { PATH: "/usr/bin" }, - exec: (spec) => { - calls.push(spec); - return Promise.resolve({ - exitCode: 0, - signal: null, - stdout: "", - stderr: "", - }); - }, + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); }); - await runtime.loginAndSwitch("/usr/local/bin/bt", { - apiKey: "bt-secret-key", - apiUrl: "https://api.test", - appUrl: "https://app.test", - orgName: "acme", - projectName: "demo", - }); + try { + const url = serverUrl(server); + const runtime = createBraintrustCliRuntime({ + env: { + ...process.env, + HOME: home, + XDG_CONFIG_HOME: join(home, ".config"), + BRAINTRUST_API_URL: url, + BRAINTRUST_APP_URL: url, + }, + }); + const discovery = await runtime.discover(); - expect(calls).toHaveLength(2); - expect(calls[0]?.args).toEqual([ - "login", - "--profile=acme", - "--no-input", - "--quiet", - ]); - expect(calls[1]?.args).toEqual([ - "switch", - "--profile=acme", - "--org=acme", - "--no-input", - "--quiet", - "--global", - "demo", - ]); - expect(calls.flatMap((call) => [...call.args])).not.toContain( - "bt-secret-key", - ); - expect(calls[0]?.env?.["BRAINTRUST_API_KEY"]).toBe("bt-secret-key"); - expect(calls[0]?.env?.["BRAINTRUST_API_URL"]).toBe("https://api.test"); - expect(calls[0]?.env?.["BRAINTRUST_APP_URL"]).toBe("https://app.test"); - expect(calls[1]?.env?.["BRAINTRUST_API_KEY"]).toBe("bt-secret-key"); - }); + expect(discovery).toMatchObject({ installed: true }); + expect(discovery.commandPath).toBeDefined(); + expect(discovery.version).toBeDefined(); - it("parses bt status JSON", async () => { - const runtime = createBraintrustCliRuntime({ - exec: () => - Promise.resolve({ - exitCode: 0, - signal: null, - stdout: JSON.stringify({ - profile: "work", - org: "acme", - project: "demo", - }), - stderr: "", - }), - }); + await runtime.loginAndSwitch(discovery.commandPath!, { + apiKey: "bt-secret-key", + apiUrl: url, + appUrl: url, + orgName: "acme", + projectName: "demo", + }); - await expect(runtime.status("/bin/bt")).resolves.toEqual({ - profile: "work", - org: "acme", - project: "demo", - }); + await expect( + runtime.status(discovery.commandPath!), + ).resolves.toMatchObject({ + profile: "acme", + org: "acme", + project: "demo", + }); + } finally { + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + await rm(home, { recursive: true, force: true }); + } }); }); + +function serverUrl(server: ReturnType): string { + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("Test API server is not listening on a TCP port."); + } + return `http://127.0.0.1:${address.port}`; +} From fa70395774093fd74a9fd0e2ffcf91467268b987 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Halber?= Date: Wed, 26 Aug 2026 15:28:10 -0700 Subject: [PATCH 2/3] chore: construct URLs with Node URL --- packages/spark/src/auth.ts | 10 +++++++--- packages/spark/src/braintrust-api.ts | 5 ++++- packages/spark/src/clack-wizard.ts | 6 +++++- packages/spark/src/cleanup.ts | 13 ++++++++---- packages/spark/src/events.ts | 23 +++++++++++----------- packages/spark/test/braintrust-cli.test.ts | 17 +++++++++------- 6 files changed, 46 insertions(+), 28 deletions(-) diff --git a/packages/spark/src/auth.ts b/packages/spark/src/auth.ts index d7a74c4..11dffda 100644 --- a/packages/spark/src/auth.ts +++ b/packages/spark/src/auth.ts @@ -1,3 +1,5 @@ +import { URL } from "node:url"; + import type { CliSetupClientContext } from "./setup-events-contract"; export type WizardSessionCreateResponse = { @@ -81,7 +83,8 @@ export async function createWizardSession( clientContext?: WizardSessionCreateClientContext, signal?: AbortSignal, ): Promise { - const res = await fetch(`${appUrl}/api/cli/wizard-session/create`, { + const url = new URL("/api/cli/wizard-session/create", appUrl); + const res = await fetch(url.href, { method: "POST", headers: { Accept: "application/json", @@ -137,10 +140,11 @@ export async function pollWizardSession(args: { const deadline = Date.now() + POLL_HARD_TIMEOUT_MS; while (Date.now() < deadline) { await sleep(interval); - const url = `${args.appUrl}/api/cli/wizard-session/poll?session_token=${encodeURIComponent(args.sessionToken)}`; + const url = new URL("/api/cli/wizard-session/poll", args.appUrl); + url.searchParams.set("session_token", args.sessionToken); let res: Response; try { - res = await fetch(url, { + res = await fetch(url.href, { method: "GET", headers: { Authorization: `Bearer ${args.pollToken}`, diff --git a/packages/spark/src/braintrust-api.ts b/packages/spark/src/braintrust-api.ts index d969148..c3ab8d9 100644 --- a/packages/spark/src/braintrust-api.ts +++ b/packages/spark/src/braintrust-api.ts @@ -1,3 +1,5 @@ +import { URL } from "node:url"; + export type Org = { readonly id: string; readonly name: string; @@ -22,7 +24,8 @@ export class BraintrustApiClient { path: string, body?: unknown, ): Promise { - const res = await fetch(`${this.apiUrl}${path}`, { + const url = new URL(path, this.apiUrl); + const res = await fetch(url.href, { method, headers: { Authorization: `Bearer ${this.token}`, diff --git a/packages/spark/src/clack-wizard.ts b/packages/spark/src/clack-wizard.ts index cefb631..76a4bed 100644 --- a/packages/spark/src/clack-wizard.ts +++ b/packages/spark/src/clack-wizard.ts @@ -1,5 +1,6 @@ import { cwd as processCwd } from "node:process"; import { relative } from "node:path"; +import { URL } from "node:url"; import * as clack from "@clack/prompts"; import clipboard from "clipboardy"; @@ -608,7 +609,10 @@ async function runClackWizardFlow( events.finishStep(instrumentationRunStep, "completed"); } - const projectLogsUrl = `${deps.options.appUrl}/app/${encodeURIComponent(session.orgName)}/p/${encodeURIComponent(session.projectName)}/logs`; + const projectLogsUrl = new URL( + `/app/${encodeURIComponent(session.orgName)}/p/${encodeURIComponent(session.projectName)}/logs`, + deps.options.appUrl, + ).href; const traceVerificationStep = events.startStep("trace_verification", { failureCategory: "trace_not_observed", }); diff --git a/packages/spark/src/cleanup.ts b/packages/spark/src/cleanup.ts index 2543f9f..597ba77 100644 --- a/packages/spark/src/cleanup.ts +++ b/packages/spark/src/cleanup.ts @@ -1,3 +1,5 @@ +import { URL } from "node:url"; + /** * URL formats from /workspace/bt-main/skills/sdk-install/braintrust-url-formats.md. * `appUrl` here is the *base* (e.g. https://www.braintrust.dev) — the docs reference @@ -14,10 +16,13 @@ export function buildLogsPermalink( appUrl: string, trace: TraceLocation, ): string { - const base = `${appUrl}/app/${encodeURIComponent(trace.org)}/p/${encodeURIComponent(trace.project)}/logs`; - const params = new URLSearchParams({ r: trace.rootSpanId }); + const url = new URL( + `/app/${encodeURIComponent(trace.org)}/p/${encodeURIComponent(trace.project)}/logs`, + appUrl, + ); + url.searchParams.set("r", trace.rootSpanId); if (trace.spanId) { - params.set("s", trace.spanId); + url.searchParams.set("s", trace.spanId); } - return `${base}?${params.toString()}`; + return url.href; } diff --git a/packages/spark/src/events.ts b/packages/spark/src/events.ts index 717c747..cd51f4a 100644 --- a/packages/spark/src/events.ts +++ b/packages/spark/src/events.ts @@ -1,4 +1,5 @@ import { existsSync } from "node:fs"; +import { URL } from "node:url"; import pkg from "../package.json" with { type: "json" }; import type { WizardSessionCreateResponse } from "./auth"; @@ -409,19 +410,17 @@ export function createWizardEvents(args: { const session = await start(); if (!session?.event_token) return; try { - const response = await fetchRequest( - `${DEFAULT_APP_URL}/api/cli/wizard-session/event`, - { - method: "POST", - headers: { - Authorization: `Bearer ${session.event_token}`, - Accept: "application/json", - "Content-Type": "application/json", - }, - body: JSON.stringify(event), - signal: AbortSignal.timeout(EVENT_REQUEST_TIMEOUT_MS), + const url = new URL("/api/cli/wizard-session/event", DEFAULT_APP_URL); + const response = await fetchRequest(url.href, { + method: "POST", + headers: { + Authorization: `Bearer ${session.event_token}`, + Accept: "application/json", + "Content-Type": "application/json", }, - ); + body: JSON.stringify(event), + signal: AbortSignal.timeout(EVENT_REQUEST_TIMEOUT_MS), + }); if (response.ok) return; void response.body?.cancel().catch(() => { // Discarding an error response is also best-effort. Some stream diff --git a/packages/spark/test/braintrust-cli.test.ts b/packages/spark/test/braintrust-cli.test.ts index 137b9f4..53e82f4 100644 --- a/packages/spark/test/braintrust-cli.test.ts +++ b/packages/spark/test/braintrust-cli.test.ts @@ -2,6 +2,7 @@ import { createServer } from "node:http"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { URL } from "node:url"; import { describe, expect, it } from "vitest"; @@ -20,7 +21,7 @@ describe("Braintrust CLI runtime", () => { { id: "org-id", name: "acme", - api_url: serverUrl(server), + api_url: serverUrl(server).href, }, ], }), @@ -56,8 +57,8 @@ describe("Braintrust CLI runtime", () => { ...process.env, HOME: home, XDG_CONFIG_HOME: join(home, ".config"), - BRAINTRUST_API_URL: url, - BRAINTRUST_APP_URL: url, + BRAINTRUST_API_URL: url.href, + BRAINTRUST_APP_URL: url.href, }, }); const discovery = await runtime.discover(); @@ -68,8 +69,8 @@ describe("Braintrust CLI runtime", () => { await runtime.loginAndSwitch(discovery.commandPath!, { apiKey: "bt-secret-key", - apiUrl: url, - appUrl: url, + apiUrl: url.href, + appUrl: url.href, orgName: "acme", projectName: "demo", }); @@ -90,10 +91,12 @@ describe("Braintrust CLI runtime", () => { }); }); -function serverUrl(server: ReturnType): string { +function serverUrl(server: ReturnType): URL { const address = server.address(); if (!address || typeof address === "string") { throw new Error("Test API server is not listening on a TCP port."); } - return `http://127.0.0.1:${address.port}`; + const url = new URL("http://127.0.0.1"); + url.port = String(address.port); + return url; } From 213f610677b6230123fe545330f08ee2e9427ea5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Halber?= Date: Wed, 26 Aug 2026 15:28:17 -0700 Subject: [PATCH 3/3] test(ci): authenticate bt with service token --- .github/workflows/ci.yml | 2 + packages/spark/test/braintrust-cli.test.ts | 162 +++++++++++---------- 2 files changed, 84 insertions(+), 80 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 714485c..682c0db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,6 +87,8 @@ jobs: echo "$HOME/.local/bin" >> "$GITHUB_PATH" - name: Run tests + env: + BRAINTRUST_SERVICE_TOKEN: ${{ secrets.BRAINTRUST_SERVICE_TOKEN }} run: pnpm test lint: diff --git a/packages/spark/test/braintrust-cli.test.ts b/packages/spark/test/braintrust-cli.test.ts index 53e82f4..75958bb 100644 --- a/packages/spark/test/braintrust-cli.test.ts +++ b/packages/spark/test/braintrust-cli.test.ts @@ -1,4 +1,3 @@ -import { createServer } from "node:http"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -7,96 +6,99 @@ import { URL } from "node:url"; import { describe, expect, it } from "vitest"; import { createBraintrustCliRuntime } from "../src/braintrust-cli"; +import { DEFAULT_API_URL, DEFAULT_APP_URL } from "../src/options"; describe("Braintrust CLI runtime", () => { - it("configures and reads context using the real bt CLI", async () => { - const home = await mkdtemp(join(tmpdir(), "braintrust-cli-test-")); - const server = createServer((request, response) => { - response.setHeader("content-type", "application/json"); - - if (request.method === "POST" && request.url === "/api/apikey/login") { - response.end( - JSON.stringify({ - org_info: [ - { - id: "org-id", - name: "acme", - api_url: serverUrl(server).href, - }, - ], - }), - ); - return; + it.runIf(process.env.CI === "true")( + "configures and reads context using the real bt CLI", + async () => { + const serviceToken = process.env.BRAINTRUST_SERVICE_TOKEN; + if (!serviceToken) { + throw new Error("BRAINTRUST_SERVICE_TOKEN is required in CI."); } - if ( - request.method === "GET" && - request.url === "/v1/project?org_name=acme&project_name=demo" - ) { - response.end( - JSON.stringify({ - objects: [{ id: "project-id", name: "demo", org_id: "org-id" }], - }), - ); - return; - } + const target = await discoverTestTarget(serviceToken); + const home = await mkdtemp(join(tmpdir(), "braintrust-cli-test-")); + + try { + const runtime = createBraintrustCliRuntime({ + env: { + ...process.env, + HOME: home, + XDG_CONFIG_HOME: join(home, ".config"), + }, + }); + const discovery = await runtime.discover(); - response.statusCode = 404; - response.end(JSON.stringify({ error: "not found" })); - }); + expect(discovery).toMatchObject({ installed: true }); + expect(discovery.commandPath).toBeDefined(); + expect(discovery.version).toBeDefined(); - await new Promise((resolve, reject) => { - server.once("error", reject); - server.listen(0, "127.0.0.1", resolve); - }); + await runtime.loginAndSwitch(discovery.commandPath!, { + apiKey: serviceToken, + apiUrl: target.apiUrl, + appUrl: DEFAULT_APP_URL, + orgName: target.orgName, + projectName: target.projectName, + }); - try { - const url = serverUrl(server); - const runtime = createBraintrustCliRuntime({ - env: { - ...process.env, - HOME: home, - XDG_CONFIG_HOME: join(home, ".config"), - BRAINTRUST_API_URL: url.href, - BRAINTRUST_APP_URL: url.href, - }, - }); - const discovery = await runtime.discover(); + await expect( + runtime.status(discovery.commandPath!), + ).resolves.toMatchObject({ + profile: target.orgName, + org: target.orgName, + project: target.projectName, + }); + } finally { + await rm(home, { recursive: true, force: true }); + } + }, + 30_000, + ); +}); - expect(discovery).toMatchObject({ installed: true }); - expect(discovery.commandPath).toBeDefined(); - expect(discovery.version).toBeDefined(); +async function discoverTestTarget(serviceToken: string): Promise<{ + readonly apiUrl: string; + readonly orgName: string; + readonly projectName: string; +}> { + const loginUrl = new URL("/api/apikey/login", DEFAULT_APP_URL); + const loginResponse = await fetch(loginUrl, { + method: "POST", + headers: { Authorization: `Bearer ${serviceToken}` }, + }); + if (!loginResponse.ok) { + throw new Error(`Braintrust login failed with ${loginResponse.status}.`); + } - await runtime.loginAndSwitch(discovery.commandPath!, { - apiKey: "bt-secret-key", - apiUrl: url.href, - appUrl: url.href, - orgName: "acme", - projectName: "demo", - }); + const login = (await loginResponse.json()) as { + readonly org_info?: readonly { + readonly name: string; + readonly api_url?: string | null; + }[]; + }; + const org = login.org_info?.[0]; + if (!org) throw new Error("The CI service token has no Braintrust org."); - await expect( - runtime.status(discovery.commandPath!), - ).resolves.toMatchObject({ - profile: "acme", - org: "acme", - project: "demo", - }); - } finally { - await new Promise((resolve, reject) => - server.close((error) => (error ? reject(error) : resolve())), - ); - await rm(home, { recursive: true, force: true }); - } + const apiUrl = org.api_url ?? DEFAULT_API_URL; + const projectsUrl = new URL("/v1/project", apiUrl); + projectsUrl.searchParams.set("org_name", org.name); + const projectsResponse = await fetch(projectsUrl, { + headers: { Authorization: `Bearer ${serviceToken}` }, }); -}); + if (!projectsResponse.ok) { + throw new Error( + `Braintrust project lookup failed with ${projectsResponse.status}.`, + ); + } -function serverUrl(server: ReturnType): URL { - const address = server.address(); - if (!address || typeof address === "string") { - throw new Error("Test API server is not listening on a TCP port."); + const projects = (await projectsResponse.json()) as { + readonly objects?: readonly { readonly name: string }[]; + }; + const project = projects.objects?.[0]; + if (!project) { + throw new Error("The CI service token's Braintrust org has no project."); } - const url = new URL("http://127.0.0.1"); - url.port = String(address.port); - return url; + + return { apiUrl, orgName: org.name, projectName: project.name }; }