diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d2ff98c..682c0db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,7 +81,14 @@ jobs: - uses: ./.github/actions/setup + - name: Install Braintrust CLI + run: | + curl --proto '=https' --tlsv1.2 -LsSf https://github.com/braintrustdata/bt/releases/latest/download/bt-installer.sh | sh + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + - name: Run tests + env: + BRAINTRUST_SERVICE_TOKEN: ${{ secrets.BRAINTRUST_SERVICE_TOKEN }} run: pnpm test lint: diff --git a/packages/spark/src/auth.ts b/packages/spark/src/auth.ts index d7a74c4..11dffda 100644 --- a/packages/spark/src/auth.ts +++ b/packages/spark/src/auth.ts @@ -1,3 +1,5 @@ +import { URL } from "node:url"; + import type { CliSetupClientContext } from "./setup-events-contract"; export type WizardSessionCreateResponse = { @@ -81,7 +83,8 @@ export async function createWizardSession( clientContext?: WizardSessionCreateClientContext, signal?: AbortSignal, ): Promise { - const res = await fetch(`${appUrl}/api/cli/wizard-session/create`, { + const url = new URL("/api/cli/wizard-session/create", appUrl); + const res = await fetch(url.href, { method: "POST", headers: { Accept: "application/json", @@ -137,10 +140,11 @@ export async function pollWizardSession(args: { const deadline = Date.now() + POLL_HARD_TIMEOUT_MS; while (Date.now() < deadline) { await sleep(interval); - const url = `${args.appUrl}/api/cli/wizard-session/poll?session_token=${encodeURIComponent(args.sessionToken)}`; + const url = new URL("/api/cli/wizard-session/poll", args.appUrl); + url.searchParams.set("session_token", args.sessionToken); let res: Response; try { - res = await fetch(url, { + res = await fetch(url.href, { method: "GET", headers: { Authorization: `Bearer ${args.pollToken}`, diff --git a/packages/spark/src/braintrust-api.ts b/packages/spark/src/braintrust-api.ts index d969148..c3ab8d9 100644 --- a/packages/spark/src/braintrust-api.ts +++ b/packages/spark/src/braintrust-api.ts @@ -1,3 +1,5 @@ +import { URL } from "node:url"; + export type Org = { readonly id: string; readonly name: string; @@ -22,7 +24,8 @@ export class BraintrustApiClient { path: string, body?: unknown, ): Promise { - const res = await fetch(`${this.apiUrl}${path}`, { + const url = new URL(path, this.apiUrl); + const res = await fetch(url.href, { method, headers: { Authorization: `Bearer ${this.token}`, diff --git a/packages/spark/src/clack-wizard.ts b/packages/spark/src/clack-wizard.ts index cefb631..76a4bed 100644 --- a/packages/spark/src/clack-wizard.ts +++ b/packages/spark/src/clack-wizard.ts @@ -1,5 +1,6 @@ import { cwd as processCwd } from "node:process"; import { relative } from "node:path"; +import { URL } from "node:url"; import * as clack from "@clack/prompts"; import clipboard from "clipboardy"; @@ -608,7 +609,10 @@ async function runClackWizardFlow( events.finishStep(instrumentationRunStep, "completed"); } - const projectLogsUrl = `${deps.options.appUrl}/app/${encodeURIComponent(session.orgName)}/p/${encodeURIComponent(session.projectName)}/logs`; + const projectLogsUrl = new URL( + `/app/${encodeURIComponent(session.orgName)}/p/${encodeURIComponent(session.projectName)}/logs`, + deps.options.appUrl, + ).href; const traceVerificationStep = events.startStep("trace_verification", { failureCategory: "trace_not_observed", }); diff --git a/packages/spark/src/cleanup.ts b/packages/spark/src/cleanup.ts index 2543f9f..597ba77 100644 --- a/packages/spark/src/cleanup.ts +++ b/packages/spark/src/cleanup.ts @@ -1,3 +1,5 @@ +import { URL } from "node:url"; + /** * URL formats from /workspace/bt-main/skills/sdk-install/braintrust-url-formats.md. * `appUrl` here is the *base* (e.g. https://www.braintrust.dev) — the docs reference @@ -14,10 +16,13 @@ export function buildLogsPermalink( appUrl: string, trace: TraceLocation, ): string { - const base = `${appUrl}/app/${encodeURIComponent(trace.org)}/p/${encodeURIComponent(trace.project)}/logs`; - const params = new URLSearchParams({ r: trace.rootSpanId }); + const url = new URL( + `/app/${encodeURIComponent(trace.org)}/p/${encodeURIComponent(trace.project)}/logs`, + appUrl, + ); + url.searchParams.set("r", trace.rootSpanId); if (trace.spanId) { - params.set("s", trace.spanId); + url.searchParams.set("s", trace.spanId); } - return `${base}?${params.toString()}`; + return url.href; } diff --git a/packages/spark/src/events.ts b/packages/spark/src/events.ts index 717c747..cd51f4a 100644 --- a/packages/spark/src/events.ts +++ b/packages/spark/src/events.ts @@ -1,4 +1,5 @@ import { existsSync } from "node:fs"; +import { URL } from "node:url"; import pkg from "../package.json" with { type: "json" }; import type { WizardSessionCreateResponse } from "./auth"; @@ -409,19 +410,17 @@ export function createWizardEvents(args: { const session = await start(); if (!session?.event_token) return; try { - const response = await fetchRequest( - `${DEFAULT_APP_URL}/api/cli/wizard-session/event`, - { - method: "POST", - headers: { - Authorization: `Bearer ${session.event_token}`, - Accept: "application/json", - "Content-Type": "application/json", - }, - body: JSON.stringify(event), - signal: AbortSignal.timeout(EVENT_REQUEST_TIMEOUT_MS), + const url = new URL("/api/cli/wizard-session/event", DEFAULT_APP_URL); + const response = await fetchRequest(url.href, { + method: "POST", + headers: { + Authorization: `Bearer ${session.event_token}`, + Accept: "application/json", + "Content-Type": "application/json", }, - ); + body: JSON.stringify(event), + signal: AbortSignal.timeout(EVENT_REQUEST_TIMEOUT_MS), + }); if (response.ok) return; void response.body?.cancel().catch(() => { // Discarding an error response is also best-effort. Some stream diff --git a/packages/spark/test/braintrust-cli.test.ts b/packages/spark/test/braintrust-cli.test.ts index 5b20c7a..75958bb 100644 --- a/packages/spark/test/braintrust-cli.test.ts +++ b/packages/spark/test/braintrust-cli.test.ts @@ -1,143 +1,104 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { URL } from "node:url"; + import { describe, expect, it } from "vitest"; import { createBraintrustCliRuntime } from "../src/braintrust-cli"; +import { DEFAULT_API_URL, DEFAULT_APP_URL } from "../src/options"; describe("Braintrust CLI runtime", () => { - it("builds the Unix installer command", async () => { - const calls: Array<{ - readonly command: string; - readonly args: readonly string[]; - readonly env?: NodeJS.ProcessEnv; - }> = []; - const runtime = createBraintrustCliRuntime({ - platform: "darwin", - env: { PATH: "/usr/bin" }, - exec: (spec) => { - calls.push(spec); - return Promise.resolve({ - exitCode: 0, - signal: null, - stdout: "", - stderr: "", - }); - }, - }); + it.runIf(process.env.CI === "true")( + "configures and reads context using the real bt CLI", + async () => { + const serviceToken = process.env.BRAINTRUST_SERVICE_TOKEN; + if (!serviceToken) { + throw new Error("BRAINTRUST_SERVICE_TOKEN is required in CI."); + } - await runtime.install(); - - expect(calls).toEqual([ - { - command: "sh", - args: [ - "-c", - "curl -fsSL https://bt.dev/cli/install.sh | bash -s -- --quiet", - ], - env: { PATH: "/usr/bin" }, - }, - ]); - }); + const target = await discoverTestTarget(serviceToken); + const home = await mkdtemp(join(tmpdir(), "braintrust-cli-test-")); - it("builds the update command", async () => { - const calls: Array<{ - readonly command: string; - readonly args: readonly string[]; - readonly env?: NodeJS.ProcessEnv; - }> = []; - const runtime = createBraintrustCliRuntime({ - env: { PATH: "/usr/bin" }, - exec: (spec) => { - calls.push(spec); - return Promise.resolve({ - exitCode: 0, - signal: null, - stdout: "", - stderr: "", + try { + const runtime = createBraintrustCliRuntime({ + env: { + ...process.env, + HOME: home, + XDG_CONFIG_HOME: join(home, ".config"), + }, }); - }, - }); + const discovery = await runtime.discover(); - await runtime.update("/usr/local/bin/bt"); + expect(discovery).toMatchObject({ installed: true }); + expect(discovery.commandPath).toBeDefined(); + expect(discovery.version).toBeDefined(); - expect(calls).toEqual([ - { - command: "/usr/local/bin/bt", - args: ["self", "update"], - env: { PATH: "/usr/bin" }, - }, - ]); - }); - - it("passes the API key only through env when configuring auth and context", async () => { - const calls: Array<{ - readonly command: string; - readonly args: readonly string[]; - readonly env?: NodeJS.ProcessEnv; - }> = []; - const runtime = createBraintrustCliRuntime({ - env: { PATH: "/usr/bin" }, - exec: (spec) => { - calls.push(spec); - return Promise.resolve({ - exitCode: 0, - signal: null, - stdout: "", - stderr: "", + await runtime.loginAndSwitch(discovery.commandPath!, { + apiKey: serviceToken, + apiUrl: target.apiUrl, + appUrl: DEFAULT_APP_URL, + orgName: target.orgName, + projectName: target.projectName, }); - }, - }); - await runtime.loginAndSwitch("/usr/local/bin/bt", { - apiKey: "bt-secret-key", - apiUrl: "https://api.test", - appUrl: "https://app.test", - orgName: "acme", - projectName: "demo", - }); + await expect( + runtime.status(discovery.commandPath!), + ).resolves.toMatchObject({ + profile: target.orgName, + org: target.orgName, + project: target.projectName, + }); + } finally { + await rm(home, { recursive: true, force: true }); + } + }, + 30_000, + ); +}); - expect(calls).toHaveLength(2); - expect(calls[0]?.args).toEqual([ - "login", - "--profile=acme", - "--no-input", - "--quiet", - ]); - expect(calls[1]?.args).toEqual([ - "switch", - "--profile=acme", - "--org=acme", - "--no-input", - "--quiet", - "--global", - "demo", - ]); - expect(calls.flatMap((call) => [...call.args])).not.toContain( - "bt-secret-key", - ); - expect(calls[0]?.env?.["BRAINTRUST_API_KEY"]).toBe("bt-secret-key"); - expect(calls[0]?.env?.["BRAINTRUST_API_URL"]).toBe("https://api.test"); - expect(calls[0]?.env?.["BRAINTRUST_APP_URL"]).toBe("https://app.test"); - expect(calls[1]?.env?.["BRAINTRUST_API_KEY"]).toBe("bt-secret-key"); +async function discoverTestTarget(serviceToken: string): Promise<{ + readonly apiUrl: string; + readonly orgName: string; + readonly projectName: string; +}> { + const loginUrl = new URL("/api/apikey/login", DEFAULT_APP_URL); + const loginResponse = await fetch(loginUrl, { + method: "POST", + headers: { Authorization: `Bearer ${serviceToken}` }, }); + if (!loginResponse.ok) { + throw new Error(`Braintrust login failed with ${loginResponse.status}.`); + } - it("parses bt status JSON", async () => { - const runtime = createBraintrustCliRuntime({ - exec: () => - Promise.resolve({ - exitCode: 0, - signal: null, - stdout: JSON.stringify({ - profile: "work", - org: "acme", - project: "demo", - }), - stderr: "", - }), - }); + const login = (await loginResponse.json()) as { + readonly org_info?: readonly { + readonly name: string; + readonly api_url?: string | null; + }[]; + }; + const org = login.org_info?.[0]; + if (!org) throw new Error("The CI service token has no Braintrust org."); - await expect(runtime.status("/bin/bt")).resolves.toEqual({ - profile: "work", - org: "acme", - project: "demo", - }); + const apiUrl = org.api_url ?? DEFAULT_API_URL; + const projectsUrl = new URL("/v1/project", apiUrl); + projectsUrl.searchParams.set("org_name", org.name); + const projectsResponse = await fetch(projectsUrl, { + headers: { Authorization: `Bearer ${serviceToken}` }, }); -}); + if (!projectsResponse.ok) { + throw new Error( + `Braintrust project lookup failed with ${projectsResponse.status}.`, + ); + } + + const projects = (await projectsResponse.json()) as { + readonly objects?: readonly { readonly name: string }[]; + }; + const project = projects.objects?.[0]; + if (!project) { + throw new Error("The CI service token's Braintrust org has no project."); + } + + return { apiUrl, orgName: org.name, projectName: project.name }; +}