chore(ci): use real bt in ci, not a mock up

.github/workflows/ci.yml

@@ -81,7 +81,14 @@ jobs:
 
       - uses: ./.github/actions/setup
 
+      - name: Install Braintrust CLI
+        run: |
+          curl --proto '=https' --tlsv1.2 -LsSf https://github.com/braintrustdata/bt/releases/latest/download/bt-installer.sh | sh
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+
       - name: Run tests
+        env:
+          BRAINTRUST_SERVICE_TOKEN: ${{ secrets.BRAINTRUST_SERVICE_TOKEN }}
         run: pnpm test
 
   lint:

packages/spark/src/auth.ts

@@ -1,3 +1,5 @@
+import { URL } from "node:url";
+
 import type { CliSetupClientContext } from "./setup-events-contract";
 
 export type WizardSessionCreateResponse = {
@@ -81,7 +83,8 @@ export async function createWizardSession(
   clientContext?: WizardSessionCreateClientContext,
   signal?: AbortSignal,
 ): Promise<WizardSessionCreateResponse> {
-  const res = await fetch(`${appUrl}/api/cli/wizard-session/create`, {
+  const url = new URL("/api/cli/wizard-session/create", appUrl);
+  const res = await fetch(url.href, {
     method: "POST",
     headers: {
       Accept: "application/json",
@@ -137,10 +140,11 @@ export async function pollWizardSession(args: {
   const deadline = Date.now() + POLL_HARD_TIMEOUT_MS;
   while (Date.now() < deadline) {
     await sleep(interval);
-    const url = `${args.appUrl}/api/cli/wizard-session/poll?session_token=${encodeURIComponent(args.sessionToken)}`;
+    const url = new URL("/api/cli/wizard-session/poll", args.appUrl);
+    url.searchParams.set("session_token", args.sessionToken);
     let res: Response;
     try {
-      res = await fetch(url, {
+      res = await fetch(url.href, {
         method: "GET",
         headers: {
           Authorization: `Bearer ${args.pollToken}`,

packages/spark/src/braintrust-api.ts

@@ -1,3 +1,5 @@
+import { URL } from "node:url";
+
 export type Org = {
   readonly id: string;
   readonly name: string;
@@ -22,7 +24,8 @@ export class BraintrustApiClient {
     path: string,
     body?: unknown,
   ): Promise<T> {
-    const res = await fetch(`${this.apiUrl}${path}`, {
+    const url = new URL(path, this.apiUrl);
+    const res = await fetch(url.href, {
       method,
       headers: {
         Authorization: `Bearer ${this.token}`,

packages/spark/src/clack-wizard.ts

@@ -1,5 +1,6 @@
 import { cwd as processCwd } from "node:process";
 import { relative } from "node:path";
+import { URL } from "node:url";
 
 import * as clack from "@clack/prompts";
 import clipboard from "clipboardy";
@@ -608,7 +609,10 @@ async function runClackWizardFlow(
     events.finishStep(instrumentationRunStep, "completed");
   }
 
-  const projectLogsUrl = `${deps.options.appUrl}/app/${encodeURIComponent(session.orgName)}/p/${encodeURIComponent(session.projectName)}/logs`;
+  const projectLogsUrl = new URL(
+    `/app/${encodeURIComponent(session.orgName)}/p/${encodeURIComponent(session.projectName)}/logs`,
+    deps.options.appUrl,
+  ).href;
   const traceVerificationStep = events.startStep("trace_verification", {
     failureCategory: "trace_not_observed",
   });

packages/spark/src/cleanup.ts

@@ -1,3 +1,5 @@
+import { URL } from "node:url";
+
 /**
  * URL formats from /workspace/bt-main/skills/sdk-install/braintrust-url-formats.md.
  * `appUrl` here is the *base* (e.g. https://www.braintrust.dev) — the docs reference
@@ -14,10 +16,13 @@ export function buildLogsPermalink(
   appUrl: string,
   trace: TraceLocation,
 ): string {
-  const base = `${appUrl}/app/${encodeURIComponent(trace.org)}/p/${encodeURIComponent(trace.project)}/logs`;
-  const params = new URLSearchParams({ r: trace.rootSpanId });
+  const url = new URL(
+    `/app/${encodeURIComponent(trace.org)}/p/${encodeURIComponent(trace.project)}/logs`,
+    appUrl,
+  );
+  url.searchParams.set("r", trace.rootSpanId);
   if (trace.spanId) {
-    params.set("s", trace.spanId);
+    url.searchParams.set("s", trace.spanId);
   }
-  return `${base}?${params.toString()}`;
+  return url.href;
 }

packages/spark/src/events.ts

@@ -1,4 +1,5 @@
 import { existsSync } from "node:fs";
+import { URL } from "node:url";
 
 import pkg from "../package.json" with { type: "json" };
 import type { WizardSessionCreateResponse } from "./auth";
@@ -409,19 +410,17 @@ export function createWizardEvents(args: {
       const session = await start();
       if (!session?.event_token) return;
       try {
-        const response = await fetchRequest(
-          `${DEFAULT_APP_URL}/api/cli/wizard-session/event`,
-          {
-            method: "POST",
-            headers: {
-              Authorization: `Bearer ${session.event_token}`,
-              Accept: "application/json",
-              "Content-Type": "application/json",
-            },
-            body: JSON.stringify(event),
-            signal: AbortSignal.timeout(EVENT_REQUEST_TIMEOUT_MS),
+        const url = new URL("/api/cli/wizard-session/event", DEFAULT_APP_URL);
+        const response = await fetchRequest(url.href, {
+          method: "POST",
+          headers: {
+            Authorization: `Bearer ${session.event_token}`,
+            Accept: "application/json",
+            "Content-Type": "application/json",
           },
-        );
+          body: JSON.stringify(event),
+          signal: AbortSignal.timeout(EVENT_REQUEST_TIMEOUT_MS),
+        });
         if (response.ok) return;
         void response.body?.cancel().catch(() => {
           // Discarding an error response is also best-effort. Some stream

packages/spark/test/braintrust-cli.test.ts

@@ -1,143 +1,104 @@
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { URL } from "node:url";
+
 import { describe, expect, it } from "vitest";
 
 import { createBraintrustCliRuntime } from "../src/braintrust-cli";
+import { DEFAULT_API_URL, DEFAULT_APP_URL } from "../src/options";
 
 describe("Braintrust CLI runtime", () => {
-  it("builds the Unix installer command", async () => {
-    const calls: Array<{
-      readonly command: string;
-      readonly args: readonly string[];
-      readonly env?: NodeJS.ProcessEnv;
-    }> = [];
-    const runtime = createBraintrustCliRuntime({
-      platform: "darwin",
-      env: { PATH: "/usr/bin" },
-      exec: (spec) => {
-        calls.push(spec);
-        return Promise.resolve({
-          exitCode: 0,
-          signal: null,
-          stdout: "",
-          stderr: "",
-        });
-      },
-    });
+  it.runIf(process.env.CI === "true")(
+    "configures and reads context using the real bt CLI",
+    async () => {
+      const serviceToken = process.env.BRAINTRUST_SERVICE_TOKEN;
+      if (!serviceToken) {
+        throw new Error("BRAINTRUST_SERVICE_TOKEN is required in CI.");
+      }
 
-    await runtime.install();
-
-    expect(calls).toEqual([
-      {
-        command: "sh",
-        args: [
-          "-c",
-          "curl -fsSL https://bt.dev/cli/install.sh | bash -s -- --quiet",
-        ],
-        env: { PATH: "/usr/bin" },
-      },
-    ]);
-  });
+      const target = await discoverTestTarget(serviceToken);
+      const home = await mkdtemp(join(tmpdir(), "braintrust-cli-test-"));
 
-  it("builds the update command", async () => {
-    const calls: Array<{
-      readonly command: string;
-      readonly args: readonly string[];
-      readonly env?: NodeJS.ProcessEnv;
-    }> = [];
-    const runtime = createBraintrustCliRuntime({
-      env: { PATH: "/usr/bin" },
-      exec: (spec) => {
-        calls.push(spec);
-        return Promise.resolve({
-          exitCode: 0,
-          signal: null,
-          stdout: "",
-          stderr: "",
+      try {
+        const runtime = createBraintrustCliRuntime({
+          env: {
+            ...process.env,
+            HOME: home,
+            XDG_CONFIG_HOME: join(home, ".config"),
+          },
         });
-      },
-    });
+        const discovery = await runtime.discover();
 
-    await runtime.update("/usr/local/bin/bt");
+        expect(discovery).toMatchObject({ installed: true });
+        expect(discovery.commandPath).toBeDefined();
+        expect(discovery.version).toBeDefined();
 
-    expect(calls).toEqual([
-      {
-        command: "/usr/local/bin/bt",
-        args: ["self", "update"],
-        env: { PATH: "/usr/bin" },
-      },
-    ]);
-  });
-
-  it("passes the API key only through env when configuring auth and context", async () => {
-    const calls: Array<{
-      readonly command: string;
-      readonly args: readonly string[];
-      readonly env?: NodeJS.ProcessEnv;
-    }> = [];
-    const runtime = createBraintrustCliRuntime({
-      env: { PATH: "/usr/bin" },
-      exec: (spec) => {
-        calls.push(spec);
-        return Promise.resolve({
-          exitCode: 0,
-          signal: null,
-          stdout: "",
-          stderr: "",
+        await runtime.loginAndSwitch(discovery.commandPath!, {
+          apiKey: serviceToken,
+          apiUrl: target.apiUrl,
+          appUrl: DEFAULT_APP_URL,
+          orgName: target.orgName,
+          projectName: target.projectName,
         });
-      },
-    });
 
-    await runtime.loginAndSwitch("/usr/local/bin/bt", {
-      apiKey: "bt-secret-key",
-      apiUrl: "https://api.test",
-      appUrl: "https://app.test",
-      orgName: "acme",
-      projectName: "demo",
-    });
+        await expect(
+          runtime.status(discovery.commandPath!),
+        ).resolves.toMatchObject({
+          profile: target.orgName,
+          org: target.orgName,
+          project: target.projectName,
+        });
+      } finally {
+        await rm(home, { recursive: true, force: true });
+      }
+    },
+    30_000,
+  );
+});
 
-    expect(calls).toHaveLength(2);
-    expect(calls[0]?.args).toEqual([
-      "login",
-      "--profile=acme",
-      "--no-input",
-      "--quiet",
-    ]);
-    expect(calls[1]?.args).toEqual([
-      "switch",
-      "--profile=acme",
-      "--org=acme",
-      "--no-input",
-      "--quiet",
-      "--global",
-      "demo",
-    ]);
-    expect(calls.flatMap((call) => [...call.args])).not.toContain(
-      "bt-secret-key",
-    );
-    expect(calls[0]?.env?.["BRAINTRUST_API_KEY"]).toBe("bt-secret-key");
-    expect(calls[0]?.env?.["BRAINTRUST_API_URL"]).toBe("https://api.test");
-    expect(calls[0]?.env?.["BRAINTRUST_APP_URL"]).toBe("https://app.test");
-    expect(calls[1]?.env?.["BRAINTRUST_API_KEY"]).toBe("bt-secret-key");
+async function discoverTestTarget(serviceToken: string): Promise<{
+  readonly apiUrl: string;
+  readonly orgName: string;
+  readonly projectName: string;
+}> {
+  const loginUrl = new URL("/api/apikey/login", DEFAULT_APP_URL);
+  const loginResponse = await fetch(loginUrl, {
+    method: "POST",
+    headers: { Authorization: `Bearer ${serviceToken}` },
   });
+  if (!loginResponse.ok) {
+    throw new Error(`Braintrust login failed with ${loginResponse.status}.`);
+  }
 
-  it("parses bt status JSON", async () => {
-    const runtime = createBraintrustCliRuntime({
-      exec: () =>
-        Promise.resolve({
-          exitCode: 0,
-          signal: null,
-          stdout: JSON.stringify({
-            profile: "work",
-            org: "acme",
-            project: "demo",
-          }),
-          stderr: "",
-        }),
-    });
+  const login = (await loginResponse.json()) as {
+    readonly org_info?: readonly {
+      readonly name: string;
+      readonly api_url?: string | null;
+    }[];
+  };
+  const org = login.org_info?.[0];
+  if (!org) throw new Error("The CI service token has no Braintrust org.");
 
-    await expect(runtime.status("/bin/bt")).resolves.toEqual({
-      profile: "work",
-      org: "acme",
-      project: "demo",
-    });
+  const apiUrl = org.api_url ?? DEFAULT_API_URL;
+  const projectsUrl = new URL("/v1/project", apiUrl);
+  projectsUrl.searchParams.set("org_name", org.name);
+  const projectsResponse = await fetch(projectsUrl, {
+    headers: { Authorization: `Bearer ${serviceToken}` },
   });
-});
+  if (!projectsResponse.ok) {
+    throw new Error(
+      `Braintrust project lookup failed with ${projectsResponse.status}.`,
+    );
+  }
+
+  const projects = (await projectsResponse.json()) as {
+    readonly objects?: readonly { readonly name: string }[];
+  };
+  const project = projects.objects?.[0];
+  if (!project) {
+    throw new Error("The CI service token's Braintrust org has no project.");
+  }
+
+  return { apiUrl, orgName: org.name, projectName: project.name };
+}