diff --git a/src/lib/tm-base-url.ts b/src/lib/tm-base-url.ts index e92165b..de85f0d 100644 --- a/src/lib/tm-base-url.ts +++ b/src/lib/tm-base-url.ts @@ -4,27 +4,88 @@ import { BrowserStackConfig } from "./types.js"; import { getBrowserStackAuth } from "./get-auth.js"; import appConfig from "../config.js"; -const TM_BASE_URLS = [ +/** + * The production regions, probed in this order. UNCHANGED, and the default in every + * deployment: the override below exists for test harnesses, not for shipping. + */ +export const TM_BASE_URLS = [ "https://test-management.browserstack.com", "https://test-management-eu.browserstack.com", "https://test-management-in.browserstack.com", ] as const; +/** + * A TEST-HARNESS AFFORDANCE. Point region discovery at a non-production environment. + * + * Plural because the probe loop takes a list. It REPLACES the built-in list rather than + * extending it — appending would leave the + * production hosts probed first, which is the whole problem it exists to avoid: preprod-only + * credentials 401 against production, and a 401 can send the model off to retry with a + * different tool, so a tool-selection measurement stops meaning what it says. + * + * An override that parses to nothing falls back to the built-in list rather than leaving an + * empty probe loop, which would surface as "unable to connect" with no detail. That fallback + * is a WARNING, not a silent one: quietly using production when someone asked for preprod is + * exactly the failure this is meant to prevent. + */ +export const TM_BASE_URLS_ENV = "BROWSERSTACK_TM_BASE_URLS"; + +export interface ResolvedBaseUrls { + urls: string[]; + /** Where the list came from, so a run pointed at the wrong environment is visible. */ + source: "built-in" | "env" | "built-in (override unusable)"; +} + +export function resolveTMBaseUrls(): ResolvedBaseUrls { + const raw = process.env[TM_BASE_URLS_ENV]; + if (!raw || !raw.trim()) return { urls: [...TM_BASE_URLS], source: "built-in" }; + + const urls = raw + .split(",") + .map((entry) => entry.trim().replace(/\/+$/, "")) + // Anything without a scheme is a typo, not a host: silently probing it would produce a + // confusing connection error rather than naming the real mistake. + .filter((entry) => /^https?:\/\/\S+$/i.test(entry)); + + if (!urls.length) { + return { urls: [...TM_BASE_URLS], source: "built-in (override unusable)" }; + } + return { urls, source: "env" }; +} + let cachedBaseUrl: string | null = null; +/** + * Which list the cached URL was discovered under. + * + * Keyed rather than skipped, so a value minted against production can never be served to a + * run pointed at preprod (or the reverse) — the override would otherwise appear to work while + * silently returning the previous environment's host. + */ +let cachedFor: string | null = null; export async function getTMBaseURL( config: BrowserStackConfig, ): Promise { + const { urls, source } = resolveTMBaseUrls(); + const listKey = urls.join(","); + // Skip the module-level cache in remote (multi-tenant) mode: it is process-shared, // so the first user's region would be served to every subsequent user — breaking // requests for users on a different region's BrowserStack account. - if (!appConfig.REMOTE_MCP && cachedBaseUrl) { + if (!appConfig.REMOTE_MCP && cachedBaseUrl && cachedFor === listKey) { logger.debug(`Using cached TM base URL: ${cachedBaseUrl}`); return cachedBaseUrl; } + if (source === "built-in (override unusable)") { + logger.warn( + `${TM_BASE_URLS_ENV} was set but no entry looked like an http(s) URL; falling back ` + + `to the built-in production list. Requests will go to production.`, + ); + } logger.info( - "No cached TM base URL found, testing available URLs with authentication", + `No cached TM base URL found, testing available URLs with authentication ` + + `(list from ${source}: ${listKey})`, ); const authString = getBrowserStackAuth(config); @@ -34,7 +95,7 @@ export async function getTMBaseURL( const failures: string[] = []; - for (const baseUrl of TM_BASE_URLS) { + for (const baseUrl of urls) { try { const res = await apiClient.get({ url: `${baseUrl}/api/v2/projects/`, @@ -51,6 +112,7 @@ export async function getTMBaseURL( // the cache must stay empty so each user discovers their own region. if (!appConfig.REMOTE_MCP) { cachedBaseUrl = baseUrl; + cachedFor = listKey; } logger.info(`Selected TM base URL: ${baseUrl}`); return baseUrl; diff --git a/src/server-factory.ts b/src/server-factory.ts index a5a926d..da3f3af 100644 --- a/src/server-factory.ts +++ b/src/server-factory.ts @@ -20,6 +20,7 @@ import addBuildInsightsTools from "./tools/build-insights.js"; import { setupOnInitialized } from "./oninitialized.js"; import { BrowserStackConfig } from "./lib/types.js"; import addRCATools from "./tools/rca-agent.js"; +import addAskBrowserstackAITool from "./tools/ask-browserstack/register.js"; /** * Wrapper class for BrowserStack MCP Server @@ -61,6 +62,10 @@ export class BrowserStackMcpServer { addSelfHealTools, addBuildInsightsTools, addRCATools, + // Hands a plain-language task to BrowserStack's agent and relays its mid-run + // permission asks back to this client, so a write can be confirmed by the human + // sitting in front of it rather than refused for want of anyone to ask. + addAskBrowserstackAITool, ]; toolAdders.forEach((adder) => { diff --git a/src/tools/ask-browserstack/central-oauth.ts b/src/tools/ask-browserstack/central-oauth.ts new file mode 100644 index 0000000..f7aea87 --- /dev/null +++ b/src/tools/ask-browserstack/central-oauth.ts @@ -0,0 +1,337 @@ +/** + * Mint a BrowserStack central-OAuth JWT from the caller's username and access key. + * + * This replaces a shared delegation token, and the upgrade is not cosmetic. + * `validate_delegation_token` refuses any token without `user.user_id`/`user.group_id`, so + * what we mint here is USER-ATTESTED: Atlas sets `principal_verified=True`, takes the acting + * user from signed claims rather than from anything we put in the request body, and reuses + * this same JWT as its `egress_token` — so the product call a human approves runs as that + * human, not as a shared service account. + * + * SECRET HYGIENE IS THE WHOLE POINT OF THIS MODULE, and Atlas's `central_oauth.py` learned + * it the hard way: "The body can echo the credential back on some errors, so it is NOT + * logged or raised — only the status." Neither the access key nor the minted token is ever + * logged, returned, or put in an error message. Only a status code is. + */ + +import { createHash } from "node:crypto"; + +import logger from "../../logger.js"; +import { AGENT_TIMEOUT_MS, AskError } from "./config.js"; +import { Credentials } from "./egress.js"; + +/** + * BOTH PARTS ARE REQUIRED, AND THERE IS NO FALLBACK TO ANOTHER SCOPE. + * + * `oauth_user_profile` stays because it is what makes the pair obtainable through the + * username+access_key flow at all. `ai_agent_notify` is what Atlas matches on + * (`delegation.required_scope`, checked as exact membership of the token's `scopes` claim in + * `web/oauth.py`); both halves move together with Atlas. + * + * THIS SCOPE MAY SIMPLY NOT BE ISSUABLE TO US, and the reasons are worth stating rather than + * discovering. From the merged `browserstack/railsApp#175367` (2026-08-24): + * + * - `ai_agent_notify` is documented there as CLIENT_ID/SECRET auth, and + * `USERNAME_ACCESS_KEY_ONLY_SCOPES` remains only `user_management, oauth_user_profile`. + * We are on the username+access_key flow, which those restrictions are not written for. + * - It is additionally covered by a new + * `APP_REGISTERED_SCOPE_REQUIRED = %w[ai_agent ai_agent_notify]` gate, requiring the + * calling APPLICATION to be registered for it — though that gate sits in the + * `client_id + client_secret` path, not ours. + * - railsApp defines it as the PRODUCT -> AGENT direction: "a product reporting progress + * back to an AI agent for work the agent dispatched." We use it in the opposite + * direction, as an agent -> Atlas inbound credential. + * - `central_ai_s2s`, which this replaces, was deliberately EXCLUDED from that new gate. + * + * So this is strictly more restricted than what it replaces. If the endpoint refuses it, that + * is a PROVISIONING problem — the scope is not available to this credential type or this + * application — and it is reported as one, naming the scope. It is never retried with a + * different scope: a silent downgrade to a different authorization is exactly the kind of + * thing nobody notices until it matters. + */ +export const CENTRAL_SCOPE = "oauth_user_profile ai_agent_notify"; + +/** What we ask for. The endpoint clamps to its own maximum, so the response wins. */ +export const REQUESTED_EXPIRES_IN = 3600; + +/** + * Treat a token as stale this long before it actually expires. + * + * NOT the usual small skew. This token is not merely used to open the request — Atlas holds + * it for the life of the run and re-uses it for product egress, so it has to outlive the + * whole call, and our own `/agent` budget is already 330s. Handing out a token with 61 + * seconds left would mean a human approves a write and the egress that follows fails on an + * expired credential, which is the exact mid-flight expiry this cache exists to prevent. + */ +export const REFRESH_SKEW_MS = AGENT_TIMEOUT_MS + 60_000; + +/** The token endpoint gets its own, much shorter budget than `/agent`. */ +export const TOKEN_TIMEOUT_MS = 15_000; + +export interface TokenResponse { + status: number; + body: unknown; + /** Only when there was no response at all to speak for itself. */ + error?: string; +} + +export type TokenTransport = ( + url: string, + form: Record, +) => Promise; + +/** + * The OAuth2 error codes we are willing to read out of a failure body. + * + * `error` is a fixed enum token in the spec, so it cannot carry a credential; `error_description` + * is free text and demonstrably CAN ("access_key is invalid"), which is why only the + * code is ever looked at and only when it is one of these. Anything else is ignored entirely + * and the classification falls back to the status. + */ +const SCOPE_ERROR_CODES = [ + "invalid_scope", + "unauthorized_client", + "invalid_request", +]; +const CREDENTIAL_ERROR_CODES = [ + "invalid_client", + "invalid_grant", + "access_denied", +]; + +/** + * Was this refusal about the SCOPE or about the CREDENTIAL? + * + * The two need completely different fixes — provisioning versus a password — so collapsing + * them into one message sends someone to the wrong place entirely. Our form has five fields + * and four of them are constants, so a refusal of the REQUEST (as opposed to the caller) can + * only really be about the scope. + * + * Nothing from the body is ever surfaced; the code is used to classify and then discarded. + */ +export function refusalIsAboutScope(status: number, body: unknown): boolean { + const payload = + typeof body === "object" && body !== null + ? (body as Record) + : {}; + const code = typeof payload.error === "string" ? payload.error : ""; + if (SCOPE_ERROR_CODES.includes(code)) return true; + if (CREDENTIAL_ERROR_CODES.includes(code)) return false; + // No usable code. OAuth2 answers a bad REQUEST with 400 and a bad CLIENT with 401/403, so + // the status is the next best evidence. + return status === 400; +} + +/** + * The ways authentication can fail, kept apart because a user cannot act on them otherwise. + * + * `scope refused` is a provisioning problem; `rejected` is "your credentials are wrong"; + * `unreachable` is "auth is down". A fourth — Atlas refusing a token we minted successfully — + * is a server misconfiguration and lives in `relay.ts`, because it is discovered from + * `/agent`. Four different fixes, so four different sentences. + */ +export const AUTH_SCOPE_REFUSED_DETAIL = (status: number): string => + `BrowserStack auth would not issue a token for the scope "${CENTRAL_SCOPE}" (HTTP ${status}). ` + + `YOUR CREDENTIALS ARE NOT THE PROBLEM — this is a provisioning problem: \`ai_agent_notify\` ` + + `is documented as a client_id/secret scope, it is not in the username+access_key allow ` + + `list, and it carries an application-registration requirement. It has to be enabled for ` + + `this account or application; a different password will not help, and this server will ` + + `NOT quietly retry with a weaker scope. NOTHING REACHED THE AGENT — no request was made, ` + + `no prompt appeared and nothing was changed.`; + +export const AUTH_REJECTED_DETAIL = (status: number): string => + `Your BrowserStack credentials were rejected by BrowserStack auth (HTTP ${status}). ` + + `NOTHING REACHED THE AGENT — no request was made, no prompt appeared and nothing was ` + + `changed. Check BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY.`; + +export const AUTH_UNREACHABLE_DETAIL = + "Could not reach BrowserStack auth to sign in. NOTHING REACHED THE AGENT — no request " + + "was made, no prompt appeared and nothing was changed. This is a connectivity or " + + "auth-server problem, not a problem with your credentials."; + +/** + * A 5xx from auth: their service is down, not your password. + * + * Split out because routing 5xx to `AUTH_REJECTED_DETAIL` actively misdirects the reader, + * and did: a preprod outage returned 503 and the tool answered "Your BrowserStack + * credentials were rejected … Check BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY", + * sending someone to audit env vars that had worked minutes earlier. The status alone + * settles it — OAuth2 says a bad client is 401/403 and a bad request is 400, so nothing in + * the 5xx range is ever a statement about the caller. + */ +export const AUTH_SERVER_ERROR_DETAIL = (status: number): string => + `BrowserStack auth is unavailable (HTTP ${status}). YOUR CREDENTIALS ARE NOT THE ` + + `PROBLEM — a 5xx is the auth service failing, not a rejection, so there is nothing to ` + + `change on your side and nothing to retry differently. NOTHING REACHED THE AGENT — no ` + + `request was made, no prompt appeared and nothing was changed. Try again once ` + + `BrowserStack auth is back.`; + +export const AUTH_UNUSABLE_DETAIL = (status: number): string => + `BrowserStack auth answered HTTP ${status} without issuing a token. NOTHING REACHED THE ` + + `AGENT — no request was made, no prompt appeared and nothing was changed.`; + +interface CacheEntry { + token: string; + expiresAt: number; + /** Shared so N concurrent tool calls mint ONCE rather than N times. */ + inflight?: Promise; +} + +const cache = new Map(); + +/** Drop every cached token. For tests, and for a credential rotation. */ +export function resetTokenCache(): void { + cache.clear(); +} + +/** + * The cache key. + * + * Keyed on the access key so that ROTATING it mints immediately rather than leaving a + * revoked credential working until expiry — but on a SHA-256 of it, never the value, so the + * secret is not left sitting in a map key for the life of the process. + */ +function cacheKey(url: string, credentials: Credentials): string { + const digest = createHash("sha256") + .update(credentials.accessKey) + .digest("hex"); + return `${url} ${credentials.username} ${CENTRAL_SCOPE} ${digest}`; +} + +/** A fetch-based transport for the token endpoint. */ +export function fetchTokenTransport( + timeoutMs = TOKEN_TIMEOUT_MS, +): TokenTransport { + return async (url, form) => { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + const response = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/x-www-form-urlencoded", + Accept: "application/json", + }, + body: new URLSearchParams(form).toString(), + redirect: "manual", + signal: controller.signal, + }); + let parsed: unknown = null; + try { + parsed = await response.json(); + } catch { + // An HTML error page behind any status. The caller only reads the status. + parsed = null; + } + return { status: response.status, body: parsed }; + } catch { + // DNS, TLS, timeout — all of them mean "no token". The reason is deliberately not + // carried: it can name the URL and, on some stacks, echo the request body. + return { status: 0, body: null, error: "auth could not be reached" }; + } finally { + clearTimeout(timer); + } + }; +} + +/** The exact form body of the `client_credentials` grant. */ +export function mintForm(credentials: Credentials): Record { + return { + grant_type: "client_credentials", + username: credentials.username, + access_key: credentials.accessKey, + scope: CENTRAL_SCOPE, + expires_in: String(REQUESTED_EXPIRES_IN), + }; +} + +async function mintOnce( + url: string, + credentials: Credentials, + transport: TokenTransport, +): Promise<{ token: string; lifetimeMs: number }> { + const response = await transport(url, mintForm(credentials)); + + if (response.status === 0) throw new AskError(AUTH_UNREACHABLE_DETAIL); + // 5xx BEFORE the refusal branch: a server error is not a refusal, and reading it as one + // is worse than saying nothing — it names the caller's credentials as the fault. + if (response.status >= 500) { + throw new AskError(AUTH_SERVER_ERROR_DETAIL(response.status)); + } + if (response.status !== 200) { + // ONLY THE STATUS CROSSES. The body is read solely to tell a provisioning problem from a + // credential one, and nothing out of it is ever put in the message — a non-200 body can + // echo the access key straight back. + throw new AskError( + refusalIsAboutScope(response.status, response.body) + ? AUTH_SCOPE_REFUSED_DETAIL(response.status) + : AUTH_REJECTED_DETAIL(response.status), + ); + } + + const body = + typeof response.body === "object" && response.body !== null + ? (response.body as Record) + : {}; + const token = body.access_token; + if (typeof token !== "string" || !token) { + throw new AskError(AUTH_UNUSABLE_DETAIL(response.status)); + } + + // Trust the SERVER's lifetime over what we asked for — it clamps to its own maximum, and + // caching for the requested hour when it granted less would hand out a dead token. + const granted = Number(body.expires_in); + const seconds = + Number.isFinite(granted) && granted > 0 ? granted : REQUESTED_EXPIRES_IN; + return { token, lifetimeMs: seconds * 1000 }; +} + +/** + * Return a valid token, minting one only when the cache has nothing fresh. + * + * Minting per tool call would add a round trip to every request and make the token endpoint + * a hot dependency of the whole surface. + */ +export async function mintCentralToken( + url: string, + credentials: Credentials, + transport: TokenTransport, + now: number = Date.now(), +): Promise { + // Refused before any network call, and by name: these ARE the auth credential now, not + // merely attribution, so an empty one is our missing configuration rather than the user's + // rejected password, and must not read like one. + if (!credentials?.username || !credentials?.accessKey) { + throw new AskError( + "BrowserStack AI is not authenticated: BROWSERSTACK_USERNAME and " + + "BROWSERSTACK_ACCESS_KEY are required to sign in", + ); + } + + const key = cacheKey(url, credentials); + const entry = cache.get(key); + if (entry && entry.token && now < entry.expiresAt - REFRESH_SKEW_MS) { + return entry.token; + } + // Double-checked through a shared promise: concurrent callers await the same mint. + if (entry?.inflight) return entry.inflight; + + const pending = mintOnce(url, credentials, transport) + .then(({ token, lifetimeMs }) => { + cache.set(key, { token, expiresAt: now + lifetimeMs }); + logger.info( + "askBrowserstackAI: signed in as %s (lifetime %ss)", + credentials.username, + Math.round(lifetimeMs / 1000), + ); + return token; + }) + .catch((error) => { + // Never leave a rejected promise cached, or every later call inherits this failure. + cache.delete(key); + throw error; + }); + + cache.set(key, { token: "", expiresAt: 0, inflight: pending }); + return pending; +} diff --git a/src/tools/ask-browserstack/config.ts b/src/tools/ask-browserstack/config.ts new file mode 100644 index 0000000..a787353 --- /dev/null +++ b/src/tools/ask-browserstack/config.ts @@ -0,0 +1,153 @@ +import logger from "../../logger.js"; + +/** + * Where Atlas lives, and the timeout ladder. + * + * The host IS compiled in, matching every other tool here — `TM_BASE_URLS`, the + * instrumentation endpoint — so an install needs no configuration to work. One env var + * overrides it. See the warning on `DEFAULT_ATLAS_URL`: the compiled-in value is currently + * STAGING and is a deliberate placeholder. + */ + +/** + * CONTRACT §4 — the timeout ladder, outermost first. EACH LAYER MUST EXCEED THE ONE INSIDE + * IT, or a layer dies before the layer it is waiting on can answer: + * + * MCP client -> tool call longest, client-side, not ours + * POST /agent HTTP request 330s <- here + * Atlas gate -> stream ask 300s Atlas's `permission_relay_timeout` + * elicitInput 270s <- here + * + * 300s is the browser path's existing PERMISSION_TIMEOUT, which also auto-rejects. + */ +export const AGENT_TIMEOUT_MS = 330_000; +export const ELICITATION_TIMEOUT_MS = 270_000; + +/** Thrown for anything this tool refuses to attempt. Never carries a credential. */ +export class AskError extends Error {} + +/** Off by default is wrong for a shipped feature, but a kill switch is not. */ +export function isEnabled(): boolean { + return (process.env.ASK_BROWSERSTACK_DISABLED || "").toLowerCase() !== "true"; +} + +/** + * May the relay be offered in the hosted (`REMOTE_MCP`) deployment? + * + * OFF BY DEFAULT, because it depends on something outside this package: the host has to + * keep one `McpServer` alive per session. Stateless hosts build a fresh server per POST, + * and an elicitation answer — which arrives as a SEPARATE POST — then reaches an instance + * that never asked anything, leaving the real one suspended until it times out. So this + * must stay opt-in per deployment rather than become a default that silently hangs. + * + * Turning it on does NOT force the relay on: `relayMode` still asks whether THIS client + * declared the `elicitation` capability, and a client that did not still gets a read-only + * run. This flag only removes the blanket refusal. + */ +export function allowRemoteRelay(): boolean { + return ( + (process.env.ASK_BROWSERSTACK_ALLOW_REMOTE_RELAY || "").toLowerCase() === "true" + ); +} + +/** + * ============================================================================ + * PRODUCTION DEFAULTS + * ============================================================================ + * + * These hosts are PRODUCTION. They replace the interim staging placeholders that this + * package shipped with while the relay was being built ("for now lets hardcode the + * base_url to staging only then we will point this to prod url later") — that step is + * now done. + * + * `https://workflows.browserstack.com` was verified, not guessed: its `/api/profiles` + * answers `401 {"detail":"authentication required"}`, byte-identical to staging Atlas. + * The production auth endpoint is `https://auth.browserstack.com/oauth2/v2/token`. + * + * WHY THIS MATTERS: this package publishes to npm as `@browserstack/mcp-server`, so an + * install with no environment variables set now talks to PRODUCTION. That is correct for + * a production deployment, but it removes the old safety property — a misconfigured or + * test deployment that forgets `ASK_BROWSERSTACK_ATLAS_URL` no longer fails safe onto + * staging, it reads and writes REAL customer data. Non-production deployments MUST set + * that variable explicitly. The resolved host is logged at info on first use, naming + * whether it came from the env var or from here, so a deployment pointing at the wrong + * Atlas is visible in a log line rather than inferred later from confusing data. + * + * Staging hosts, for anyone setting the override: + * ASK_BROWSERSTACK_ATLAS_URL = https://ai-platform-service.bsstag.com + * ASK_BROWSERSTACK_AUTH_TOKEN_URL = https://auth-preprod.bsstag.com/oauth2/v2/token + * + * The tests assert these literals precisely so that repointing has to be deliberate + * rather than something that slips through. + * + * grep: DEFAULT-PROD-HOSTS + */ +export const DEFAULT_ATLAS_URL = "https://workflows.browserstack.com"; +export const DEFAULT_AUTH_TOKEN_URL = + "https://auth.browserstack.com/oauth2/v2/token"; + +/** An operator's override may carry a trailing slash; the constants above do not. */ +function trimUrl(value: string): string { + return value.trim().replace(/\/+$/, ""); +} + +/** + * Announced ONCE per distinct resolution, not per tool call. + * + * The point is that a deployment talking to the wrong Atlas shows up in the log; repeating it + * on every call would only make it easier to scroll past. + */ +const announced = new Set(); + +/** For tests, and for anything that legitimately re-resolves. */ +export function resetHostAnnouncements(): void { + announced.clear(); +} + +function announce(what: string, url: string, source: "env" | "default"): void { + const line = `${what}|${url}|${source}`; + if (announced.has(line)) return; + announced.add(line); + logger.info("askBrowserstackAI: %s is %s (source: %s)", what, url, source); +} + +/** + * Resolve Atlas's base URL: + * + * 1. ASK_BROWSERSTACK_ATLAS_URL explicit override + * 2. the built-in staging default (see the warning above) + * + * Matching every other tool here, which ships its host in the code and treats the env var as + * an override — `TM_BASE_URLS`, the instrumentation endpoint. There is no environment map and + * no selector: one default, one override. + */ +export function atlasBaseUrl(): string { + const explicit = process.env.ASK_BROWSERSTACK_ATLAS_URL; + const url = + explicit && explicit.trim() ? trimUrl(explicit) : DEFAULT_ATLAS_URL; + announce("Atlas", url, explicit && explicit.trim() ? "env" : "default"); + return url; +} + +/** Resolved per call, never captured at construction. */ +export function agentUrl(): string { + return `${atlasBaseUrl()}/agent`; +} + +/** + * Where a central-OAuth JWT is minted (CONTRACT v1.2 §I, as amended by task 7). + * + * The shared `delegation.token` path is gone from Atlas, so a user-attested central JWT is + * the only way in. Same two rungs as the host, and the same staging default. + */ +export function authTokenUrl(): string { + const explicit = process.env.ASK_BROWSERSTACK_AUTH_TOKEN_URL; + const url = + explicit && explicit.trim() ? trimUrl(explicit) : DEFAULT_AUTH_TOKEN_URL; + announce( + "auth token endpoint", + url, + explicit && explicit.trim() ? "env" : "default", + ); + return url; +} diff --git a/src/tools/ask-browserstack/egress.ts b/src/tools/ask-browserstack/egress.ts new file mode 100644 index 0000000..1e89128 --- /dev/null +++ b/src/tools/ask-browserstack/egress.ts @@ -0,0 +1,44 @@ +/** + * The pieces of the outbound `POST /agent` that are not the transport itself. + * + * The transport moved to `stream.ts` when A2 was removed: `/agent` is read as an event + * stream now, so a one-request-one-response `AgentTransport` has nothing left to describe. + * What stays here is what both halves always shared — the header set, the credential pair, + * and the response shape `relay.ts` reads to tell a refusal from an unreachable service + * apart, which the stream's JSON-degrade path still produces. + * + * AUTH HERE IS NOT THE PRODUCT-API AUTH. `/agent` accepts exactly two credentials, both in + * `Authorization`: the shared delegation token or a BrowserStack central JWT. There is no + * `Api-Token` path on this route (CONTRACT v1.2 §I), so sending one would not merely be + * useless — it would push the user's `access_key` across a trust boundary to an endpoint + * that has no use for it, and into every request log on the way. The capability registry's + * `authHeaders` remains right for PRODUCT calls; it is simply not the header set for this + * one, and is deliberately not imported here. + */ + +export interface Credentials { + username: string; + accessKey: string; +} + +/** + * The complete header set for `POST /agent` (CONTRACT v1.2 §4). Three headers, no more. + * + * The token is a secret and appears nowhere else: not in a log line, not in a result, not in + * an error message. + */ +export function agentHeaders(token: string): Record { + return { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + // Attribution, so the downstream service can see the call came from an agent. + "request-source": "ai-chatbot", + }; +} + +export interface AgentResponse { + status: number; + body: unknown; + /** Only when there was no response at all to speak for itself. */ + error?: string; +} diff --git a/src/tools/ask-browserstack/register.ts b/src/tools/ask-browserstack/register.ts new file mode 100644 index 0000000..210089b --- /dev/null +++ b/src/tools/ask-browserstack/register.ts @@ -0,0 +1,555 @@ +/** + * `askBrowserstackAI` — one tool call in, one tool result out, with a human's approval + * relayed through the middle of it. + * + * The shape, and why: + * + * 1. NEGOTIATE FIRST. `relayMode()` is consulted BEFORE Atlas is called, so Atlas learns + * whether a human is reachable before it starts rather than discovering it at the gate. + * Anything other than "offered" means `permission_relay` is omitted entirely and Atlas + * runs read-only — today's exact behaviour, and the path opencode and goose stay on. + * That also covers the hosted `REMOTE_MCP` deployment, where the relay cannot work at + * all; see `relayMode` for why. Nothing here depends on `sampling`, which Claude Code + * does not declare. + * 2. LISTEN ON LOOPBACK. Transport is A2, so Atlas calls US back; because it initiates, + * the decision returns on the same connection to the same pod and PLAN.md's affinity + * problem never arises for this stdio deployment. + * 3. ELICIT, ONCE. Atlas's `description` is the message, nothing is requested in the form, + * and the ACTION is mapped by CONTRACT §7 with no second chances. + * 4. RETURN THE TRAIL. `approvals` and `applied_before_stop` are what let a caller tell + * "nothing happened" from "some steps applied, then stopped". + * + * FAIL CLOSED THROUGHOUT. A decline, a cancel, a timeout, a bad token, a body we cannot + * parse, a handler that throws — every one of them denies. An unattended run cannot approve + * itself because a headless client returns `cancel`, which is a deny. + */ + +import { + McpServer, + RegisteredTool, +} from "@modelcontextprotocol/sdk/server/mcp.js"; +import { + CallToolResult, + ElicitResult, + ErrorCode, + McpError, + RequestId, +} from "@modelcontextprotocol/sdk/types.js"; +import { z } from "zod"; + +import appConfig from "../../config.js"; +import { trackMCP } from "../../lib/instrumentation.js"; +import { BrowserStackConfig } from "../../lib/types.js"; +import logger from "../../logger.js"; +import { + AskError, + ELICITATION_TIMEOUT_MS, + agentUrl, + allowRemoteRelay, + authTokenUrl, + isEnabled, +} from "./config.js"; +import { fetchTokenTransport, mintCentralToken } from "./central-oauth.js"; +import { Credentials, agentHeaders } from "./egress.js"; +import { + EVENT_PERMISSION, + EVENT_RESULT, + EVENT_RUN, + decisionUrl, + fetchAgentStreamTransport, + fetchDecisionTransport, + parseAsk, +} from "./stream.js"; +import type { AgentStreamTransport, DecisionTransport } from "./stream.js"; +import { + buildResult, + decide, + elicitationMessage, + elicitationShape, + errorResult, +} from "./relay.js"; +import { + AgentRequest, + ApprovalRecord, + AskResult, + PermissionAsk, + PermissionDecision, + PRODUCTS, + RelayMode, +} from "./types.js"; + +export interface AskDeps { + /** Resolved per call: a deployment's host is configuration, not a constructor argument. */ + agentUrl: () => string; + /** + * Sign in and return a bearer for `POST /agent`. Cached behind this, not minted per call. + * + * A function rather than a value because it is resolved per call for the same reason the + * host is, and because it can fail in ways a caller needs told apart. + */ + mintToken: () => Promise; + /** + * Read per call, not captured: the remote server rebuilds config per session, so a + * captured credential would outlive the session it belongs to. + * + * These are now THE AUTH CREDENTIAL, not merely attribution: the access key is exchanged + * for a user-attested central JWT, which is what lets Atlas run the approved product call + * as the human rather than as a shared service account. + */ + credentialsFor: () => Credentials; + /** + * The two transport seams, injectable so a test can drive a whole approval round trip + * — ask, elicit, decide, result — without a socket. + */ + streamTransport?: AgentStreamTransport; + decisionTransport?: DecisionTransport; +} + +/** + * THE FALLBACK POSITIONING IN THE FIRST SENTENCE IS LOAD-BEARING. + * + * This server registers 45 tools, most of them hand-written for one endpoint each. Those are + * faster, cheaper and more predictable than handing a task to an agent that has to work out + * its own API calls, so they should win whenever one of them actually fits. What this tool + * covers is the gap: a task nothing here has a tool for, or one where the specific tools were + * tried and did not get there. + * + * A description is the ONLY thing steering that choice — the client picks a tool from these + * words alone, before any call is made — so the ordering is deliberate: when to reach for it + * first, what it does second, and the consent behaviour last. + */ +const DESCRIPTION = + "Use this when no other BrowserStack tool here fits the task, or when the ones you tried " + + "did not get you there. Prefer a specific tool whenever one fits: it is faster and more " + + "predictable than handing the job to an agent. " + + "Otherwise, describe what you want in plain language and BrowserStack's agent decides " + + "which calls to make, then returns its answer plus the steps it took. Anything that would " + + "change data pauses and asks you to confirm it first, in your own client; deletes are " + + "refused outright. If your client cannot show you a prompt, the run is read-only and " + + "everything it wanted to change comes back in `needs_approval` instead. One task per call."; + +/** + * `isError` marks a call that FAILED, not one that was refused. + * + * A `blocked` run is the feature working: the agent asked, a human said no, and the trail + * says so. Flagging that as a tool error makes a client render a correct refusal in red and + * — worse — invites it to retry, which is exactly the "retry forever" loop the distinct + * `permission_relay` reasons exist to prevent. `ok` still means `status === "ok"`. + */ +function toResult(payload: AskResult): CallToolResult { + const failed = + payload.status === "error" || payload.status === "rate_limited"; + return { + content: [{ type: "text", text: JSON.stringify(payload) }], + ...(failed ? { isError: true } : {}), + }; +} + +function isTimeout(error: unknown): boolean { + return error instanceof McpError && error.code === ErrorCode.RequestTimeout; +} + +/** + * Relay one ask to the human and record what they said. + * + * The elicitation is NEVER retried. A client that timed out or errored has told us it + * cannot get an answer, and asking again only produces a second prompt for the same action. + */ +async function relayOneAsk( + server: McpServer, + ask: PermissionAsk, + approvals: ApprovalRecord[], + relatedRequestId?: RequestId, +): Promise { + let answer: ElicitResult; + try { + // `relatedRequestId` IS LOAD-BEARING OVER HTTP, and its absence fails silently. + // Streamable HTTP routes a server->client message onto the stream of the request it + // relates to (`_requestToStreamMapping`). With no id the SDK falls back to the + // standalone SSE stream, and a host that answers GET /mcp with 405 has none — so the + // SDK drops the message with "Stream is disconnected", the tool waits out its 270s, + // and Atlas's gate expires into `reason: "timeout"`. The human is told they did not + // answer a question they were never shown. + // + // Measured exactly that way against the hosted server before this was threaded + // through. On stdio it is irrelevant — one pipe, nothing to route — which is why no + // local test could have caught it. + answer = await server.server.elicitInput( + { + mode: "form", + // Framed with the PRODUCT and nothing else (v1.1 §G): a bare sentence with no + // attribution is a worse prompt than a framed one, and `product` is all the + // ask carries — the route, method, path and op_key never reach this side by design, + // and that is the whole privacy boundary (the ask is four named fields). The + // description goes through VERBATIM: paraphrasing it would mean the human approves + // something other than what the model actually said. Atlas no longer rewrites it + // either — it used to replace a route-shaped one with a placeholder, which asked a + // person to approve a sentence they could not read; CONTRACT v2 §3 was amended and + // that guard removed. An older Atlas can still send the placeholder, which is why + // the framing is tested against it. + message: elicitationMessage(ask.product, ask.description), + requestedSchema: { + // NOTHING IS REQUESTED. The action IS the answer: `accept` already means the human + // approved, and `decline` already gives them an unambiguous refusal in the same + // dialog. A `confirm` boolean used to live here and produced a FALSE DENIAL — a + // user approved on preprod and was told they had refused, because a client renders + // a form field and submits its unset value. We cannot distinguish that from a + // deliberate untick, so the field is gone rather than guessed at. + // + // Fail-closed is untouched by this and never rested on the boolean: a headless + // client with nobody at the terminal returns `cancel` (measured, HANDOFF.md), and + // `cancel` is a deny. That is what stops an unattended run self-approving. + type: "object", + properties: {}, + }, + }, + // The inner rung of CONTRACT §4's ladder, strictly shorter than Atlas's 300s gate. + { timeout: ELICITATION_TIMEOUT_MS, relatedRequestId }, + ); + } catch (error) { + if (isTimeout(error)) { + approvals.push({ + description: ask.description, + decision: "deny", + reason: "timeout", + }); + return { perm_id: ask.perm_id, decision: "deny", reason: "timeout" }; + } + // An unexpected failure has no honest `reason` in CONTRACT §2's vocabulary, so it is + // not given one on the wire: the throw says "this side broke" without claiming a human + // decided anything. `runStreamed` catches it and sends the explicit deny the throw + // implies — see the comment there for why A1 cannot let it escape. + approvals.push({ + description: ask.description, + decision: "deny", + reason: "error", + }); + throw error; + } + + // The SHAPE of the answer only — a fixed action enum and a boolean, never the description + // or anything a user typed. Logged so that what a client actually submits can be read next + // time rather than inferred from a compiled binary. + logger.info( + "askBrowserstackAI: elicitation answered %s", + elicitationShape(answer), + ); + + const { decision, reason } = decide(answer); + approvals.push({ description: ask.description, decision, reason }); + return { perm_id: ask.perm_id, decision, reason }; +} + +/** + * Decide whether to offer the approval channel at all. + * + * STDIO ALWAYS. HOSTED ONLY WHEN ITS OPERATOR OPTS IN — and the reason is a property of + * the HOST, not of this tool. + * + * Elicitation is a SERVER-INITIATED message whose answer arrives on a SEPARATE POST. A + * stateless host builds a fresh `McpServer` per POST, so that answer reaches an instance + * which never asked anything, while the one actually suspended on `await` waits out its + * timeout. Nothing in this package can fix that; what it holds is a live Promise resolver + * and a paused function in the host's heap, and a paused call cannot be moved. + * + * This is why `841c6358` was right to remove sessions from the hosted server on the + * grounds that "we use neither server-initiated messages nor subscriptions/sampling" — + * this feature is the exception that commit did not have to consider. + * + * MEASURED, not assumed: with the host keeping one server per session + * (browserstack/remote-mcp-server#96), a tool call and its elicitation answer were served + * by the same instance over hosted Streamable HTTP, and the relay completed. So the + * refusal below is now conditional rather than absolute. + * + * It stays OFF by default because it depends on a deployment property this package cannot + * observe. A hosted operator turns it on only once their host keeps sessions AND pins a + * session to a pod — sessions are per-process, so without affinity the answer POST can + * land on a replica that has never seen it. That failure is intermittent and reads like a + * client bug, which is exactly why it must not be the default. + * + * When refused, Atlas runs read-only — a supported path that already works — and + * `permission_relay.reason` says `remote_mode` so nobody mistakes it for a human's no. + */ +/** + * CONTRACT v2 (A1) — drive one run over the stream. + * + * The loop is the whole orchestration: read events, elicit on each `permission`, POST + * the decision, hand the `result` to `buildResult`. It decides nothing itself — + * `relayOneAsk` owns the elicitation and the allow/deny mapping, unchanged from the + * transport it replaced. That is deliberate: the transport changed, the judgement did + * not, and the judgement is the part that is dangerous to get wrong. + * + * Reading pauses while a human is being prompted, which is correct rather than merely + * tolerable: Atlas is blocked on that decision and will emit nothing but heartbeats + * until it arrives, and heartbeats are dropped by the parser. + * + * A run that ends with no `result` is an error, not an empty success. A stream that + * simply stops is indistinguishable from a network drop, and reporting it as a finished + * run with no answer would be the transport quietly speaking for the agent. + */ +async function runStreamed( + server: McpServer, + streamTransport: AgentStreamTransport, + decisionTransport: DecisionTransport, + url: string, + headers: Record, + body: AgentRequest, + approvals: ApprovalRecord[], + mode: RelayMode, + product: string, + relatedRequestId?: RequestId, +): Promise { + let runId = ""; + let result: unknown; + let sawResult = false; + // 200 unless the reply was not a stream at all, in which case the transport carries + // the real status — `relay.ts` needs it to tell 401 from 403 from a plain failure. + let resultStatus = 200; + + for await (const event of streamTransport(url, headers, body)) { + if (event.event === EVENT_RUN) { + runId = String((event.data as { run_id?: string })?.run_id || ""); + continue; + } + if (event.event === EVENT_RESULT) { + result = event.data; + sawResult = true; + if (typeof event.status === "number") resultStatus = event.status; + continue; + } + if (event.event !== EVENT_PERMISSION) continue; + + // Validated, not cast: a frame missing a usable `perm_id` or carrying a blank + // description cannot produce an answerable prompt, so it must not produce a prompt. + const ask = parseAsk(event.data); + if (!ask) { + logger.error( + "askBrowserstackAI: unusable permission ask on the stream; ignoring", + ); + continue; + } + if (!runId) { + // Atlas emits `run` before any ask precisely so this cannot happen. If it does, + // there is nowhere to send a decision — so do not prompt a human for an answer + // that could never be delivered. + logger.error( + "askBrowserstackAI: permission ask arrived before run_id; cannot answer", + ); + continue; + } + + // `relayOneAsk` RETHROWS on an unexpected elicitation failure. Under A2 that was + // load-bearing: the throw made the inbound callback answer 500, which Atlas's + // fail-closed rule read as a deny. Under A1 there is no inbound request to fail, so + // letting it escape would abandon the run and leave Atlas waiting out its full 300s + // gate — turning a client hiccup into a five-minute stall. So it is caught here and + // converted into the explicit deny the throw used to imply. `relayOneAsk` has + // already recorded the approvals entry, so only the wire decision is missing. + let decision: PermissionDecision; + try { + decision = await relayOneAsk(server, ask, approvals, relatedRequestId); + } catch (error) { + logger.warn( + "askBrowserstackAI: elicitation failed, denying explicitly: %s", + error instanceof Error ? error.message : String(error), + ); + decision = { perm_id: ask.perm_id, decision: "deny", reason: "error" }; + } + const status = await decisionTransport(decisionUrl(url, runId), headers, { + perm_id: decision.perm_id, + decision: decision.decision, + reason: decision.reason || "", + }); + if (status !== 204) { + // Never fatal, and never re-sent. Atlas's gate is still waiting and denies on its + // own expiry, so a lost decision is safe — it can only cost an approval, never + // grant one. Retrying risks the opposite: a duplicate that 409s, or worse, an + // approval applied to a step the run has already moved past. + logger.warn( + "askBrowserstackAI: decision for %s was not accepted (HTTP %s)", + decision.perm_id, + status, + ); + } + } + + if (!sawResult) { + return errorResult( + "BrowserStack AI ended the run without a result. Nothing was changed " + + "beyond any step you already approved.", + approvals, + ); + } + // Shaped as an `AgentResponse` so `buildResult` — written for the transport A1 replaced + // and unchanged — sees exactly what it always saw. + return buildResult( + { status: resultStatus, body: result }, + approvals, + mode, + product, + ); +} + +export function relayMode(server: McpServer): RelayMode { + // The hosted deployment refuses UNLESS its operator has opted in, because whether an + // elicitation can be answered there depends on the host keeping one server alive per + // session — see `allowRemoteRelay`. Verified working against the hosted Streamable + // HTTP server once it does (browserstack/remote-mcp-server#96). + if (appConfig.REMOTE_MCP && !allowRemoteRelay()) return "remote_mode"; + // The real gate either way: can THIS client be asked? A client that never declared + // `elicitation` gets a read-only run whatever the deployment. + return server.server.getClientCapabilities()?.elicitation + ? "offered" + : "no_human"; +} + +export function addAskBrowserstackAITool( + server: McpServer, + deps: AskDeps, + config?: BrowserStackConfig, +): Record { + // A1 (CONTRACT v2) is the only path; A2 is gone. No version flag is needed to talk to + // an Atlas that predates the stream: such a server answers `POST /agent` with ordinary + // JSON, the parser sees no `text/event-stream`, and the run degrades to a read-only + // answer carrying that response's own status. + const streamTransport = deps.streamTransport || fetchAgentStreamTransport(); + const decisionTransport = deps.decisionTransport || fetchDecisionTransport(); + const tools: Record = {}; + + /** Instrumentation in the house style, and never fatal to the call it wraps. */ + const track = (name: string) => { + try { + trackMCP(name, server.server.getClientVersion()!, undefined, config); + } catch { + // Telemetry must not decide whether a tool call succeeds. + } + }; + + tools.askBrowserstackAI = server.tool( + "askBrowserstackAI", + DESCRIPTION, + { + product: z + .enum(PRODUCTS) + .describe( + "Which product to work in: tm (Test Management), a11y (Accessibility), " + + "tra (Test Reporting & Analytics).", + ), + query: z + .string() + .describe("What you want, in plain language. One thing per call."), + }, + { + // It can write now, which is the whole point of the relay. Destructive operations + // stay refused, so `destructiveHint` is false for the same reason invokeEndpoint + // sets it false: consent is not a licence to delete. + readOnlyHint: false, + destructiveHint: false, + title: "Ask BrowserStack AI", + }, + async ({ product, query }, extra): Promise => { + track("askBrowserstackAI"); + const approvals: ApprovalRecord[] = []; + // Negotiated before anything else so the failure paths below report the mode they + // would have run in. + const mode = relayMode(server); + + try { + const url = deps.agentUrl(); + // Signed in BEFORE the listener is opened and before the run starts. Minting + // lazily mid-call would put a token round-trip inside the window where a human is + // being prompted, and a mint that failed there would strand an open port. + const headers = agentHeaders(await deps.mintToken()); + const body: AgentRequest = { task: query, product }; + + // Attribution, and now belt-and-braces rather than the source of truth: the minted + // JWT is user-attested, so Atlas sets `principal_verified=True` and takes the acting + // user from signed claims instead of this field. It is still sent because it is part + // of the frozen wire format (CONTRACT v1.2 §3) and dropping it would be a one-sided + // change — but nothing should trust it, and Atlas no longer does. + // Omitted ENTIRELY when unset, never sent as "". + const username = (deps.credentialsFor().username || "").trim(); + if (username) body.user_id = username; + + // A1: asking for a stream costs nothing to set up — no port, no listener, no + // per-run bearer, because nothing dials in. Which is the whole point: the + // callback this replaces could never reach a laptop behind NAT, so the feature + // was read-only for every real user regardless of what was configured. + if (mode === "offered") { + body.permission_relay = { mode: "stream" }; + } else { + // Omitted ENTIRELY, not sent empty: its absence is what selects Atlas's + // read-only HeadlessGate. + logger.info( + "askBrowserstackAI: no permission relay (%s); running read-only", + mode, + ); + } + + // `product` reaches the result so an entitlement refusal can name it: the flags + // are per product, and a bare "not enabled" sends the user to their admin + // asking about the wrong thing. + return toResult( + await runStreamed( + server, + streamTransport, + decisionTransport, + url, + headers, + body, + approvals, + mode, + product, + // The tool call's own id, so each elicitation is routed onto THIS request's + // stream. Over Streamable HTTP there is nowhere else for it to go. + extra?.requestId, + ), + ); + } catch (error) { + const message = + error instanceof AskError || error instanceof Error + ? error.message + : String(error); + logger.error("askBrowserstackAI failed: %s", message); + // No `canElicit` argument: the request never left this process, so whether the + // client could have been prompted is not what the reader needs to know. + return toResult(errorResult(message, approvals)); + } + // No teardown: A1 opens no port and binds nothing, so there is nothing that can + // leak across calls or survive an error. The stream is closed by its own + // iteration ending, and Atlas drops the run when the response completes. + }, + ); + + return tools; +} + +/** The tool-adder the server factory calls. */ +export function addAskBrowserstackAIToolFromConfig( + server: McpServer, + config: BrowserStackConfig, +): Record { + if (!isEnabled()) { + logger.info("askBrowserstackAI disabled by ASK_BROWSERSTACK_DISABLED"); + return {}; + } + const credentials = () => ({ + username: config["browserstack-username"], + accessKey: config["browserstack-access-key"], + }); + const tokenTransport = fetchTokenTransport(); + return addAskBrowserstackAITool( + server, + { + // Both resolved per call. An unconfigured host surfaces as a named error from the + // tool rather than as a missing tool, so the cause is visible to whoever hits it. + agentUrl, + mintToken: () => + mintCentralToken(authTokenUrl(), credentials(), tokenTransport), + credentialsFor: credentials, + }, + config, + ); +} + +export default addAskBrowserstackAIToolFromConfig; diff --git a/src/tools/ask-browserstack/relay.ts b/src/tools/ask-browserstack/relay.ts new file mode 100644 index 0000000..4054d45 --- /dev/null +++ b/src/tools/ask-browserstack/relay.ts @@ -0,0 +1,681 @@ +/** + * The decision mapping and the result assembly — the two places where being wrong is + * expensive, kept pure so they can be tested without a server, a socket or a client. + */ + +import { ElicitResult } from "@modelcontextprotocol/sdk/types.js"; + +import { AgentResponse } from "./egress.js"; +import { + ApprovalRecord, + AskResult, + AskStatus, + ASK_STATUSES, + Decision, + DecisionReason, + RelayMode, +} from "./types.js"; + +export const RELAY_ON_DETAIL = + "This client can prompt you, so BrowserStack asked before each change and the answers " + + "are in `approvals`."; + +/** + * Used when the channel worked and every ask came back with nobody behind it. + * + * `RELAY_ON_DETAIL` cannot be used here and saying it was a bug, observed live: it claims + * "BrowserStack asked before each change and the answers are in `approvals`" while the + * trail says `refused: nobody was there to be asked`. Both sentences were in the same + * result, contradicting each other — the same class of confusion as `disabled` vs a human + * saying no, and it is the reader who pays for it. The channel being usable and a person + * actually answering are different facts, and only the second one licenses the word + * "answers". + */ +export const RELAY_ON_NO_ANSWER_DETAIL = + "BrowserStack asked before each change, but this client answered without a person " + + "present, so every change was refused. Nothing was modified. Approve from a client " + + "that can show you the prompt."; + +/** + * One sentence per way the relay can fail to run, because they call for different things + * from the person reading them, and a caller who cannot tell them apart retries forever. + * + * `no_human` is ours (CONTRACT §7's last row); the rest are Atlas's (v1.1 §D). + */ +export const RELAY_OFF_DETAILS: Record = { + no_human: + "This client does not support MCP elicitation, so there was no way to ask you mid-run. " + + "BrowserStack ran read-only: anything that would have changed data was refused and is " + + "listed in `needs_approval`. Re-run the same request from a client that supports " + + "elicitation to be asked for confirmation instead.", + + // The one that matters most to a human. NOBODY REFUSED ANYTHING HERE — the server has the + // knob off, so saying "your change was declined" would be a lie and retrying cannot help. + disabled: + "NOBODY DECLINED THIS. BrowserStack's permission relay is switched off on the server " + + "(`delegation.permission_relay`), so it ignored the approval channel this client " + + "offered and ran read-only. Everything in `needs_approval` was refused for that " + + "configuration reason alone. Retrying will keep failing the same way until an " + + "administrator turns the relay on.", + + // ONLY AN ATLAS THAT PREDATES A1 CAN SEND THIS. It described the old outbound transport: + // the server was handed a URL to call back and refused the address. Nothing dials out any + // more, so the condition cannot arise — the reason stays mapped because an older + // deployment is still entitled to an explanation rather than a raw enum. + host_not_allowed: + "BrowserStack refused to call this client back, which means it is running a version " + + "that predates the current approval channel: that version could only reach a client on " + + "the same host. The run went read-only. Nothing was declined by a person, and the fix " + + "is a BrowserStack-side upgrade rather than anything about this request.", + + malformed: + "BrowserStack could not use the approval channel this client offered and ran read-only. " + + "That is a bug on this side, not something you did; everything in `needs_approval` was " + + "refused because of it.", + + // Neither the human nor the client is the constraint here — the DEPLOYMENT is, and no + // change either of them can make will help. + remote_mode: + "NOBODY DECLINED THIS, AND YOUR CLIENT IS NOT THE PROBLEM. This BrowserStack MCP server " + + "is running in its hosted, multi-tenant mode, which has no way to put an approval " + + "prompt in front of you, so it ran read-only and everything in `needs_approval` was " + + "refused for that reason alone. Mid-run approval works when the server runs locally over stdio; retrying " + + "against this deployment will keep failing the same way.", + + // Not a refusal by anyone and not a relay problem at all: the account is not on the + // product's agent flag. The product-specific sentence and what to do about it live in + // `error`, so this one points there rather than duplicating the plumbing. + not_entitled: + "NOBODY DECLINED THIS AND NOTHING RAN. BrowserStack AI is not enabled for this account, " + + "so the request was refused before the agent started. `error` says which product and " + + "what to do about it. This is an entitlement on the account, not a problem with your " + + "credentials and not a decision anyone made about your request.", + + // The request never got as far as the agent. Distinct from `disabled` (the agent ran, with + // the relay switched off) and from a decline (someone was asked and said no), because the + // three call for completely different things from whoever reads them. + not_reached: + "NOTHING WAS ASKED AND NOTHING WAS REFUSED. BrowserStack rejected this request before " + + "the agent started, so no step ran, no prompt appeared, and nothing was changed. " + + "`error` says why. This is not a decision anyone made about your request — fix what " + + "`error` names and run it again.", +}; + +/** Kept for anything still importing the old single constant. */ +export const RELAY_OFF_DETAIL = RELAY_OFF_DETAILS.no_human; + +/** A reason from a newer Atlas than this build. Say so plainly rather than crash. */ +function unknownRelayDetail(used: boolean, reason: string): string { + // Bounded: this string came off the wire and goes into a result a human reads. + const quoted = JSON.stringify(reason.slice(0, 64)); + return used + ? `BrowserStack used the approval channel and reported ${quoted}, which this version ` + + "does not recognise. The answers it did collect are in `approvals`." + : `BrowserStack did not use the approval channel this client offered, reporting ` + + `${quoted}, which this version does not recognise. The run was read-only, so ` + + "anything in `needs_approval` was refused without anyone being asked."; +} + +/** The sentence that goes with a `{used, reason}` pair, whoever produced it. */ +export function relayDetail(used: boolean, reason: string): string { + if (used) return reason ? unknownRelayDetail(true, reason) : RELAY_ON_DETAIL; + return RELAY_OFF_DETAILS[reason] ?? unknownRelayDetail(false, reason); +} + +/** + * Did this request die before the agent ever started? + * + * Atlas omits its `permission_relay` verdict on refusals that never reach the delegation + * layer — 401 unauthorized, 400 bad body, 503 delegation-not-enabled all answer with a bare + * `{"detail": …}` — and a transport failure has no body at all. Read naively, "no verdict" + * looks identical to "an Atlas older than v1.1", and the optimistic fallback for THAT case + * then claims the channel was used and answers were collected when zero prompts appeared. + * + * Which is the same confusion the `disabled` sentence exists to prevent, one layer earlier: + * a caller who cannot tell "nobody was asked" from "somebody said no" retries forever. + */ +export function looksLikeDelegationResult(body: unknown): boolean { + const payload = asRecord(body); + return ( + "ok" in payload || + "status" in payload || + "answer" in payload || + "approvals" in payload + ); +} + +/** + * Is this Atlas saying the ACCOUNT is not enabled for the product's agent? + * + * Atlas gained an entitlement gate on `POST /agent` (`flags.is_agent_enabled`, AIC-386) that + * the WebSocket path already had. The flags are PER PRODUCT — `aiHarnessAgent` for tm, + * `aiHarnessAgentTRA`, `aiHarnessAgentA11y` — so an account can be entitled for one product + * and not another. It is also FAIL-OPEN on Atlas's side: Redis down, a flag never seeded, or + * an unknown product all allow the request. So a 403 here is a real, deliberate "this account + * is not enabled", never an outage. + * + * KEYED STRUCTURALLY, NEVER ON THE PROSE. Matching the sentence would break silently the + * first time someone rewords it, falling through to a generic error. + * + * TODO(atlas/9.md): Atlas is deciding whether to add a `code` field to this body. When it + * lands, prefer it over the status — add the check as the first rung here and leave the + * status as the fallback for an older Atlas. Until then the status IS the structural signal. + */ +export function isNotEntitled(response: AgentResponse): boolean { + return response.status === 403; +} + +/** + * The sentence the user asked for, with the product named. + * + * Entitlement is per product, so a bare "not enabled" sends someone to their admin asking + * about the wrong thing. The disambiguation from a 401 is deliberate and load-bearing: + * without it, a working access key gets rotated in response to a permissions problem. + */ +export const NOT_ENTITLED_DETAIL = (product: string): string => { + const scope = product && product.trim() ? ` for \`${product.trim()}\`` : ""; + return ( + `BrowserStack AI is not enabled${scope} on your account. Please contact your admin. ` + + `YOUR CREDENTIALS ARE FINE — they authenticated successfully; this is a per-product ` + + `entitlement on the account. Nothing was run and nobody declined anything.` + ); +}; + +export function neverReachedAgent(response: AgentResponse): boolean { + // No response at all: nothing could have run. + if (response.status === 0) return true; + // THE BODY DECIDES, NOT THE STATUS. + // + // This rung first read "any non-2xx", and that was wrong in the one direction that + // matters. Atlas answers HTTP 502 with a COMPLETE result body when a delegation ran and a + // step then failed (`delegation/http.py:317-320`), and 429 carries a full body too. So an + // approved write whose egress failed came back saying nobody had been asked, while + // `approvals` in the same payload showed the prompt shown and approved — the exact lie + // this predicate was added to prevent, now told on the one run where it costs the most. + // + // The HTTP code describes the OUTCOME; the body describes whether there was a RUN. Only + // the second question is being asked here. + return !looksLikeDelegationResult(response.body); +} + +/** + * Atlas's own verdict on the relay (v1.1 §D), when it gave one. + * + * Present ONLY when we supplied a `permission_relay` block, so its absence is either "we + * never offered one" or "this Atlas predates v1.1" — neither of which is an error. A block + * we cannot read is treated as no block at all rather than half-trusted. + */ +export function atlasRelayVerdict( + payload: Record, +): { used: boolean; reason: string } | null { + const raw = payload.permission_relay; + if (typeof raw !== "object" || raw === null || Array.isArray(raw)) + return null; + const record = raw as Record; + if (typeof record.used !== "boolean") return null; + return { + used: record.used, + reason: typeof record.reason === "string" ? record.reason : "", + }; +} + +/** + * Product-language framing for the prompt (v1.1 §G, approved). + * + * `product` is the ONLY thing added — it is all §2 carries, and the route, method and path + * never reach this side by design. The description itself is passed through untouched: + * paraphrasing or truncating it would mean the human approves something other than what the + * model actually said. Atlas sends it as the model wrote it (its route guard was removed — + * CONTRACT v2 §3, amended), so this prefix is the only thing in front of the model's own + * sentence. A placeholder can still arrive from an older Atlas and reads fine after it. + */ +export const PRODUCT_LABELS: Record = { + tm: "Test Management", + a11y: "Accessibility", + tra: "Test Reporting & Analytics", +}; + +export function elicitationMessage( + product: string, + description: string, +): string { + const label = PRODUCT_LABELS[product] || product.trim(); + const who = label + ? `BrowserStack AI (${label})` + : // An unnamed product beats an empty pair of brackets. + "BrowserStack AI"; + return `${who} needs your approval to continue:\n\n${description}`; +} + +/** + * CONTRACT §7. THE ACTION IS THE WHOLE ANSWER. + * + * | accept | allow | "" | + * | decline | deny | declined | + * | cancel | deny | cancelled | + * + * Nothing is requested in the form any more, so nothing can contradict the action. There used + * to be a `confirm` boolean, and it had to go: with an `accept` action that ALREADY means the + * human approved, `accept` + `confirm: false` is genuinely ambiguous between "I approved, and + * a checkbox I never saw defaulted to false" and "I unticked it deliberately". The first is a + * FALSE DENIAL — indistinguishable in the result from a human refusing, which is the exact + * confusion D3 and N1 existed to remove — and a user hit it live. We cannot tell the two + * apart, and guessing either way is wrong for the other. `decline` already gives an + * unambiguous refusal in the same dialog, so the boolean bought nothing. + * + * FAIL-CLOSED IS UNCHANGED, and the boolean was never what provided it. A headless client with + * no human at a terminal returns `cancel` — measured, not assumed (HANDOFF.md) — and `cancel` + * is a deny. That is why an unattended run still cannot self-approve. It is also why an + * elicitation is never retried: a second ask cannot conjure a human, only wear one down. + */ +export function decide(result: ElicitResult): { + decision: Decision; + reason: DecisionReason; +} { + if (result.action === "accept") { + // DEFENSIVE ONLY. We no longer request this field, so no client can be expected to send + // it — but one that volunteers an explicit `false` has said something, and honouring it + // costs nothing. Absence, which is the normal case, is consent. + if (result.content?.confirm === false) { + return { decision: "deny", reason: "declined" }; + } + return { decision: "allow", reason: "" }; + } + if (result.action === "decline") + return { decision: "deny", reason: "declined" }; + // `cancel`, and anything a future client sends that we do not recognise: no explicit + // answer was given, which is not an answer we may read as yes. + return { decision: "deny", reason: "cancelled" }; +} + +/** + * A one-line description of the SHAPE of what a client answered with — never its content. + * + * Which client sends what is currently guesswork: the elicitation bug in task 9 had to be + * fixed without being able to confirm what Claude Code actually submits, because its binary + * is compiled and its strings too fragmented to read. This line means the next person can + * look it up instead of inferring it. + * + * `action` and `confirm` are a fixed enum and a boolean; neither can carry a description, a + * credential or anything else a user typed. + */ +export function elicitationShape(result: ElicitResult): string { + const content = result.content; + const confirm = content?.confirm; + const seen = + confirm === undefined + ? "absent" + : typeof confirm === "boolean" + ? String(confirm) + : "non-boolean"; + return `action=${result.action} content=${content ? "present" : "absent"} confirm=${seen}`; +} + +/** + * Read Atlas's `applied_before_stop`. NEVER DERIVE IT. + * + * This side used to compute it as CONTRACT §5's literal "any allow preceded a deny", which + * could only ever be a guess: an approval whose request then failed counted as applied, so + * the field lied in the exact direction it exists to prevent (D2). Atlas now computes it + * from `applied`, which only Atlas can know, and sends it whenever a gate ran — including + * `false`, including with an empty trail. + * + * So a MISSING field is never "false". It is "nobody measured this": either no gate ran, or + * this Atlas predates the field. `null` says that out loud instead of asserting a fact. + */ +export function readAppliedBeforeStop( + payload: Record, +): boolean | null { + const reported = payload.applied_before_stop; + return typeof reported === "boolean" ? reported : null; +} + +/** + * Atlas's approval trail, when it sent one. + * + * Returns `null` — not `[]` — when the key is absent, because an empty trail Atlas DID send + * ("the relay ran and nothing was asked") is a different fact from no trail at all, and only + * the second is a reason to fall back to ours. + * + * Every entry is rebuilt rather than trusted: a `decision` that is not exactly `"allow"` + * becomes `"deny"`, so a garbled trail fails closed in the reporting the same way the wire + * does, and `applied` is carried only when it is genuinely a boolean. + */ +export function parseAtlasApprovals( + payload: Record, +): ApprovalRecord[] | null { + const raw = payload.approvals; + if (!Array.isArray(raw)) return null; + const trail: ApprovalRecord[] = []; + for (const item of raw) { + if (typeof item !== "object" || item === null || Array.isArray(item)) + continue; + const entry = item as Record; + trail.push({ + description: + typeof entry.description === "string" ? entry.description : "", + decision: entry.decision === "allow" ? "allow" : "deny", + reason: typeof entry.reason === "string" ? entry.reason : "", + ...(typeof entry.applied === "boolean" ? { applied: entry.applied } : {}), + }); + } + return trail; +} + +/** + * One phrase per entry, because "approved, then it failed" and "refused" must not read + * alike — conflating them is the whole reason D2 mattered. + * + * An `allow` with no `applied` key is NOT rendered as a failure: nobody measured it, and + * saying otherwise would invent the very fact this is meant to report. + */ +/** + * Every ask was refused because no person was there — not because one said no. + * + * `cancelled` is what a client with nobody at the terminal returns (measured), and + * `no_human` is our own word for the same thing. An EMPTY trail is not this case: nothing + * was asked at all, which the existing sentences already describe correctly. + */ +export function nobodyAnswered(trail: ApprovalRecord[]): boolean { + return ( + trail.length > 0 && + trail.every( + (e) => + e.decision !== "allow" && + (e.reason === "cancelled" || e.reason === "no_human"), + ) + ); +} + +export function approvalOutcome(entry: ApprovalRecord): string { + if (entry.decision === "allow") { + if (entry.applied === true) return "approved, and the change went through"; + if (entry.applied === false) { + return ( + "APPROVED, BUT THE CHANGE DID NOT GO THROUGH — nobody refused it; the request " + + "failed after it was approved" + ); + } + return "approved; whether the change went through was not reported"; + } + switch (entry.reason) { + case "declined": + return "refused: a human said no"; + case "cancelled": + return "refused: nobody was there to be asked"; + case "timeout": + return "refused: nobody answered in time"; + case "error": + return "refused: the approval channel broke before any answer arrived"; + default: + return "refused"; + } +} + +function withOutcomes(trail: ApprovalRecord[]): ApprovalRecord[] { + return trail.map((entry) => ({ ...entry, outcome: approvalOutcome(entry) })); +} + +function asRecord(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Record) + : {}; +} + +/** + * Atlas's own status wins when it declares one; otherwise it is derived from what we can + * see. Deriving is a last resort, not an interpretation of the answer. + */ +export function deriveStatus( + response: AgentResponse, + approvals: ApprovalRecord[], + needsApproval: unknown[], +): AskStatus { + if (looksLikeDelegationResult(response.body)) { + // A run happened, so its own status is the answer — on a 502 and a 429 as much as on a + // 200. Reading the HTTP code first would overwrite what the run said about itself. + const declared = asRecord(response.body).status; + if ( + typeof declared === "string" && + (ASK_STATUSES as readonly string[]).includes(declared) + ) { + return declared as AskStatus; + } + if (response.status === 429) return "rate_limited"; + if (response.status < 200 || response.status >= 300) return "error"; + const denied = approvals.some((entry) => entry.decision === "deny"); + return denied || needsApproval.length > 0 ? "blocked" : "ok"; + } + + // No run to speak of. A 2xx lands here too when the body is not a result — which is what + // made that case report `ok: true` alongside an `error` (N4). + if (response.status === 429) return "rate_limited"; + return "error"; +} + +/** + * Reconcile our view of the relay with Atlas's. + * + * ATLAS WINS WHEN IT SPOKE. It is the side that decided whether `RelayGate` actually ran, so + * its `used`/`reason` beat anything inferred here — that is the whole point of v1.1 §D, and + * `disabled` in particular is a fact only Atlas knows. The `detail` sentence stays ours. + * + * When we never offered a block, ours wins unconditionally: `no_human` describes a client + * that cannot be prompted, which Atlas is never told about and could not report. + */ +function relayVerdict( + payload: Record, + mode: RelayMode, + reachedAgent: boolean, + notEntitled: boolean, +): AskResult["permission_relay"] { + // FIRST, ahead of `not_reached`. Both are true of a 403 — the request certainly did not + // reach the agent — but only one of them tells the reader what to do about it. + if (notEntitled) { + return { + used: false, + reason: "not_entitled", + detail: RELAY_OFF_DETAILS.not_entitled, + }; + } + // FIRST, because it outranks both of the cases below. If the request never reached the + // agent then no channel was exercised whether or not one was offered, and saying the run + // went read-only (`no_human`) would be just as wrong as saying it was used: nothing ran at + // all. The client's inability to be prompted is not the actionable fact here and will + // surface on the next run, once whatever `error` names is fixed. + if (!reachedAgent) { + return { + used: false, + reason: "not_reached", + detail: RELAY_OFF_DETAILS.not_reached, + }; + } + // BEFORE `no_human`, deliberately, when both are true. In the hosted deployment even a + // client that CAN be prompted is of no use, so the deployment is the binding constraint and + // the one the reader can act on; telling them to switch clients would waste their time. + if (mode === "remote_mode") { + return { + used: false, + reason: "remote_mode", + detail: RELAY_OFF_DETAILS.remote_mode, + }; + } + if (mode === "no_human") { + // CONTRACT §7's last row: no elicitation capability means the field was never sent. + return { + used: false, + reason: "no_human", + detail: RELAY_OFF_DETAILS.no_human, + }; + } + const verdict = atlasRelayVerdict(payload); + if (verdict) { + return { ...verdict, detail: relayDetail(verdict.used, verdict.reason) }; + } + // No verdict: an Atlas older than v1.1. We offered the channel and have no reason to + // believe it was refused, so the optimistic read is the honest one — and it is advisory + // either way, never deciding whether an action proceeded. + return { used: true, reason: "", detail: RELAY_ON_DETAIL }; +} + +/** Assemble CONTRACT §5's result. Atlas's payload is carried, never rewritten. */ +export function buildResult( + response: AgentResponse, + approvals: ApprovalRecord[], + mode: RelayMode, + /** Named in the not-entitled sentence, because entitlement is per product. */ + product = "", +): AskResult { + const payload = asRecord(response.body); + // ABSENT WHEN EMPTY, never `[]` (v1.1 §B): Atlas's `public()` omits the key entirely, as + // it does for `narration`, `artifacts`, `error`, `cost_breach` and `usage`. Missing is + // read as empty, not as a malformed response. + const needsApproval = Array.isArray(payload.needs_approval) + ? (payload.needs_approval as unknown[]) + : []; + // ATLAS'S TRAIL WINS WHEN IT SENT ONE. It is the only side that can fill in `applied`, and + // it records what happened to the STEP: an ask answered without a prompt appearing is a + // denial there and nothing at all here. Ours is kept separately rather than discarded, + // because that difference is exactly how an answer this side never prompted for shows up. + const atlasTrail = parseAtlasApprovals(payload); + const trail = atlasTrail ?? approvals; + const status = deriveStatus(response, trail, needsApproval); + const reachedAgent = !neverReachedAgent(response); + const relay = relayVerdict( + payload, + mode, + reachedAgent, + isNotEntitled(response), + ); + // Correct the sentence when the channel was fine but nobody was ever behind it. The + // verdict is computed from the MODE and Atlas's advisory field, neither of which can + // see what the elicitation actually returned — so only here, with the trail in hand, + // is "was a person really asked" knowable. + if (relay.used && relay.reason === "" && nobodyAnswered(trail)) { + relay.detail = RELAY_ON_NO_ANSWER_DETAIL; + } + + return { + ok: status === "ok", + status, + // The product's answer, as the product wrote it. Nothing here summarises or re-reads it. + answer: payload.answer ?? null, + approvals: withOutcomes(trail), + approvals_source: atlasTrail ? "atlas" : "mcp", + elicitations: withOutcomes(approvals), + needs_approval: needsApproval, + applied_before_stop: readAppliedBeforeStop(payload), + permission_relay: relay, + atlas_response: response.body ?? null, + ...atlasError(response, payload, product), + }; +} + +/** + * A rejected credential and a refused action are unrelated problems, and a result that lets + * them read alike sends someone hunting for a human who said no when the real answer is that + * this server never got through the door. + * + * Atlas answers a bad `Authorization` with `401 {"detail": "unauthorized"}` — no `error` + * string of its own — so without this the caller would see a bare "error" and nothing else. + * A denial, by contrast, is `status: "blocked"` with a populated `approvals` trail. + * + * The token itself is NOT named here, only the variable that should hold it. + */ +export const UNAUTHENTICATED_DETAIL = + "Signing in with your BrowserStack credentials SUCCEEDED, but BrowserStack AI refused " + + "the resulting token (HTTP 401). YOUR CREDENTIALS ARE NOT THE PROBLEM — this is a " + + "server-side configuration one, most likely `delegation.required_scope` not matching the " + + "scope the token was minted with. NOBODY DECLINED ANYTHING: the request never reached " + + "the agent, so no permission was sought and nothing was refused."; + +/** + * The `error` field, from whichever side has one. + * + * Ours when there was no response to speak for itself or the credentials were rejected; + * otherwise Atlas's own `error` string, which `public()` includes ONLY when non-empty + * (v1.1 §B) — and failing that, its bare `detail`. + * + * That last rung matters: Atlas's pre-run refusals (400 bad body, 503 delegation not + * enabled) carry `detail` and no `error`, so without it a caller got `status: "error"` and + * nothing whatsoever to act on. + */ +function atlasError( + response: AgentResponse, + payload: Record, + product: string, +): { error?: string } { + if (response.status === 0 && response.error) return { error: response.error }; + + // BEFORE Atlas's own string: for an entitlement refusal ours names the product and says + // what to do, where Atlas's is a bare `detail` a user cannot act on. + if (isNotEntitled(response)) return { error: NOT_ENTITLED_DETAIL(product) }; + + // Atlas's own error string wherever it sent one — on a 502 that carries a full result, + // this is the run explaining its own failure, and nothing here should talk over it. + const reported = payload.error; + if (typeof reported === "string" && reported) return { error: reported }; + + if (response.status === 401) return { error: UNAUTHENTICATED_DETAIL }; + + if (!neverReachedAgent(response)) { + // The delegation ran. `status`, `approvals` and `needs_approval` already say what + // happened; "refused before the agent started" would simply be false. + return {}; + } + + // Bounded: this came off the wire and ends up in front of a person. + const detail = payload.detail; + const quoted = + typeof detail === "string" && detail + ? `: ${JSON.stringify(detail.slice(0, 200))}` + : ""; + + if (response.status < 200 || response.status >= 300) { + return { + error: + `BrowserStack AI refused this request before the agent started ` + + `(HTTP ${response.status})${quoted}.`, + }; + } + // A 2xx carrying something that is not a delegation result at all. Real Atlas does not + // emit this; saying so plainly beats reporting a success with an error attached (N4). + return { + error: + `BrowserStack AI answered HTTP ${response.status} with no delegation ` + + `result${quoted}.`, + }; +} + +/** + * A result for a call that never reached, or never got past, Atlas. + * + * It keeps §5's shape — including the approval trail — because a failure AFTER an approval + * was granted is exactly the case where a caller most needs to know something may already + * have been applied. + */ +export function errorResult( + message: string, + approvals: ApprovalRecord[], +): AskResult { + return { + ok: false, + status: "error", + answer: null, + // Atlas never answered, so there is no authoritative trail to prefer. Ours is all there + // is, and `applied_before_stop` is null because nobody measured anything. + approvals: withOutcomes(approvals), + approvals_source: "mcp", + elicitations: withOutcomes(approvals), + needs_approval: [], + applied_before_stop: null, + // The request never left this process, so it certainly never reached the agent. + permission_relay: { + used: false, + reason: "not_reached", + detail: RELAY_OFF_DETAILS.not_reached, + }, + atlas_response: null, + error: message, + }; +} diff --git a/src/tools/ask-browserstack/stream.ts b/src/tools/ask-browserstack/stream.ts new file mode 100644 index 0000000..307a7c8 --- /dev/null +++ b/src/tools/ask-browserstack/stream.ts @@ -0,0 +1,278 @@ +/** + * CONTRACT v2 (A1) — read the ask off the response, send the decision separately. + * + * WHY THIS REPLACES THE CALLBACK. `callback.ts` binds `127.0.0.1:` and hands + * Atlas the URL. That works only when Atlas is on the same loopback. For a real user it + * cannot work at all: this server runs on their machine, and a laptop behind NAT is not + * addressable from a pod in BrowserStack's cluster. There is no route, and no + * configuration creates one. Every successful relay run to date used a locally-run + * Atlas — the shape the v1 contract was written for, and not the shape a user is in. + * + * A1 inverts it. Both connections are outbound from here: + * + * 1. `POST /agent` — the response is an SSE stream carrying `run`, then any + * `permission` asks, then exactly one `result`. + * 2. `POST /agent/{run_id}/permission` — a fresh short request per decision. + * + * So NAT, firewalls and loopback stop mattering, because nothing ever dials in. + * + * WHAT THIS FILE DELIBERATELY DOES NOT DO: decide anything. Whether a human approved, + * how an elicitation outcome maps to allow/deny, what the result looks like — all of + * that stays in `relay.ts`, untouched, and is shared with the callback transport. This + * is a pipe. Keeping the judgement out of the transport is why swapping A2 for A1 does + * not risk the fail-closed behaviour. + */ + +import logger from "../../logger.js"; +import { AskError } from "./config.js"; +import type { AgentRequest, PermissionAsk } from "./types.js"; + +/** One SSE frame, already parsed. `data` is whatever JSON the frame carried. */ +export interface StreamEvent { + event: string; + data: unknown; + /** + * The HTTP status, carried ONLY on a `result` synthesised from a non-stream reply. + * + * Load-bearing, and it was a bug to omit it. `relay.ts` reads the status to tell a + * rejected credential (401) from an account without the feature (403) from an + * ordinary failure, and those produce three different sentences for the user. A + * result event that dropped the status made every one of them read as a generic + * error. On a real SSE stream the status is 200 by definition, so this is absent. + */ + status?: number; +} + +/** + * CONTRACT v2 §4 — the whole-run guard, so a runaway run cannot hold a tool call open + * forever. Deliberately generous: it is a backstop against a hung server, not a budget + * for a human's attention, and the thing that actually bounds one approval is Atlas's + * 300s gate. + */ +export const WHOLE_RUN_TIMEOUT_MS = 1_800_000; + +export const EVENT_RUN = "run"; +export const EVENT_PERMISSION = "permission"; +export const EVENT_RESULT = "result"; + +/** Atlas's `f"perm-{uuid.uuid4().hex}"`, and nothing else. */ +export const PERM_ID_PATTERN = /^perm-[0-9a-f]{32}$/; + +/** + * Read an ask out of a `permission` frame's data, or return null. + * + * Came over from the transport this one replaced, and the reasons it existed did not + * change with the transport — only the direction the ask arrives from did. It is not a + * trust check on Atlas: it is what keeps a malformed frame from turning into a prompt + * that cannot be honoured. + * + * A blank description is rejected rather than relayed: the description IS the whole of + * what the human is shown, so an empty one is a prompt asking a person to approve + * nothing. A `perm_id` off Atlas's shape is rejected because it is the only thing that + * routes the answer back — the decision endpoint matches on it, so an id we could not + * have received is an answer that can never be delivered. + * + * Only the four fields of CONTRACT §2 are carried forward. `op_key`, `method`, `path` + * and `host` are Atlas-private (v1.1 §A) and it does not send them; if a future one + * ever did, they would stop here rather than reach an elicitation prompt or a result. + */ +export function parseAsk(data: unknown): PermissionAsk | null { + if (typeof data !== "object" || data === null || Array.isArray(data)) + return null; + const record = data as Record; + const permId = record.perm_id; + const description = record.description; + if (typeof permId !== "string" || !PERM_ID_PATTERN.test(permId)) return null; + if (typeof description !== "string" || !description.trim()) return null; + return { + perm_id: permId, + product: typeof record.product === "string" ? record.product : "", + mode: typeof record.mode === "string" ? record.mode : "", + description, + }; +} + +/** + * The transport seam: the shape `AgentTransport` had before A2 was removed, except that it + * yields many events instead of returning one body. Injectable for the same reason that one + * was: the tests must be able to drive a whole approval round trip without a socket. + */ +export type AgentStreamTransport = ( + url: string, + headers: Record, + body: AgentRequest, +) => AsyncIterable; + +/** Posts one decision. Separate seam because it is a separate connection. */ +export type DecisionTransport = ( + url: string, + headers: Record, + body: { perm_id: string; decision: string; reason: string }, +) => Promise; + +/** + * Split a buffer into complete SSE frames, returning the leftover. + * + * Exported for its own tests because chunk boundaries are where SSE parsers break: a + * frame can arrive split across two reads, two frames can arrive in one read, and a + * `data:` line can contain anything except a newline. Getting this wrong shows up as an + * ask that is silently dropped — a write that never gets approved and never explains + * why — so it is tested directly rather than only through the happy path. + */ +export function splitFrames(buffer: string): { + frames: string[]; + rest: string; +} { + const frames: string[] = []; + let rest = buffer; + for (;;) { + const idx = rest.indexOf("\n\n"); + if (idx === -1) break; + frames.push(rest.slice(0, idx)); + rest = rest.slice(idx + 2); + } + return { frames, rest }; +} + +/** + * Parse one frame. Returns null for anything that is not an event we can use — + * including the heartbeat, which is a bare `:` comment and is SUPPOSED to be ignored + * here: its only job is to be a read on the socket so the ingress does not time the + * connection out while a human is thinking. + */ +export function parseFrame(frame: string): StreamEvent | null { + let event = ""; + const dataLines: string[] = []; + for (const line of frame.split("\n")) { + if (line.startsWith(":")) continue; // comment / heartbeat + if (line.startsWith("event:")) event = line.slice(6).trim(); + else if (line.startsWith("data:")) dataLines.push(line.slice(5).trim()); + } + if (!event) return null; + if (dataLines.length === 0) return { event, data: null }; + try { + return { event, data: JSON.parse(dataLines.join("\n")) }; + } catch { + // A frame we cannot read is not a frame we may guess at. Dropping it is safe + // because the only consequence is that an ask goes unanswered and the gate denies + // on its own expiry — never that something is approved. + logger.warn("askBrowserstackAI: unparseable stream frame, ignoring"); + return null; + } +} + +/** + * A fetch-based streaming transport. + * + * `timeoutMs` bounds the WHOLE run, not one request — CONTRACT v2 §4 replaced the old + * 330s outer rung because under A1 the stream lives for the run and may contain several + * 300s approvals in series. The per-ask rung (270s elicitation inside Atlas's 300s gate) + * is unchanged and still enforced where it belongs. + */ +export function fetchAgentStreamTransport( + timeoutMs = WHOLE_RUN_TIMEOUT_MS, +): AgentStreamTransport { + return function stream(url, headers, body) { + return { + async *[Symbol.asyncIterator]() { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + let response: Response; + try { + response = await fetch(url, { + method: "POST", + headers: { ...headers, Accept: "text/event-stream" }, + body: JSON.stringify(body), + // A redirect from an authenticated API is usually a login bounce, and + // following it turns a clear 401 into a 200 carrying an HTML page. + redirect: "manual", + signal: controller.signal, + }); + } catch { + // The same sentence the request/response transport gives, and for the same + // reason: the upstream detail ("connection reset", "fetch failed") names our + // plumbing rather than anything the reader can act on, and letting it through + // once already produced a result that read like the user had done something + // wrong. What they need to know is that BrowserStack was not reachable. + throw new AskError("BrowserStack AI could not be reached"); + } + + const contentType = response.headers.get("content-type") || ""; + + // GRACEFUL DEGRADE, and the reason A1 is safe to ship before Atlas has it + // everywhere: an Atlas that does not know `mode: "stream"` answers with an + // ordinary JSON body (a read-only run, `permission_relay.reason: "disabled"`). + // Yielding it as a single `result` means the caller needs no version + // negotiation and no flag — it gets a correct read-only answer instead of a + // parse failure against a body that was never SSE. + if (contentType.includes("json")) { + const parsed = await response.json().catch(() => null); + yield { + event: EVENT_RESULT, + data: parsed, + status: response.status, + }; + return; + } + + if (!response.ok || !response.body) { + // Neither a stream nor a JSON result. Surface it as a first-class failure + // rather than an empty iteration, which the caller could not tell apart + // from "the run finished and said nothing". + throw new AskError( + `BrowserStack AI refused the stream (HTTP ${response.status}).`, + ); + } + + const decoder = new TextDecoder(); + let buffer = ""; + for await (const chunk of response.body) { + buffer += decoder.decode(chunk as Uint8Array, { stream: true }); + const { frames, rest } = splitFrames(buffer); + buffer = rest; + for (const frame of frames) { + const parsed = parseFrame(frame); + if (parsed) yield parsed; + } + } + // A trailing frame with no terminating blank line still counts. + const tail = parseFrame(buffer); + if (tail) yield tail; + } finally { + clearTimeout(timer); + } + }, + }; + }; +} + +/** The decision POST. 30s, because it is an ordinary short request. */ +export function fetchDecisionTransport(timeoutMs = 30_000): DecisionTransport { + return async (url, headers, body) => { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + const response = await fetch(url, { + method: "POST", + headers, + body: JSON.stringify(body), + redirect: "manual", + signal: controller.signal, + }); + return response.status; + } catch { + // The gate on the far side is still waiting and will deny on its own expiry, so + // a lost decision is safe — it is never an approval. 0 says "never delivered" so + // the caller can say that rather than implying a human refused. + return 0; + } finally { + clearTimeout(timer); + } + }; +} + +/** `POST /agent/{run_id}/permission`, built from the base URL the tool already resolved. */ +export function decisionUrl(agentUrl: string, runId: string): string { + return `${agentUrl.replace(/\/+$/, "")}/${encodeURIComponent(runId)}/permission`; +} diff --git a/src/tools/ask-browserstack/types.ts b/src/tools/ask-browserstack/types.ts new file mode 100644 index 0000000..df99af7 --- /dev/null +++ b/src/tools/ask-browserstack/types.ts @@ -0,0 +1,217 @@ +/** + * CONTRACT v1, expressed as types. + * + * Everything in this file is one half of a wire format whose other half is being written + * against `~/.claude/orchestration/shared/CONTRACT.md` by a different session, in a + * different repo, at the same time. A field renamed here to read better is a field the + * other half will never send. Nothing here changes without changing that document first. + */ + +export const PRODUCTS = ["tm", "a11y", "tra"] as const; +export type Product = (typeof PRODUCTS)[number]; + +/** CONTRACT §2 — what Atlas emits on the run's stream when its gate needs a human. */ +export interface PermissionAsk { + /** + * Atlas's own `perm-<32 hex>` uuid4, carried, never re-minted. + * + * Commit 737f6f57 replaced a per-Bridge sequential scheme with this one after ids + * collided across pods and left a turn blocked until its 300s timeout. A second id + * scheme on this side would reintroduce that bug on a new axis, so we echo theirs. + */ + perm_id: string; + product: string; + /** "ask-always" | "ask-once" — Atlas's vocabulary, not re-interpreted here. */ + mode: string; + /** + * The model's `thought`: product language, route-free. + * + * The privacy boundary is that `op_key`, `method`, `path` and `host` stay inside Atlas's + * private record. This field is the whole of what a human is shown. + */ + description: string; +} + +export type Decision = "allow" | "deny"; + +/** + * CONTRACT §2/§7. Advisory ONLY — `decision` alone decides whether the action proceeds. + * It exists so a result can distinguish "a human said no" from "no human was there". + */ +export type DecisionReason = + | "" + | "declined" + | "cancelled" + | "no_human" + | "timeout" + // v1.1 §C enumerated "error" — the relay broke before or during the ask, so no human ever + // answered. This side never puts it ON THE WIRE: an unexpected relay failure answers HTTP + // 500, which Atlas's fail-closed rule already reads as a deny and records as `error_relay`. + // It appears only in `ApprovalRecord.reason`, where the caller can see what happened. + | "error"; + +/** CONTRACT §2 — the body we POST to `/agent/{run_id}/permission` to answer one ask. */ +export interface PermissionDecision { + perm_id: string; + decision: Decision; + reason: DecisionReason; +} + +/** CONTRACT §1 — the one new optional field on `POST /agent`. */ +/** + * The `permission_relay` block — one shape, because there is one transport. + * + * `{ mode: "stream" }` is A1 / CONTRACT v2: nothing to address and nothing to + * authenticate inbound, because nothing dials in. A2's `{ callback_url, token }` was + * removed on both halves (v2 §7.4) — it could only ever reach a co-located caller, and + * an Atlas that still sees that shape now reports the relay as `disabled` rather than + * trying it. + * + * `mode` stays optional rather than required so this stays a superset of the block an + * older Atlas will simply ignore: the field's ABSENCE is what selects the read-only + * gate, and that has to remain expressible. + */ +export interface PermissionRelay { + mode?: string; +} + +/** + * CONTRACT §1 — the `POST /agent` body. + * + * `permission_relay` is OMITTED ENTIRELY, not sent empty or null, when the client cannot + * elicit: its absence is what selects Atlas's read-only `HeadlessGate`, which is today's + * byte-identical behaviour and the opencode/goose path. + */ +export interface AgentRequest { + task: string; + product: string; + /** + * Who the run is for (CONTRACT v1.2 §3). + * + * The shared delegation token authenticates the caller but not the principal + * (`principal_verified=false`), so Atlas reads the acting user from here. Omitted entirely + * when no username is configured — never sent as `""`. + * + * Note the asymmetry and do not try to close it: on this path a caller can CLAIM any + * `user_id`. That is Atlas's documented design for the shared-token route; a signed + * principal requires the central-JWT path, which is out of scope. + */ + user_id?: string; + permission_relay?: PermissionRelay; +} + +/** CONTRACT §5 — one entry in an approval trail, in the order the asks arrived. */ +export interface ApprovalRecord { + description: string; + decision: Decision; + reason: string; + /** + * Did this approved step's request actually land? (Atlas task 3 §1.) + * + * `true` only when the entry was `allow` AND its request then returned 2xx. Everything + * else — a dead port, a 4xx, a 5xx, an async-dispatch refusal — resolves to `false`: + * unknown fails toward not-applied, never the other way. + * + * ONLY ATLAS CAN KNOW THIS. The gate returns before any request is sent, which was the + * whole of D2; the fact is written by the tool layer into the same record object the gate + * keeps by reference, so it correlates by object identity rather than by position. + * + * ABSENT means not reported (an Atlas that predates this, or our own elicitation trail, + * which has no way to know). Never render an absent value as a measured `false`. + */ + applied?: boolean; + /** + * A human-readable phrase for this entry. + * + * Exists because `allow` + `applied: false` — approved, then the request failed — is a + * genuinely different thing to tell a person than a refusal, and the two must not read + * alike. Derived, never sent by anyone. + */ + outcome?: string; +} + +/** + * Whether the approval channel was offered to Atlas at all, and if not, why not. + * + * Three different facts about this deployment and this client, each needing a different + * thing from whoever reads the result — so they are three values rather than one boolean. + */ +export type RelayMode = + /** `permission_relay` was sent and the run's stream carried the asks. */ + | "offered" + /** The client declares no `elicitation` capability, so nobody could be prompted. */ + | "no_human" + /** This process is the hosted multi-tenant server, which cannot prompt a human. */ + | "remote_mode"; + +export const ASK_STATUSES = ["ok", "blocked", "error", "rate_limited"] as const; +export type AskStatus = (typeof ASK_STATUSES)[number]; + +/** CONTRACT §5 — the single tool result. */ +export interface AskResult { + ok: boolean; + status: AskStatus; + answer: unknown; + /** + * THE AUTHORITATIVE approval trail: Atlas's whenever it supplied one, ours otherwise. + * + * Atlas's wins because it is the only side that can populate `applied`, and because it + * records what happened to the STEP — an ask answered without a prompt appearing (a + * 401'd probe, a shape rejection) is a denial there and nothing at all here. + */ + approvals: ApprovalRecord[]; + /** Which side produced `approvals`, so a reader never has to infer it. */ + approvals_source: "atlas" | "mcp"; + /** + * OUR trail: one entry per prompt this server actually put in front of a human. + * + * Kept beside `approvals` rather than folded into it, because where the two disagree the + * disagreement is the signal. An entry Atlas records as a denial with nothing here means a + * ask was answered without any prompt appearing — which is what an attacker probing + * the loopback port looks like. + */ + elicitations: ApprovalRecord[]; + needs_approval: unknown[]; + /** + * Atlas's own verdict: this run stopped on a refusal AND something had already changed. + * + * READ, NEVER DERIVED. Atlas computes it because only Atlas knows `applied`, and it sends + * the field whenever a relay gate ran — including `false`, including with an empty trail. + * + * `null` means NOT REPORTED, which is not the same as `false`: either no gate ran (so + * Atlas has nothing to say about applications) or this Atlas predates the field. Treating + * it as `false` would assert something nobody measured. + */ + applied_before_stop: boolean | null; + /** + * Why a write may have been refused. + * + * `reason` is ATLAS'S OWN when it reported one (CONTRACT v1.1 §D: `"" | disabled | + * host_not_allowed | malformed`), and ours — `no_human` (§7's last row) — when the client + * could not be prompted at all, which Atlas never learns about because we omit the block. + * Typed as a plain string rather than a union because an Atlas newer than this build may + * name a reason this one has never heard of, and an unrecognised reason must degrade to a + * sentence, not throw. + */ + permission_relay: { + used: boolean; + reason: string; + detail: string; + }; + /** + * Atlas's public payload, verbatim. + * + * The mapped fields above are the contract; this is the belt to their braces. The two + * halves are being built in parallel, so a field named slightly differently on the other + * side would otherwise silently become `null` here rather than reaching the caller. + */ + atlas_response: unknown; + /** + * Why the call failed, when it did. + * + * Ours when egress never completed; otherwise Atlas's own `error` string, which its + * `public()` includes ONLY when non-empty (v1.1 §B). Lifted out of `atlas_response` so a + * caller reading the top level is told why rather than having to go looking. + */ + error?: string; +} diff --git a/tests/lib/tm-base-url.test.ts b/tests/lib/tm-base-url.test.ts index eb8c1a2..4061368 100644 --- a/tests/lib/tm-base-url.test.ts +++ b/tests/lib/tm-base-url.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, vi, beforeEach } from "vitest"; +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; vi.mock("../../src/lib/apiClient", () => ({ apiClient: { @@ -7,7 +7,7 @@ vi.mock("../../src/lib/apiClient", () => ({ })); vi.mock("../../src/logger", () => ({ - default: { error: vi.fn(), info: vi.fn(), debug: vi.fn() }, + default: { error: vi.fn(), info: vi.fn(), debug: vi.fn(), warn: vi.fn() }, })); // Re-imported per-test by resetting modules so the module-level @@ -21,9 +21,21 @@ async function loadModule(remoteMcp: boolean) { })); const apiClientMod = await import("../../src/lib/apiClient"); const tmMod = await import("../../src/lib/tm-base-url"); - return { apiClient: apiClientMod.apiClient, getTMBaseURL: tmMod.getTMBaseURL }; + return { + apiClient: apiClientMod.apiClient, + getTMBaseURL: tmMod.getTMBaseURL, + resolveTMBaseUrls: tmMod.resolveTMBaseUrls, + TM_BASE_URLS: tmMod.TM_BASE_URLS, + }; } +const BUILT_IN = [ + "https://test-management.browserstack.com", + "https://test-management-eu.browserstack.com", + "https://test-management-in.browserstack.com", +]; +const PREPROD = "https://test-management-preprod.bsstag.com"; + const mockConfig = { "browserstack-username": "u", "browserstack-access-key": "k", @@ -107,3 +119,122 @@ describe("getTMBaseURL — failure details", () => { expect(err.message).toMatch(/HTTP 503/); }); }); + +describe("BROWSERSTACK_TM_BASE_URLS — the test-harness override", () => { + const saved = { ...process.env }; + + beforeEach(() => { + vi.clearAllMocks(); + delete process.env.BROWSERSTACK_TM_BASE_URLS; + }); + + afterEach(() => { + process.env = { ...saved }; + }); + + /** The URLs actually probed, in the order they were probed. */ + function probed(apiClient: any): string[] { + return (apiClient.get as any).mock.calls.map((args: any[]) => args[0].url); + } + + it("the built-in list is exactly the three production regions, in order", async () => { + const { TM_BASE_URLS } = await loadModule(false); + // If this ever changes, it is a production change and not a harness one. + expect([...(TM_BASE_URLS as readonly string[])]).toEqual(BUILT_IN); + }); + + it("unset: probes exactly the built-in list, in order", async () => { + const { apiClient, getTMBaseURL, resolveTMBaseUrls } = await loadModule(false); + expect(resolveTMBaseUrls()).toEqual({ urls: BUILT_IN, source: "built-in" }); + + (apiClient.get as any).mockResolvedValue({ ok: false, status: 401 }); + await getTMBaseURL(mockConfig).catch(() => undefined); + expect(probed(apiClient)).toEqual(BUILT_IN.map((u) => `${u}/api/v2/projects/`)); + }); + + it("one URL: REPLACES the list, so production is never probed first", async () => { + // Appending would leave the prod hosts ahead of it and the 401s would come straight + // back, which is the entire problem this exists to remove. + process.env.BROWSERSTACK_TM_BASE_URLS = PREPROD; + const { apiClient, getTMBaseURL, resolveTMBaseUrls } = await loadModule(false); + expect(resolveTMBaseUrls()).toEqual({ urls: [PREPROD], source: "env" }); + + (apiClient.get as any).mockResolvedValueOnce({ ok: true }); + expect(await getTMBaseURL(mockConfig)).toBe(PREPROD); + expect(probed(apiClient)).toEqual([`${PREPROD}/api/v2/projects/`]); + }); + + it("two comma-separated URLs: both, in the given order", async () => { + process.env.BROWSERSTACK_TM_BASE_URLS = `${PREPROD},https://tm-two.bsstag.com`; + const { apiClient, getTMBaseURL } = await loadModule(false); + (apiClient.get as any) + .mockResolvedValueOnce({ ok: false, status: 401 }) + .mockResolvedValueOnce({ ok: true }); + + expect(await getTMBaseURL(mockConfig)).toBe("https://tm-two.bsstag.com"); + expect(probed(apiClient)).toEqual([ + `${PREPROD}/api/v2/projects/`, + "https://tm-two.bsstag.com/api/v2/projects/", + ]); + }); + + it("trims whitespace, drops empty entries and a trailing slash", async () => { + process.env.BROWSERSTACK_TM_BASE_URLS = ` ${PREPROD}/ , , https://tm-two.bsstag.com ,`; + const { resolveTMBaseUrls } = await loadModule(false); + expect(resolveTMBaseUrls()).toEqual({ + urls: [PREPROD, "https://tm-two.bsstag.com"], + source: "env", + }); + }); + + it("drops an entry with no scheme, keeping the rest", async () => { + process.env.BROWSERSTACK_TM_BASE_URLS = `tm-preprod.bsstag.com,${PREPROD}`; + const { resolveTMBaseUrls } = await loadModule(false); + expect(resolveTMBaseUrls()).toEqual({ urls: [PREPROD], source: "env" }); + }); + + it("all garbage: falls back to the built-in list rather than probing nothing", async () => { + // An empty probe loop would surface as "unable to connect" with no detail. + process.env.BROWSERSTACK_TM_BASE_URLS = "not-a-url, ,???"; + const { apiClient, getTMBaseURL, resolveTMBaseUrls } = await loadModule(false); + expect(resolveTMBaseUrls()).toEqual({ + urls: BUILT_IN, + source: "built-in (override unusable)", + }); + + (apiClient.get as any).mockResolvedValue({ ok: false, status: 401 }); + await getTMBaseURL(mockConfig).catch(() => undefined); + expect(probed(apiClient)).toEqual(BUILT_IN.map((u) => `${u}/api/v2/projects/`)); + }); + + it("warns when it falls back, because silently using production is the danger", async () => { + process.env.BROWSERSTACK_TM_BASE_URLS = "not-a-url"; + const { apiClient, getTMBaseURL } = await loadModule(false); + const logger = (await import("../../src/logger")).default; + (apiClient.get as any).mockResolvedValue({ ok: false, status: 401 }); + await getTMBaseURL(mockConfig).catch(() => undefined); + + const warning = (logger.warn as any).mock.calls.map(String).join(" "); + expect(warning).toMatch(/BROWSERSTACK_TM_BASE_URLS/); + expect(warning).toMatch(/go to production/); + }); + + it("does not serve a cached URL minted under a DIFFERENT list", async () => { + // The cache is module-level; without keying it, an override would appear to work while + // quietly returning the previous environment's host. + const { apiClient, getTMBaseURL } = await loadModule(false); + (apiClient.get as any).mockResolvedValueOnce({ ok: true }); + expect(await getTMBaseURL(mockConfig)).toBe(BUILT_IN[0]); + expect(apiClient.get).toHaveBeenCalledTimes(1); + + // Same process, override now set: the cached production host must NOT come back. + process.env.BROWSERSTACK_TM_BASE_URLS = PREPROD; + (apiClient.get as any).mockResolvedValueOnce({ ok: true }); + expect(await getTMBaseURL(mockConfig)).toBe(PREPROD); + expect(apiClient.get).toHaveBeenCalledTimes(2); + + // ...and it still caches within one list. + expect(await getTMBaseURL(mockConfig)).toBe(PREPROD); + expect(apiClient.get).toHaveBeenCalledTimes(2); + }); +}); diff --git a/tests/tools/askBrowserstack.test.ts b/tests/tools/askBrowserstack.test.ts new file mode 100644 index 0000000..746c33b --- /dev/null +++ b/tests/tools/askBrowserstack.test.ts @@ -0,0 +1,1294 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { parseAsk } from "../../src/tools/ask-browserstack/stream.js"; +import { + AskError, + DEFAULT_ATLAS_URL, + DEFAULT_AUTH_TOKEN_URL, + agentUrl, + atlasBaseUrl, + authTokenUrl, + resetHostAnnouncements, +} from "../../src/tools/ask-browserstack/config.js"; +import { + AUTH_REJECTED_DETAIL, + AUTH_SCOPE_REFUSED_DETAIL, + AUTH_UNREACHABLE_DETAIL, + CENTRAL_SCOPE, + refusalIsAboutScope, + REFRESH_SKEW_MS, + mintCentralToken, + mintForm, + resetTokenCache, +} from "../../src/tools/ask-browserstack/central-oauth.js"; +import { agentHeaders } from "../../src/tools/ask-browserstack/egress.js"; +import { + approvalOutcome, + buildResult, + decide, + elicitationShape, + errorResult, + deriveStatus, + elicitationMessage, + isNotEntitled, + looksLikeDelegationResult, + neverReachedAgent, + UNAUTHENTICATED_DETAIL, +} from "../../src/tools/ask-browserstack/relay.js"; +import { ApprovalRecord, RelayMode } from "../../src/tools/ask-browserstack/types.js"; + +const PERM = "perm-" + "a".repeat(32); + +function ask(overrides: Record = {}) { + return { + perm_id: PERM, + product: "tm", + mode: "ask-always", + description: "Create the folder \"Regression\" under Sprint 42.", + ...overrides, + }; +} + +describe("decide — CONTRACT §7, and nothing but", () => { + it("allows an accept whose form carried no confirm field at all", () => { + // THE LIVE BUG. `confirm` used to be required with `default: false`, so a client + // rendered one unchecked checkbox and pressing APPROVE sent accept + confirm: false — + // and a human who approved was told "refused: a human said no". The approve path was + // unreachable. Absence of the field is now consent; `accept` already IS the answer. + expect(decide({ action: "accept" })) + .toEqual({ decision: "allow", reason: "" }); + expect(decide({ action: "accept", content: {} })) + .toEqual({ decision: "allow", reason: "" }); + }); + + it("allows an accept that volunteers confirm: true", () => { + expect(decide({ action: "accept", content: { confirm: true } })) + .toEqual({ decision: "allow", reason: "" }); + }); + + it("still honours an explicit untick as the human saying no", () => { + expect(decide({ action: "accept", content: { confirm: false } })) + .toEqual({ decision: "deny", reason: "declined" }); + }); + + it("denies a decline", () => { + expect(decide({ action: "decline" })) + .toEqual({ decision: "deny", reason: "declined" }); + }); + + it("denies a cancel, and says WHICH — no human was there", () => { + // The load-bearing row: a headless Claude Code returns cancel, so an unattended run + // must never be able to self-approve. + expect(decide({ action: "cancel" })) + .toEqual({ decision: "deny", reason: "cancelled" }); + }); + + it("ignores whatever a client volunteers, except an explicit false", () => { + // Nothing is requested any more, so the action decides. A volunteered `false` is still + // honoured as defensive belt, but no other value can override an accept. + for (const confirm of ["true", 1, "yes", null, undefined]) { + expect(decide({ action: "accept", content: { confirm } } as never)) + .toEqual({ decision: "allow", reason: "" }); + } + expect(decide({ action: "accept", content: { confirm: false } })) + .toEqual({ decision: "deny", reason: "declined" }); + }); + + it("describes the shape a client answered with, and nothing a human wrote", () => { + expect(elicitationShape({ action: "accept" })) + .toBe("action=accept content=absent confirm=absent"); + expect(elicitationShape({ action: "accept", content: {} })) + .toBe("action=accept content=present confirm=absent"); + expect(elicitationShape({ action: "accept", content: { confirm: false } })) + .toBe("action=accept content=present confirm=false"); + expect(elicitationShape({ action: "cancel" })) + .toBe("action=cancel content=absent confirm=absent"); + expect(elicitationShape({ action: "accept", content: { confirm: "yes" } } as never)) + .toBe("action=accept content=present confirm=non-boolean"); + + // It can only ever contain a fixed action enum and a boolean — never a description, a + // credential, or anything a user typed into a form. + const line = elicitationShape({ + action: "accept", + content: { confirm: "SECRET", notes: 'Creating the folder "askrelay-smoke-1"' }, + } as never); + expect(line).not.toContain("SECRET"); + expect(line).not.toContain("askrelay-smoke-1"); + expect(line).toBe("action=accept content=present confirm=non-boolean"); + }); + + it("treats an action it does not recognise as no answer at all", () => { + expect(decide({ action: "something-new" } as never).decision).toBe("deny"); + }); +}); + +describe("applied_before_stop — read from Atlas, never derived (D2/D4)", () => { + function build(body: Record) { + return buildResult({ status: 200, body: { status: "ok", answer: "", ...body } }, [], true); + } + + it("reports exactly what Atlas said, both ways", () => { + expect(build({ applied_before_stop: true }).applied_before_stop).toBe(true); + expect(build({ applied_before_stop: false }).applied_before_stop).toBe(false); + }); + + it("reports null — NOT false — when Atlas said nothing", () => { + // Absence means no gate ran, or an Atlas that predates the field. Rendering it as + // `false` would assert something nobody measured, in the direction that makes a caller + // retry a task that already half-applied. + expect(build({}).applied_before_stop).toBeNull(); + }); + + it("ignores a non-boolean rather than coercing it", () => { + expect(build({ applied_before_stop: "true" }).applied_before_stop).toBeNull(); + expect(build({ applied_before_stop: 1 }).applied_before_stop).toBeNull(); + expect(build({ applied_before_stop: null }).applied_before_stop).toBeNull(); + }); + + it("does not re-derive it from the trail it can see", () => { + // The old rule ("any allow preceded a deny") would have said true here. Atlas says + // false, because the approved step's request never landed — which only Atlas knows. + const result = build({ + approvals: [ + { description: "a", decision: "allow", reason: "", applied: false }, + { description: "b", decision: "deny", reason: "declined", applied: false }, + ], + applied_before_stop: false, + }); + expect(result.applied_before_stop).toBe(false); + }); +}); + +describe("the authoritative approval trail (D4)", () => { + const MINE: ApprovalRecord[] = [ + { description: "Creating the Alpha folder", decision: "allow", reason: "" }, + ]; + + function build(body: Record, mine = MINE) { + return buildResult({ status: 200, body: { status: "ok", answer: "", ...body } }, mine, true); + } + + it("prefers Atlas's trail over ours, and says which it used", () => { + const result = build({ + approvals: [ + { description: "Creating the Alpha folder", decision: "allow", reason: "", applied: true }, + ], + }); + expect(result.approvals_source).toBe("atlas"); + expect(result.approvals[0].applied).toBe(true); + }); + + it("keeps our own trail beside it, because the disagreement IS the signal", () => { + // An ask Atlas recorded as answered while no prompt ever appeared on this side is a + // denial to Atlas and nothing at all to us. Folding the two together would destroy + // the only evidence that it happened. + const result = build({ + approvals: [{ description: "Creating the Alpha folder", decision: "deny", reason: "error", applied: false }], + }, []); + expect(result.approvals_source).toBe("atlas"); + expect(result.approvals[0].decision).toBe("deny"); + expect(result.elicitations).toEqual([]); + }); + + it("falls back to ours only when Atlas sent no trail at all", () => { + const result = build({}); + expect(result.approvals_source).toBe("mcp"); + expect(result.approvals[0].description).toBe("Creating the Alpha folder"); + }); + + it("treats an EMPTY trail from Atlas as a trail, not as absence", () => { + // "The relay ran and nothing was asked" is a fact Atlas sent; ours is not a better + // answer to it. + const result = build({ approvals: [] }); + expect(result.approvals_source).toBe("atlas"); + expect(result.approvals).toEqual([]); + }); + + it("rebuilds each entry rather than trusting it, failing closed on a garbled decision", () => { + const result = build({ + approvals: [ + { description: "x", decision: "ALLOW", reason: "" }, + { description: "y", decision: "allow", reason: "" }, + "not an entry", + null, + ], + }); + // Anything that is not exactly "allow" reports as a refusal. + expect(result.approvals.map((e) => e.decision)).toEqual(["deny", "allow"]); + }); + + it("carries `applied` only when it is genuinely a boolean", () => { + const result = build({ + approvals: [ + { description: "a", decision: "allow", reason: "", applied: "yes" }, + { description: "b", decision: "allow", reason: "" }, + ], + }); + expect("applied" in result.approvals[0]).toBe(false); + expect("applied" in result.approvals[1]).toBe(false); + }); +}); + +describe("approvalOutcome — 'approved then failed' must not read like a refusal", () => { + const base = { description: "Creating the Alpha folder", reason: "" }; + + it("distinguishes an applied allow from one whose request failed", () => { + const applied = approvalOutcome({ ...base, decision: "allow", applied: true }); + const failed = approvalOutcome({ ...base, decision: "allow", applied: false }); + expect(applied).toBe("approved, and the change went through"); + expect(failed).toMatch(/APPROVED, BUT THE CHANGE DID NOT GO THROUGH/); + expect(failed).toMatch(/nobody refused it/); + expect(applied).not.toBe(failed); + }); + + it("does not render an unmeasured allow as a failure", () => { + // An older Atlas, or our own trail, simply does not know. + const unknown = approvalOutcome({ ...base, decision: "allow" }); + expect(unknown).toBe("approved; whether the change went through was not reported"); + expect(unknown).not.toMatch(/DID NOT GO THROUGH/); + }); + + it("keeps a failed write clearly apart from every kind of refusal", () => { + const failed = approvalOutcome({ ...base, decision: "allow", applied: false }); + for (const reason of ["declined", "cancelled", "timeout", "error", "anything-else"]) { + const refusal = approvalOutcome({ ...base, decision: "deny", reason }); + expect(refusal).toMatch(/^refused/); + expect(refusal).not.toBe(failed); + } + }); + + it("says WHICH kind of refusal, since they call for different things", () => { + const of = (reason: string) => approvalOutcome({ ...base, decision: "deny", reason }); + expect(of("declined")).toMatch(/a human said no/); + expect(of("cancelled")).toMatch(/nobody was there/); + expect(of("timeout")).toMatch(/nobody answered in time/); + expect(of("error")).toMatch(/channel broke/); + expect(of("something-new")).toBe("refused"); + }); + + it("is attached to every entry in both trails", () => { + const result = buildResult( + { + status: 200, + body: { + status: "ok", answer: "", + approvals: [{ description: "a", decision: "allow", reason: "", applied: false }], + }, + }, + [{ description: "a", decision: "allow", reason: "" }], + true, + ); + expect(result.approvals[0].outcome).toMatch(/DID NOT GO THROUGH/); + expect(result.elicitations[0].outcome).toMatch(/not reported/); + }); +}); + +describe("result assembly", () => { + it("prefers the status Atlas declared over anything it could infer", () => { + expect(deriveStatus({ status: 200, body: { status: "blocked" } }, [], [])).toBe("blocked"); + }); + + it("calls a run blocked when something was denied or left needing approval", () => { + // A real result that simply declared no usable status — `ok` and `answer` are what mark + // it as a delegation result at all. + const ran = { ok: true, answer: "" }; + const denied: ApprovalRecord[] = [{ description: "d", decision: "deny", reason: "cancelled" }]; + expect(deriveStatus({ status: 200, body: ran }, denied, [])).toBe("blocked"); + expect(deriveStatus({ status: 200, body: ran }, [], ["a write"])).toBe("blocked"); + expect(deriveStatus({ status: 200, body: ran }, [], [])).toBe("ok"); + }); + + it("maps 429 to rate_limited and every other non-2xx, including 0, to error", () => { + expect(deriveStatus({ status: 429, body: {} }, [], [])).toBe("rate_limited"); + expect(deriveStatus({ status: 500, body: {} }, [], [])).toBe("error"); + expect(deriveStatus({ status: 0, body: null }, [], [])).toBe("error"); + }); + + it("carries Atlas's payload through and never rewrites the answer", () => { + const body = { status: "ok", answer: "Created folder 12.", needs_approval: [], extra: "kept" }; + const result = buildResult({ status: 200, body }, [], true); + expect(result.ok).toBe(true); + expect(result.answer).toBe("Created folder 12."); + // Belt and braces: nothing Atlas sent is lost, even a field this side does not map. + expect(result.atlas_response).toEqual(body); + expect(result.permission_relay).toEqual({ + used: true, reason: "", detail: expect.stringContaining("asked before each change"), + }); + }); + + it("explains a read-only run rather than leaving a refused write unexplained", () => { + const result = buildResult( + { status: 200, body: { status: "blocked", answer: "", needs_approval: ["create folder"] } }, + [], "no_human", + ); + expect(result.status).toBe("blocked"); + expect(result.needs_approval).toEqual(["create folder"]); + expect(result.permission_relay.used).toBe(false); + expect(result.permission_relay.reason).toBe("no_human"); + expect(result.permission_relay.detail).toMatch(/does not support MCP elicitation/); + }); +}); + +// The transport it was written for is gone; every reason it exists survived the move, +// because they are all properties of the ASK rather than of how the ask arrived. +describe("parseAsk", () => { + it("keeps only the four fields of CONTRACT §2", () => { + // op_key, method, path and host stay on Atlas's side; if one ever arrived it would not + // be carried onward from here. + expect(parseAsk({ ...ask(), op_key: "x", method: "POST", path: "/api/v1/x" })) + .toEqual(ask()); + }); + + it("rejects a non-object", () => { + expect(parseAsk(null)).toBeNull(); + expect(parseAsk([ask()])).toBeNull(); + expect(parseAsk("perm-x")).toBeNull(); + }); + + it("refuses a perm_id that is not Atlas's shape", () => { + // It is the only thing that routes the decision back, so an id we could not have + // been sent is an answer that could never be delivered. Better to drop the frame + // than to prompt a human for a decision with nowhere to go. + expect(parseAsk(ask({ perm_id: "perm-nope" }))).toBeNull(); + expect(parseAsk(ask({ perm_id: "1234" }))).toBeNull(); + expect(parseAsk(ask({ perm_id: undefined }))).toBeNull(); + expect(parseAsk(ask({ perm_id: PERM.toUpperCase() }))).toBeNull(); + }); + + it("refuses a blank description: a prompt asking a human to approve nothing", () => { + expect(parseAsk(ask({ description: " " }))).toBeNull(); + expect(parseAsk(ask({ description: "" }))).toBeNull(); + expect(parseAsk(ask({ description: 42 }))).toBeNull(); + }); + + it("defaults product and mode rather than rejecting, because neither is load-bearing", () => { + // `description` is the whole of what the human reads and `perm_id` is what routes + // the answer. These two only decorate the prompt, so a missing one degrades it + // instead of dropping an ask a person could still have answered. + expect(parseAsk(ask({ product: undefined, mode: undefined }))) + .toEqual({ ...ask(), product: "", mode: "" }); + }); +}); + + +describe("the verified /agent response shape — CONTRACT v1.1 §B", () => { + it("reads an ABSENT needs_approval as empty, because public() omits it when empty", () => { + // The trap: Atlas never sends `[]`, it sends nothing at all. Same for narration, + // artifacts, error, cost_breach and usage. + const result = buildResult( + { status: 200, body: { ok: true, status: "ok", answer: "done", steps: [] } }, + [], true, + ); + expect(result.needs_approval).toEqual([]); + expect(result.status).toBe("ok"); + expect(result.ok).toBe(true); + }); + + it("passes through every status in the verified vocabulary", () => { + for (const status of ["ok", "error", "blocked", "rate_limited"]) { + expect(deriveStatus({ status: 200, body: { status } }, [], [])).toBe(status); + } + }); + + it("ignores a status outside the vocabulary and derives one instead", () => { + // `interrupted` is in the dataclass comment but is not emitted on this path. + expect(deriveStatus({ status: 200, body: { status: "interrupted" } }, [], [])).toBe("ok"); + expect(deriveStatus( + { status: 200, body: { status: "interrupted", needs_approval: ["x"] } }, [], ["x"], + )).toBe("blocked"); + }); + + it("lifts Atlas's own error string to the top level rather than burying it", () => { + const result = buildResult( + { status: 200, body: { ok: true, status: "error", answer: "", error: "the run died" } }, + [], true, + ); + expect(result.status).toBe("error"); + expect(result.error).toBe("the run died"); + }); + + it("has no error key when Atlas reported none", () => { + const result = buildResult({ status: 200, body: { status: "ok", answer: "" } }, [], true); + expect("error" in result).toBe(false); + }); +}); + +describe("Atlas's permission_relay verdict — CONTRACT v1.1 §D", () => { + function relayOf(permission_relay: unknown, mode: RelayMode = "offered") { + return buildResult( + { status: 200, body: { status: "blocked", answer: "", permission_relay } }, + [], mode, + ).permission_relay; + } + + it("prefers Atlas's used/reason over anything this side inferred", () => { + // We offered the channel and would have inferred used: true. Atlas knows better. + expect(relayOf({ used: false, reason: "disabled" })).toMatchObject({ + used: false, reason: "disabled", + }); + expect(relayOf({ used: true, reason: "" })).toMatchObject({ used: true, reason: "" }); + }); + + it("says plainly that a disabled relay is NOT anyone declining", () => { + // The distinction a user cannot make for themselves, and will retry forever without. + const relay = relayOf({ used: false, reason: "disabled" }); + expect(relay.detail).toMatch(/NOBODY DECLINED THIS/); + expect(relay.detail).toMatch(/administrator turns the relay on/); + expect(relay.detail).not.toMatch(/does not support MCP elicitation/); + }); + + it("gives host_not_allowed and malformed their own distinct sentences", () => { + // `host_not_allowed` can now only come from an Atlas older than A1 — the current one + // never dials out, so there is no host to refuse. The sentence stays mapped because + // such a deployment still deserves an explanation rather than a raw enum. + const host = relayOf({ used: false, reason: "host_not_allowed" }).detail; + const malformed = relayOf({ used: false, reason: "malformed" }).detail; + expect(host).toMatch(/predates the current approval channel/); + expect(host).toMatch(/same host/); + expect(malformed).toMatch(/bug on this side/); + expect(host).not.toBe(malformed); + }); + + it("degrades an unrecognised reason to a sentence instead of crashing", () => { + // Forward compatibility: an Atlas newer than this build may name a reason we have + // never heard of, and a result is not the place to throw. + const relay = relayOf({ used: false, reason: "quota_exhausted" }); + expect(relay.used).toBe(false); + expect(relay.reason).toBe("quota_exhausted"); + expect(relay.detail).toMatch(/does not recognise/); + expect(relay.detail).toMatch(/quota_exhausted/); + }); + + it("bounds an absurd reason before putting it in a sentence a human reads", () => { + const relay = relayOf({ used: false, reason: "x".repeat(5000) }); + expect(relay.detail.length).toBeLessThan(500); + }); + + it("falls back to the inferred verdict when Atlas sends none (an Atlas older than v1.1)", () => { + expect(relayOf(undefined)).toEqual({ + used: true, reason: "", detail: expect.stringContaining("asked before each change"), + }); + }); + + it("treats an unreadable verdict as no verdict rather than half-trusting it", () => { + expect(relayOf({ reason: "disabled" }).used).toBe(true); // no `used` boolean + expect(relayOf("disabled").used).toBe(true); + expect(relayOf([{ used: false }]).used).toBe(true); + }); + + it("keeps no_human ours: Atlas is never told the client cannot be prompted", () => { + // We omitted the block, so anything Atlas says about a relay cannot be about this run. + const relay = relayOf({ used: true, reason: "" }, "no_human"); + expect(relay).toEqual({ + used: false, + reason: "no_human", + detail: expect.stringContaining("does not support MCP elicitation"), + }); + }); +}); + +describe("REMOTE_MCP — the relay is a stdio-only feature", () => { + function relayOf(mode: RelayMode, permission_relay?: unknown) { + return buildResult( + { status: 200, body: { status: "blocked", answer: "", permission_relay } }, + [], mode, + ).permission_relay; + } + + it("says the DEPLOYMENT is why, not the human and not the client", () => { + const relay = relayOf("remote_mode"); + expect(relay).toEqual({ + used: false, + reason: "remote_mode", + detail: expect.stringContaining("NOBODY DECLINED THIS, AND YOUR CLIENT IS NOT THE PROBLEM"), + }); + expect(relay.detail).toMatch(/hosted, multi-tenant mode/); + expect(relay.detail).toMatch(/works when the server runs locally over stdio/); + }); + + it("reads differently from every other reason", () => { + const remote = relayOf("remote_mode").detail; + const noHuman = relayOf("no_human").detail; + const notReached = buildResult( + { status: 401, body: { detail: "unauthorized" } }, [], "remote_mode", + ).permission_relay.detail; + const disabled = relayOf("offered", { used: false, reason: "disabled" }).detail; + expect(new Set([remote, noHuman, notReached, disabled]).size).toBe(4); + // Telling a hosted user to switch to a client that can be prompted would waste their time. + expect(remote).not.toMatch(/does not support MCP elicitation/); + }); + + it("beats no_human, because switching clients cannot help a hosted deployment", () => { + // Both facts can be true at once. The deployment is the binding constraint and the only + // one the reader can act on. + expect(relayOf("remote_mode").reason).toBe("remote_mode"); + }); + + it("still loses to not_reached: a request that never arrived says so first", () => { + const relay = buildResult( + { status: 401, body: { detail: "unauthorized" } }, [], "remote_mode", + ).permission_relay; + expect(relay.reason).toBe("not_reached"); + }); +}); + +describe("the elicitation message — CONTRACT v1.1 §G", () => { + it("names the product and leaves the description untouched", () => { + const message = elicitationMessage("tm", "Create folder \"Regression\"."); + expect(message).toBe( + "BrowserStack AI (Test Management) needs your approval to continue:\n\n" + + "Create folder \"Regression\".", + ); + // Verbatim: the human must approve what the model actually said. + expect(message.endsWith("Create folder \"Regression\".")).toBe(true); + }); + + it("labels every product the tool accepts", () => { + expect(elicitationMessage("a11y", "x")).toMatch(/\(Accessibility\)/); + expect(elicitationMessage("tra", "x")).toMatch(/\(Test Reporting & Analytics\)/); + }); + + it("carries a product it has no label for rather than dropping it", () => { + expect(elicitationMessage("newproduct", "x")).toMatch(/\(newproduct\)/); + }); + + it("omits the brackets entirely rather than showing an empty pair", () => { + expect(elicitationMessage("", "x")).toBe( + "BrowserStack AI needs your approval to continue:\n\nx", + ); + }); + + it("still reads as a prompt when an older Atlas withheld the description", () => { + // BACK-COMPAT ONLY NOW. Atlas used to replace a route-shaped description with this + // placeholder; that guard is gone (CONTRACT v2 §3, amended) because it asked a human to + // approve a sentence they could not read. A deployment predating the change still sends + // it, so the framing must not make it look like a bug. + // + // THE REAL BYTES, as `permissions.py` emitted them — `f"({kind} withheld: it referenced + // internal API detail)"` with kind="approval request" — and observed on the wire in the + // integration run. An invented placeholder is the one string in this feature a test can + // assert on and be confidently wrong about. + const withheld = "(approval request withheld: it referenced internal API detail)"; + expect(elicitationMessage("tm", withheld)).toBe( + "BrowserStack AI (Test Management) needs your approval to continue:\n\n" + withheld, + ); + }); +}); + +describe("POST /agent headers — CONTRACT v1.2 §4", () => { + it("is exactly three headers, and Api-Token is not one of them", () => { + // /agent has no Api-Token path, and that header carries the user's access key. Asserting + // the exact key set is what stops it being reintroduced by a helpful future edit. + expect(agentHeaders("minted.central.jwt")).toEqual({ + Authorization: "Bearer minted.central.jwt", + "Content-Type": "application/json", + "request-source": "ai-chatbot", + }); + }); +}); + + +describe("minting a central JWT", () => { + const URL_ = "https://auth.example/oauth2/v2/token"; + const CREDS = { username: "ing_Xx", accessKey: "SECRET" }; + const OK = { status: 200, body: { access_token: "jwt.aaa", expires_in: 3600 } }; + + beforeEach(() => resetTokenCache()); + afterEach(() => resetTokenCache()); + + function recording(response: any = OK) { + const seen: { url: string; form: Record }[] = []; + const transport = async (url: string, form: Record) => { + seen.push({ url, form }); + return typeof response === "function" ? response() : response; + }; + return { seen, transport }; + } + + it("sends the client_credentials grant verbatim, with BOTH scope parts", () => { + expect(mintForm(CREDS)).toEqual({ + grant_type: "client_credentials", + username: "ing_Xx", + access_key: "SECRET", + scope: "oauth_user_profile ai_agent_notify", + expires_in: "3600", + }); + // Exact string, both members, in order: `ai_agent_notify` is what Atlas matches on, and + // `oauth_user_profile` is what makes the pair obtainable through this flow at all. + expect(CENTRAL_SCOPE).toBe("oauth_user_profile ai_agent_notify"); + expect(CENTRAL_SCOPE.split(" ")).toEqual(["oauth_user_profile", "ai_agent_notify"]); + }); + + it("never falls back to another scope, on any path", async () => { + // A silent downgrade to a different authorization is the kind of thing nobody notices + // until it matters. Every refusal shape must fail, once, with the scope we chose. + for (const response of [ + { status: 400, body: { error: "invalid_scope" } }, + { status: 401, body: { error: "invalid_client" } }, + { status: 403, body: { error: "unauthorized_client" } }, + { status: 500, body: null }, + ]) { + resetTokenCache(); + const { seen, transport } = recording(response); + await mintCentralToken(URL_, CREDS, transport, 0).catch(() => undefined); + expect(seen).toHaveLength(1); + expect(seen[0].form.scope).toBe("oauth_user_profile ai_agent_notify"); + } + // and nothing anywhere in the module names the scope it replaced + expect(JSON.stringify([AUTH_SCOPE_REFUSED_DETAIL(400), AUTH_REJECTED_DETAIL(401)])) + .not.toContain("central_ai_s2s"); + }); + + it("returns the token the endpoint issued", async () => { + const { seen, transport } = recording(); + expect(await mintCentralToken(URL_, CREDS, transport, 0)).toBe("jwt.aaa"); + expect(seen).toHaveLength(1); + expect(seen[0].url).toBe(URL_); + }); + + it("does not re-mint inside the cache window", async () => { + const { seen, transport } = recording(); + await mintCentralToken(URL_, CREDS, transport, 0); + await mintCentralToken(URL_, CREDS, transport, 60_000); + await mintCentralToken(URL_, CREDS, transport, 1_000_000); + expect(seen).toHaveLength(1); + }); + + it("refreshes once the token is inside the skew", async () => { + const { seen, transport } = recording(); + await mintCentralToken(URL_, CREDS, transport, 0); + // Expires at 3_600_000; the skew is the whole /agent budget plus a minute, because + // Atlas holds this token for the life of the run and re-uses it for product egress. + expect(REFRESH_SKEW_MS).toBe(390_000); + await mintCentralToken(URL_, CREDS, transport, 3_600_000 - REFRESH_SKEW_MS + 1); + expect(seen).toHaveLength(2); + }); + + it("mints ONCE for concurrent callers rather than once each", async () => { + const { seen, transport } = recording(); + const answers = await Promise.all([ + mintCentralToken(URL_, CREDS, transport, 0), + mintCentralToken(URL_, CREDS, transport, 0), + mintCentralToken(URL_, CREDS, transport, 0), + ]); + expect(seen).toHaveLength(1); + expect(answers).toEqual(["jwt.aaa", "jwt.aaa", "jwt.aaa"]); + }); + + it("mints again when the access key rotates, rather than serving a revoked one", async () => { + const { seen, transport } = recording(); + await mintCentralToken(URL_, CREDS, transport, 0); + await mintCentralToken(URL_, { ...CREDS, accessKey: "ROTATED" }, transport, 0); + expect(seen).toHaveLength(2); + }); + + it("trusts the lifetime the SERVER granted, not the one we asked for", async () => { + const { seen, transport } = recording({ + status: 200, body: { access_token: "jwt.aaa", expires_in: 600 }, + }); + await mintCentralToken(URL_, CREDS, transport, 0); + // Granted 600s, so it is already inside the 390s skew at t=300s. + await mintCentralToken(URL_, CREDS, transport, 300_000); + expect(seen).toHaveLength(2); + }); + + it("reports a refused SCOPE as a provisioning problem, naming the scope", async () => { + // The likely outcome: `ai_agent_notify` is documented as client_id/secret auth and is + // not in USERNAME_ACCESS_KEY_ONLY_SCOPES. Sending someone to check their password would + // be sending them to the wrong place entirely. + const transport = async () => ({ + status: 400, + body: { + error: "invalid_scope", + error_description: "scope ai_agent_notify only valid for: user_management, access_key SECRET", + }, + }); + const message = await mintCentralToken(URL_, CREDS, transport, 0).catch((e) => e.message); + expect(message).toContain("oauth_user_profile ai_agent_notify"); + expect(message).toMatch(/provisioning problem/); + expect(message).toMatch(/YOUR CREDENTIALS ARE NOT THE PROBLEM/); + expect(message).toMatch(/NOT quietly retry with a weaker scope/); + expect(message).not.toMatch(/Check BROWSERSTACK_USERNAME/); + // The body still never crosses, even while being read to classify. + expect(message).not.toContain("SECRET"); + expect(message).not.toContain("error_description"); + expect(message).not.toContain("only valid for"); + }); + + it("classifies a refusal by the OAuth2 code, falling back to the status", () => { + // The `error` code is a fixed spec token and cannot carry a credential; + // `error_description` is free text and can, so only the code is ever consulted. + expect(refusalIsAboutScope(400, { error: "invalid_scope" })).toBe(true); + expect(refusalIsAboutScope(403, { error: "unauthorized_client" })).toBe(true); + expect(refusalIsAboutScope(400, { error: "invalid_request" })).toBe(true); + expect(refusalIsAboutScope(401, { error: "invalid_client" })).toBe(false); + expect(refusalIsAboutScope(400, { error: "invalid_grant" })).toBe(false); + // No usable code: 400 is a bad request (for us, the scope), 401/403 a bad caller. + expect(refusalIsAboutScope(400, null)).toBe(true); + expect(refusalIsAboutScope(400, { error: 42 })).toBe(true); + expect(refusalIsAboutScope(400, { error: "something_new" })).toBe(true); + expect(refusalIsAboutScope(401, {})).toBe(false); + expect(refusalIsAboutScope(500, { error: "server_error" })).toBe(false); + }); + + it("keeps a scope refusal and a credential refusal as two different problems", async () => { + const scope = await mintCentralToken( + URL_, CREDS, async () => ({ status: 400, body: { error: "invalid_scope" } }), 0, + ).catch((e) => e.message); + resetTokenCache(); + const credential = await mintCentralToken( + URL_, CREDS, async () => ({ status: 401, body: { error: "invalid_client" } }), 0, + ).catch((e) => e.message); + expect(scope).not.toBe(credential); + expect(scope).toMatch(/provisioning/); + expect(credential).toMatch(/credentials were rejected/); + expect(credential).not.toMatch(/provisioning/); + }); + + it("surfaces ONLY the status when the CREDENTIAL is refused, never the body", async () => { + // The real endpoint's error body echoes the credential straight back. + const transport = async () => ({ + status: 401, + body: { error: "invalid_client", error_description: "access_key SECRET is invalid" }, + }); + await expect(mintCentralToken(URL_, CREDS, transport, 0)).rejects.toThrow(AskError); + const message = await mintCentralToken(URL_, CREDS, transport, 0).catch((e) => e.message); + expect(message).toMatch(/rejected by BrowserStack auth \(HTTP 401\)/); + expect(message).not.toContain("SECRET"); + expect(message).not.toContain("invalid_client"); + expect(message).not.toContain("access_key"); + }); + + it("says unreachable, distinctly from refused", async () => { + const transport = async () => ({ status: 0, body: null, error: "auth could not be reached" }); + const message = await mintCentralToken(URL_, CREDS, transport, 0).catch((e) => e.message); + expect(message).toBe(AUTH_UNREACHABLE_DETAIL); + expect(message).not.toMatch(/rejected/); + }); + + it("says so when a 200 carries no access_token", async () => { + const transport = async () => ({ status: 200, body: { token_type: "Bearer" } }); + const message = await mintCentralToken(URL_, CREDS, transport, 0).catch((e) => e.message); + expect(message).toMatch(/without issuing a token/); + }); + + it("does not cache a failure", async () => { + let calls = 0; + const transport = async () => { + calls += 1; + return calls === 1 ? { status: 500, body: null } : OK; + }; + await mintCentralToken(URL_, CREDS, transport, 0).catch(() => undefined); + expect(await mintCentralToken(URL_, CREDS, transport, 0)).toBe("jwt.aaa"); + expect(calls).toBe(2); + }); + + it("refuses before any network call when a credential is missing", async () => { + const { seen, transport } = recording(); + for (const creds of [ + { username: "", accessKey: "SECRET" }, + { username: "ing_Xx", accessKey: "" }, + ]) { + const message = await mintCentralToken(URL_, creds, transport, 0).catch((e) => e.message); + expect(message).toMatch(/BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY/); + // Our missing configuration, not the user's password being wrong. + expect(message).not.toMatch(/rejected/); + } + expect(seen).toHaveLength(0); + }); + + it("keeps the four auth failures readable as four different problems", () => { + const scopeRefused = AUTH_SCOPE_REFUSED_DETAIL(400); + const rejected = AUTH_REJECTED_DETAIL(401); + const unreachable = AUTH_UNREACHABLE_DETAIL; + const refusedByAtlas = UNAUTHENTICATED_DETAIL; + const all = [scopeRefused, rejected, unreachable, refusedByAtlas]; + expect(new Set(all).size).toBe(4); + // provisioning vs "your credentials are wrong" vs "auth is down" vs "server misconfigured" + expect(scopeRefused).toMatch(/provisioning problem/); + expect(rejected).toMatch(/credentials were rejected/); + expect(unreachable).toMatch(/Could not reach BrowserStack auth/); + expect(refusedByAtlas).toMatch(/SUCCEEDED/); + // The two that are NOT the user's credentials say so in as many words. + for (const message of [scopeRefused, refusedByAtlas]) { + expect(message).toMatch(/YOUR CREDENTIALS ARE NOT THE PROBLEM/); + } + // None of them is a permission denial. + for (const message of all) { + expect(message).toMatch(/NOTHING REACHED THE AGENT|never reached the agent/); + } + }); +}); + + +describe("pre-run refusals — the request never reached the agent (D3)", () => { + // Atlas omits its permission_relay verdict on every refusal that dies before the + // delegation layer, so "no verdict" alone cannot be read as "an old Atlas". + const REFUSALS: [string, { status: number; body: unknown; error?: string }][] = [ + ["401 unauthorized", { status: 401, body: { detail: "unauthorized" } }], + ["400 bad body", { status: 400, body: { detail: "task is required" } }], + ["503 delegation not enabled", { status: 503, body: { detail: "delegation is not enabled" } }], + ["unreachable", { status: 0, body: null, error: "BrowserStack AI could not be reached" }], + ]; + + it.each(REFUSALS)("detects %s", (_name, response) => { + expect(neverReachedAgent(response)).toBe(true); + }); + + it.each(REFUSALS)("never claims the channel was used on %s", (_name, response) => { + // The bug: `used: true` with an empty `approvals` and an `error` saying nothing was + // asked — three fields in one payload contradicting each other. + const relay = buildResult(response, [], true).permission_relay; + expect(relay.used).toBe(false); + expect(relay.reason).toBe("not_reached"); + expect(relay.detail).toMatch(/NOTHING WAS ASKED AND NOTHING WAS REFUSED/); + expect(relay.detail).not.toMatch(/asked before each change/); + }); + + it.each(REFUSALS)("always says why, on %s", (_name, response) => { + // A pre-run refusal with `status: "error"` and no `error` string leaves a caller with + // nothing to act on. 400 and 503 carry `detail`, never `error`. + const result = buildResult(response, [], true); + expect(result.status).toBe("error"); + expect(result.ok).toBe(false); + expect(typeof result.error).toBe("string"); + expect(result.error!.length).toBeGreaterThan(0); + expect(result.approvals).toEqual([]); + // Nobody measured anything, so the field asserts nothing. + expect(result.applied_before_stop).toBeNull(); + }); + + it("names the HTTP status and quotes Atlas's detail on a 400 and a 503", () => { + expect(buildResult({ status: 400, body: { detail: "task is required" } }, [], true).error) + .toBe('BrowserStack AI refused this request before the agent started (HTTP 400): "task is required".'); + expect(buildResult( + { status: 503, body: { detail: "delegation is not enabled" } }, [], true, + ).error).toMatch(/HTTP 503.*delegation is not enabled/); + }); + + it("bounds a detail string before putting it in front of a person", () => { + const result = buildResult({ status: 400, body: { detail: "x".repeat(5000) } }, [], true); + expect(result.error!.length).toBeLessThan(400); + }); + + it("still says something when a non-2xx has nothing to say for itself", () => { + expect(buildResult({ status: 502, body: null }, [], true).error) + .toBe("BrowserStack AI refused this request before the agent started (HTTP 502)."); + }); + + it("outranks no_human: a client that cannot elicit did not 'run read-only' either", () => { + // Saying the run went read-only would be as wrong as saying the channel was used — + // nothing ran at all. The elicitation gap resurfaces on the next run. + const relay = buildResult( + { status: 401, body: { detail: "unauthorized" } }, [], "no_human", + ).permission_relay; + expect(relay.reason).toBe("not_reached"); + expect(relay.detail).not.toMatch(/does not support MCP elicitation/); + }); + + it("treats a 200 carrying a bare detail as not a delegation result", () => { + expect(neverReachedAgent({ status: 200, body: { detail: "nope" } })).toBe(true); + // ...but a real result that happens to carry a detail field is still a result. + expect(neverReachedAgent({ + status: 200, body: { ok: true, status: "ok", answer: "", detail: "fyi" }, + })).toBe(false); + }); + + it("leaves a genuine agent run alone", () => { + const response = { status: 200, body: { ok: true, status: "ok", answer: "done" } }; + expect(neverReachedAgent(response)).toBe(false); + expect(buildResult(response, [], true).permission_relay).toEqual({ + used: true, reason: "", detail: expect.stringContaining("asked before each change"), + }); + }); + + it("says the same thing when the call never left this process", () => { + // No token, no host, a transport that threw: nothing was sent, so nothing was asked. + const relay = errorResult("BrowserStack AI is not authenticated: set …", []).permission_relay; + expect(relay).toEqual({ + used: false, + reason: "not_reached", + detail: expect.stringContaining("NOTHING WAS ASKED AND NOTHING WAS REFUSED"), + }); + }); + + it("keeps a pre-run refusal from reading like a relay that was switched off", () => { + // `disabled` means the agent RAN with the relay off — retry after an admin acts. + // `not_reached` means it never ran at all — fix what `error` names and retry now. + const notReached = buildResult( + { status: 401, body: { detail: "unauthorized" } }, [], true, + ).permission_relay; + const disabled = buildResult( + { status: 200, body: { status: "blocked", permission_relay: { used: false, reason: "disabled" } } }, + [], true, + ).permission_relay; + expect(notReached.reason).not.toBe(disabled.reason); + expect(notReached.detail).not.toBe(disabled.detail); + expect(disabled.detail).toMatch(/administrator turns the relay on/); + expect(notReached.detail).toMatch(/run it again/); + }); +}); + +describe("a result body outranks the HTTP status (N1)", () => { + // Atlas answers 502 with a COMPLETE result when a delegation ran and a step failed, and + // 429 carries a full body too. The status describes the OUTCOME; the body describes + // whether there was a RUN, and only the second question decides `not_reached`. + const RAN = { + ok: false, + status: "error", + answer: "The folder was not created.", + approvals: [ + { + description: 'Creating the "Regression" folder.', + decision: "allow", + reason: "", + applied: false, + }, + ], + applied_before_stop: false, + permission_relay: { used: true, reason: "" }, + }; + + it.each([[502], [429], [500], [503]])( + "treats HTTP %i carrying a real result as a run that happened", + (status) => { + expect(neverReachedAgent({ status, body: RAN })).toBe(false); + }, + ); + + it("reports the approval that WAS shown, on the 502 that regressed", () => { + const result = buildResult({ status: 502, body: RAN }, [], true); + + // The whole object, so `permission_relay` and `approvals` can never disagree again + // without this failing. + expect(result.permission_relay).toEqual({ + used: true, + reason: "", + detail: expect.stringContaining("asked before each change"), + }); + expect(result.permission_relay.detail).not.toMatch(/NOTHING WAS ASKED/); + expect(result.approvals).toEqual([ + { + description: 'Creating the "Regression" folder.', + decision: "allow", + reason: "", + applied: false, + outcome: expect.stringContaining("APPROVED, BUT THE CHANGE DID NOT GO THROUGH"), + }, + ]); + expect(result.approvals_source).toBe("atlas"); + // Atlas's status and its own verdict, not one derived from the 502. + expect(result.status).toBe("error"); + expect(result.applied_before_stop).toBe(false); + // "refused before the agent started" would be false — the agent ran. + expect(result.error).toBeUndefined(); + }); + + it("keeps Atlas's own status on a 429 with a full body", () => { + const result = buildResult( + { + status: 429, + body: { ok: false, status: "rate_limited", answer: "", approvals: [], applied_before_stop: false }, + }, + [], true, + ); + expect(result.status).toBe("rate_limited"); + expect(result.permission_relay.used).toBe(true); + expect(result.permission_relay.reason).toBe(""); + expect(result.applied_before_stop).toBe(false); + }); + + it("still honours Atlas's `disabled` verdict on a result-carrying non-2xx", () => { + const result = buildResult( + { + status: 502, + body: { ok: false, status: "error", answer: "", + permission_relay: { used: false, reason: "disabled" } }, + }, + [], true, + ); + expect(result.permission_relay.reason).toBe("disabled"); + expect(result.permission_relay.detail).toMatch(/NOBODY DECLINED THIS/); + }); + + it("lets Atlas's own error string speak on a 502 rather than talking over it", () => { + const result = buildResult( + { status: 502, body: { ok: false, status: "error", answer: "", error: "the folder API returned 500" } }, + [], true, + ); + expect(result.error).toBe("the folder API returned 500"); + }); + + it("still calls a bare {detail} refusal not_reached, whatever else changed", () => { + // 403 is deliberately NOT in this list any more — it has its own outcome, below. + for (const status of [400, 401, 502, 503]) { + const result = buildResult({ status, body: { detail: "nope" } }, [], true); + expect(neverReachedAgent({ status, body: { detail: "nope" } })).toBe(true); + expect(result.permission_relay.used).toBe(false); + expect(result.permission_relay.reason).toBe("not_reached"); + expect(result.applied_before_stop).toBeNull(); + expect(result.approvals).toEqual([]); + } + }); + + it("still calls a transport failure not_reached", () => { + const result = buildResult( + { status: 0, body: null, error: "BrowserStack AI could not be reached" }, [], true, + ); + expect(result.permission_relay.reason).toBe("not_reached"); + expect(result.applied_before_stop).toBeNull(); + expect(result.error).toBe("BrowserStack AI could not be reached"); + }); + + it("recognises a result by any of the four keys, and nothing by none of them", () => { + for (const key of ["ok", "status", "answer", "approvals"]) { + expect(looksLikeDelegationResult({ [key]: null })).toBe(true); + } + expect(looksLikeDelegationResult({ detail: "x" })).toBe(false); + expect(looksLikeDelegationResult({})).toBe(false); + expect(looksLikeDelegationResult(null)).toBe(false); + expect(looksLikeDelegationResult("a string")).toBe(false); + // A result that also carries a detail field is still a result. + expect(looksLikeDelegationResult({ ok: true, detail: "fyi" })).toBe(true); + }); +}); + +describe("a 2xx carrying no delegation result is internally consistent (N4)", () => { + it("does not report success alongside an error", () => { + const result = buildResult({ status: 200, body: { detail: "nope" } }, [], true); + // It used to say ok: true AND carry an error — a shape real Atlas never emits, but one + // that contradicted itself. + expect(result.ok).toBe(false); + expect(result.status).toBe("error"); + expect(result.error).toBe( + 'BrowserStack AI answered HTTP 200 with no delegation result: "nope".', + ); + expect(result.permission_relay.reason).toBe("not_reached"); + expect(result.applied_before_stop).toBeNull(); + }); + + it("says the same about a 2xx with an empty or unparseable body", () => { + for (const body of [{}, null, "not json"]) { + const result = buildResult({ status: 200, body }, [], true); + expect(result.ok).toBe(false); + expect(result.status).toBe("error"); + expect(result.error).toBe("BrowserStack AI answered HTTP 200 with no delegation result."); + } + }); +}); + +describe("host resolution — one hardcoded staging default, one override", () => { + const saved = { ...process.env }; + + beforeEach(() => { + delete process.env.ASK_BROWSERSTACK_ATLAS_URL; + delete process.env.ASK_BROWSERSTACK_AUTH_TOKEN_URL; + resetHostAnnouncements(); + }); + + afterEach(() => { + process.env = { ...saved }; + resetHostAnnouncements(); + }); + + it("with NO env vars, resolves the staging pair — exact literals", () => { + // Asserted literally so repointing to production has to be a deliberate test change + // rather than something that slips through. See DEFAULT-PROD-HOSTS. + expect(atlasBaseUrl()).toBe("https://workflows.browserstack.com"); + expect(agentUrl()).toBe("https://workflows.browserstack.com/agent"); + expect(authTokenUrl()).toBe("https://auth.browserstack.com/oauth2/v2/token"); + expect(DEFAULT_ATLAS_URL).toBe("https://workflows.browserstack.com"); + expect(DEFAULT_AUTH_TOKEN_URL).toBe("https://auth.browserstack.com/oauth2/v2/token"); + }); + + it("never refuses for want of configuration — an install needs no env var", () => { + expect(() => atlasBaseUrl()).not.toThrow(); + expect(() => authTokenUrl()).not.toThrow(); + }); + + it("lets the explicit override win, for both", () => { + process.env.ASK_BROWSERSTACK_ATLAS_URL = "https://atlas.example"; + process.env.ASK_BROWSERSTACK_AUTH_TOKEN_URL = "https://auth.example/t"; + expect(atlasBaseUrl()).toBe("https://atlas.example"); + expect(authTokenUrl()).toBe("https://auth.example/t"); + }); + + it("ignores a blank override rather than resolving to an empty host", () => { + process.env.ASK_BROWSERSTACK_ATLAS_URL = " "; + expect(atlasBaseUrl()).toBe(DEFAULT_ATLAS_URL); + }); + + it("strips trailing slashes, and the defaults carry none", () => { + process.env.ASK_BROWSERSTACK_ATLAS_URL = "https://atlas.example///"; + // Otherwise `${base}/agent` becomes `//agent`. + expect(atlasBaseUrl()).toBe("https://atlas.example"); + expect(agentUrl()).toBe("https://atlas.example/agent"); + expect(DEFAULT_ATLAS_URL.endsWith("/")).toBe(false); + expect(DEFAULT_AUTH_TOKEN_URL.endsWith("/")).toBe(false); + }); + + it("takes no notice of an environment selector any more", () => { + // The map and ASK_BROWSERSTACK_ENV are gone; a stale selector must not change anything. + process.env.ASK_BROWSERSTACK_ENV = "prod"; + process.env.CAPABILITY_REGISTRY_ENV = "prod"; + process.env.ASK_BROWSERSTACK_ATLAS_URL_PROD = "https://should-be-ignored.example"; + expect(atlasBaseUrl()).toBe(DEFAULT_ATLAS_URL); + expect(authTokenUrl()).toBe(DEFAULT_AUTH_TOKEN_URL); + }); +}); + +describe("the resolved host is announced, so a wrong deployment is visible", () => { + const saved = { ...process.env }; + let lines: string[]; + + beforeEach(async () => { + delete process.env.ASK_BROWSERSTACK_ATLAS_URL; + resetHostAnnouncements(); + lines = []; + const { setLogger } = await import("../../src/logger.js"); + const capture = (...args: unknown[]) => lines.push(args.map(String).join(" ")); + setLogger({ info: capture, warn: capture, error: capture, debug: capture, flush: () => {} }); + }); + + afterEach(async () => { + process.env = { ...saved }; + resetHostAnnouncements(); + const { setLogger } = await import("../../src/logger.js"); + const { pino } = await import("pino"); + setLogger(pino({ level: "silent" })); + }); + + it("names the default as the source when nothing is configured", () => { + atlasBaseUrl(); + authTokenUrl(); + // The logger is printf-style, so the captured args arrive alongside the format string. + const everything = lines.join("\n"); + expect(everything).toContain("source:"); + expect(everything).toContain("Atlas https://workflows.browserstack.com default"); + expect(everything).toContain( + "auth token endpoint https://auth.browserstack.com/oauth2/v2/token default", + ); + }); + + it("names the env var as the source when one is set", () => { + process.env.ASK_BROWSERSTACK_ATLAS_URL = "https://atlas.example"; + atlasBaseUrl(); + expect(lines.join("\n")).toContain("Atlas https://atlas.example env"); + expect(lines.join("\n")).not.toContain("default"); + }); + + it("announces once, not on every call, and never carries a credential", () => { + for (let i = 0; i < 5; i += 1) atlasBaseUrl(); + expect(lines.filter((l) => l.includes("Atlas"))).toHaveLength(1); + const everything = lines.join("\n"); + expect(everything).not.toContain("SECRET"); + expect(everything).not.toContain("access_key"); + expect(everything).not.toMatch(/Bearer/); + }); + + it("announces again when the host actually changes", () => { + atlasBaseUrl(); + process.env.ASK_BROWSERSTACK_ATLAS_URL = "https://atlas.example"; + atlasBaseUrl(); + expect(lines.filter((l) => l.includes("Atlas"))).toHaveLength(2); + }); +}); + +describe("403 — the account is not entitled, which is none of the other failures", () => { + // The shape Atlas's entitlement gate returns; the sentence in it is deliberately not what + // we key on. + const REFUSED = { status: 403, body: { detail: "agent is not enabled for this account" } }; + + it("keys on the status, not the prose", () => { + expect(isNotEntitled(REFUSED)).toBe(true); + // A reworded body must classify identically — pattern-matching the sentence would fail + // silently into a generic error the first time someone edits it. + expect(isNotEntitled({ status: 403, body: { detail: "totally different wording" } })) + .toBe(true); + expect(isNotEntitled({ status: 403, body: null })).toBe(true); + expect(isNotEntitled({ status: 403, body: { ok: false, status: "error", answer: "" } })) + .toBe(true); + // ...and nothing else is an entitlement problem. + for (const status of [0, 200, 400, 401, 429, 500, 502, 503]) { + expect(isNotEntitled({ status, body: { detail: "agent is not enabled" } })).toBe(false); + } + }); + + it("gives the user the sentence they asked for, naming the product", () => { + const result = buildResult(REFUSED, [], "offered", "a11y"); + expect(result.error).toContain( + "BrowserStack AI is not enabled for `a11y` on your account. Please contact your admin.", + ); + }); + + it("drops the product clause rather than naming an empty one", () => { + expect(buildResult(REFUSED, [], "offered").error).toContain( + "BrowserStack AI is not enabled on your account. Please contact your admin.", + ); + expect(buildResult(REFUSED, [], "offered", " ").error).not.toContain("``"); + }); + + it("says outright that the credentials are fine, so a working key is not rotated", () => { + const result = buildResult(REFUSED, [], "offered", "tm"); + expect(result.error).toMatch(/YOUR CREDENTIALS ARE FINE/); + expect(result.error).toMatch(/per-product entitlement/); + }); + + it("is its own reason, not not_reached and not a denial", () => { + const result = buildResult(REFUSED, [], "offered", "tm"); + expect(result.permission_relay.used).toBe(false); + expect(result.permission_relay.reason).toBe("not_entitled"); + expect(result.permission_relay.detail).toMatch(/NOBODY DECLINED THIS AND NOTHING RAN/); + // The relay vocabulary stays clean. + expect(result.permission_relay.reason).not.toBe("not_reached"); + expect(result.approvals).toEqual([]); + expect(result.elicitations).toEqual([]); + }); + + it("beats not_reached and remote_mode, which are both also true of a 403", () => { + expect(buildResult(REFUSED, [], "remote_mode", "tm").permission_relay.reason) + .toBe("not_entitled"); + expect(buildResult(REFUSED, [], "no_human", "tm").permission_relay.reason) + .toBe("not_entitled"); + }); + + it("leaves applied_before_stop null: no gate ran", () => { + const result = buildResult(REFUSED, [], "offered", "tm"); + expect(result.applied_before_stop).toBeNull(); + expect(result.status).toBe("error"); + expect(result.ok).toBe(false); + }); + + it("is a FIFTH distinct thing, not any of the four auth failures", () => { + const entitlement = buildResult(REFUSED, [], "offered", "tm").error!; + // /agent's 401 is "we signed in fine, Atlas refused the token" — a server + // misconfiguration. A 403 is neither that nor a credential problem. + const atlasRefusedToken = buildResult( + { status: 401, body: { detail: "unauthorized" } }, [], "offered", "tm", + ).error!; + + const all = [ + entitlement, + atlasRefusedToken, + AUTH_SCOPE_REFUSED_DETAIL(400), + AUTH_REJECTED_DETAIL(401), + AUTH_UNREACHABLE_DETAIL, + ]; + expect(new Set(all).size).toBe(5); + + // The entitlement one sends the reader to an admin, and nowhere near a credential. + expect(entitlement).toMatch(/contact your admin/); + expect(entitlement).not.toMatch(/rejected/); + expect(entitlement).not.toMatch(/ASK_BROWSERSTACK/); + expect(entitlement).not.toMatch(/required_scope/); + expect(atlasRefusedToken).toMatch(/required_scope/); + }); + + it("prefers our actionable sentence over Atlas's bare detail", () => { + const withError = buildResult( + { status: 403, body: { detail: "nope", error: "forbidden" } }, [], "offered", "tm", + ); + expect(withError.error).toMatch(/Please contact your admin/); + expect(withError.error).not.toBe("forbidden"); + }); +}); diff --git a/tests/tools/askBrowserstackE2E.test.ts b/tests/tools/askBrowserstackE2E.test.ts new file mode 100644 index 0000000..8631b97 --- /dev/null +++ b/tests/tools/askBrowserstackE2E.test.ts @@ -0,0 +1,1438 @@ +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { ErrorCode, McpError } from "@modelcontextprotocol/sdk/types.js"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { AskError } from "../../src/tools/ask-browserstack/config.js"; +import { resetTokenCache } from "../../src/tools/ask-browserstack/central-oauth.js"; +import { addAskBrowserstackAITool } from "../../src/tools/ask-browserstack/register.js"; + +const CONFIG = { + "browserstack-username": "ing_Xx", + "browserstack-access-key": "SECRET", +} as any; + +const PERM_A = "perm-" + "a".repeat(32); +const AUTH_URL = "https://auth.example/oauth2/v2/token"; +const MINTED = "minted.central.jwt"; +const PERM_B = "perm-" + "b".repeat(32); + +interface AtlasCall { + url: string; + headers: Record; + body: any; +} + +/** + * Plays Atlas: receives POST /agent, then calls the MCP server back over REAL loopback + * HTTP for each ask before answering. Every hop except Atlas's own logic is genuine. + */ +/** + * A fake Atlas speaking CONTRACT v2 (A1). + * + * The shape change from A2 is the whole point and it is visible right here: this stub no + * longer dials back into the tool. It streams `run`, then a `permission` frame per + * scripted ask, then one `result` — and each ask is held open until the tool answers it + * on a SEPARATE `POST /agent/{run_id}/permission`, which lands on this same stub. + * + * Every assertion these tests make is about `relay.ts` and `buildResult`, which A1 does + * not touch. Repointing this one function is therefore the whole migration: if the + * behaviour those tests pin were transport-dependent, that would be the bug. + */ +function atlas(options: { + asks?: { perm_id: string; description: string }[]; + payload?: (decisions: any[]) => unknown; + throws?: boolean; + authStatus?: number; + authError?: string; + /** Answer the decision POST with something other than 204. */ + decisionStatus?: number; + /** Reply to `POST /agent` with plain JSON, as an Atlas that predates A1 does. */ + json?: boolean; +}) { + const calls: AtlasCall[] = []; + const decisions: any[] = []; + const mints: Record[] = []; + const RUN_ID = "run-" + "a".repeat(32); + const encoder = new TextEncoder(); + let answered: ((body: unknown) => void) | null = null; + + const frame = (event: string, data: unknown) => + encoder.encode(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`); + + const jsonResponse = (status: number, payload: unknown) => ({ + ok: status >= 200 && status < 300, + status, + headers: { get: (k: string) => (k === "content-type" ? "application/json" : "") }, + json: async () => payload, + }); + + const stub = async (url: string, init: any) => { + if (String(url) === AUTH_URL) { + mints.push(Object.fromEntries(new URLSearchParams(init.body))); + return jsonResponse( + options.authStatus ?? 200, + options.authStatus && options.authStatus !== 200 + ? { + error: options.authError ?? "invalid_client", + error_description: + "scope ai_agent_notify only valid for: user_management, access_key SECRET", + } + : { access_token: MINTED, expires_in: 3600, token_type: "Bearer" }, + ); + } + + // The decision endpoint. Recording it and releasing the stream is what makes this + // an A1 round trip rather than a scripted playback. + if (/\/agent\/[^/]+\/permission$/.test(String(url))) { + const body = JSON.parse(init.body); + const status = options.decisionStatus ?? 204; + decisions.push({ status, body }); + answered?.(body); + answered = null; + return { ok: status < 300, status, headers: { get: () => "" }, json: async () => null }; + } + + const body = JSON.parse(init.body); + calls.push({ url: String(url), headers: init.headers, body }); + if (options.throws) throw new Error("connection reset"); + + const payloadFor = () => + options.payload + ? options.payload(decisions) + : { status: "ok", answer: "done", needs_approval: [] }; + + // No relay asked for, or an Atlas that does not know A1: a plain JSON body. The + // tool's stream transport degrades to a single `result`, which is exactly how it + // stays safe against a deployment that has not shipped v2 yet. + if (!body.permission_relay || options.json) { + return jsonResponse(200, payloadFor()); + } + + const stream = new ReadableStream({ + async start(controller) { + controller.enqueue(frame("run", { run_id: RUN_ID })); + for (const ask of options.asks || []) { + const wait = new Promise((resolve) => { + answered = resolve; + }); + controller.enqueue( + frame("permission", { + ...ask, + product: body.product, + mode: "ask-always", + }), + ); + // Held open deliberately: Atlas's gate blocks here, and a stub that raced + // ahead would test a sequence the real server cannot produce. + await wait; + } + controller.enqueue(frame("result", payloadFor())); + controller.close(); + }, + }); + return { + ok: true, + status: 200, + headers: { + get: (k: string) => (k === "content-type" ? "text/event-stream" : ""), + }, + body: stream, + json: async () => null, + }; + }; + + vi.stubGlobal("fetch", stub); + return { calls, decisions, mints, RUN_ID }; +} + +/** + * A bare fetch stub that still signs in. Every path to `/agent` now mints first, so a stub + * that only knows about `/agent` would have the mint land on it instead. + */ +function withAuth(agent: (url: string, init: any) => Promise) { + return async (url: string, init: any) => { + if (String(url) === AUTH_URL) { + return { + status: 200, + headers: { get: () => "application/json" }, + json: async () => ({ access_token: MINTED, expires_in: 3600 }), + }; + } + return agent(url, init); + }; +} + +async function buildServer() { + const { BrowserStackMcpServer } = await import("../../src/server-factory.js"); + return new BrowserStackMcpServer(CONFIG); +} + +/** Give the server a client that can (or cannot) be prompted, and script its answers. */ +function fakeClient( + mcp: McpServer, + capabilities: Record | undefined, + answers: any[] = [], +) { + vi.spyOn(mcp.server, "getClientCapabilities").mockReturnValue(capabilities as never); + const elicit = vi.spyOn(mcp.server, "elicitInput"); + for (const answer of answers) { + if (answer instanceof Error) elicit.mockRejectedValueOnce(answer); + else elicit.mockResolvedValueOnce(answer); + } + return elicit; +} + +async function call(tools: Record, args = { product: "tm", query: "make a folder" }) { + const result = await tools.askBrowserstackAI.handler(args, {} as any); + return { result, payload: JSON.parse(result.content[0].text) }; +} + +describe("askBrowserstackAI, end to end through the server factory", () => { + beforeEach(() => { + process.env.ASK_BROWSERSTACK_ATLAS_URL = "https://atlas.example"; + process.env.ASK_BROWSERSTACK_AUTH_TOKEN_URL = AUTH_URL; + resetTokenCache(); + delete process.env.ASK_BROWSERSTACK_DISABLED; + delete process.env.ASK_BROWSERSTACK_ENV; + vi.resetModules(); + }); + + afterEach(() => { + delete process.env.ASK_BROWSERSTACK_ATLAS_URL; + delete process.env.ASK_BROWSERSTACK_AUTH_TOKEN_URL; + resetTokenCache(); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + }); + + it("registers alongside the existing surface, and honours its kill switch", async () => { + const server = await buildServer(); + expect(server.getTools().askBrowserstackAI).toBeDefined(); + expect(server.getTools().listTestCases ?? server.getTools().createTestCase).toBeDefined(); + + process.env.ASK_BROWSERSTACK_DISABLED = "true"; + vi.resetModules(); + expect((await buildServer()).getTools().askBrowserstackAI).toBeUndefined(); + }); + + describe("the client CAN elicit", () => { + it("advertises itself as the fallback, so specific tools win when one fits", async () => { + // The description is the ONLY thing steering tool choice — the client picks from these + // words before any call happens — so the fallback framing has to be in it, and near + // the front where it is read. + const server = await buildServer(); + const description = (server.getTools().askBrowserstackAI as any).description as string; + + expect(description).toMatch(/^Use this when no other BrowserStack tool here fits/); + expect(description).toMatch(/or when the ones you tried did not get you there/); + expect(description).toMatch(/Prefer a specific tool whenever one fits/); + // ...and it still says what it does and what consent looks like. + expect(description).toMatch(/plain language/); + expect(description).toMatch(/asks you to confirm/); + expect(description).toMatch(/deletes are refused outright/); + expect(description).toMatch(/One task per call/); + }); + + it("relays an ask, approves it, and forwards the caller's own credentials", async () => { + const server = await buildServer(); + const elicit = fakeClient(server.getInstance(), { elicitation: {} }, [ + { action: "accept", content: { confirm: true } }, + ]); + const stub = atlas({ + asks: [{ perm_id: PERM_A, description: "Create folder \"Regression\"." }], + payload: () => ({ + ok: true, status: "ok", answer: "Created folder 12.", steps: [], + // Atlas's authoritative trail, with the `applied` bit only it can fill in. + approvals: [ + { description: "Create folder \"Regression\".", decision: "allow", reason: "", applied: true }, + ], + applied_before_stop: false, + permission_relay: { used: true, reason: "" }, + }), + }); + + const { payload } = await call(server.getTools()); + + // 1. the request: authenticated with the SHARED DELEGATION TOKEN, attributed, relay + // offered on loopback with a per-run token + expect(stub.calls[0].url).toBe("https://atlas.example/agent"); + expect(stub.calls[0].headers.Authorization).toBe(`Bearer ${MINTED}`); + // The EXACT key set. /agent has no Api-Token path, and that header carries the user's + // access key — this assertion is what stops it creeping back in. + expect(Object.keys(stub.calls[0].headers).sort()) + // `Accept: text/event-stream` is A1's: it asks for the stream explicitly. The + // point of pinning the exact set is unchanged — no access key, no cookie, no + // second credential may appear here. + .toEqual(["Accept", "Authorization", "Content-Type", "request-source"]); + expect(stub.calls[0].body.task).toBe("make a folder"); + expect(stub.calls[0].body.product).toBe("tm"); + expect(stub.calls[0].body.user_id).toBe("ing_Xx"); + // A1: the block names the transport and carries NOTHING else. No URL, because + // there is nothing to dial; no per-run bearer, because there is no inbound + // connection to authenticate. That absence is the fix — the URL this used to + // carry was a loopback address a pod could never reach. + expect(stub.calls[0].body.permission_relay).toEqual({ mode: "stream" }); + + // 2. the prompt: framed with the product, Atlas's description verbatim, boolean confirm + expect(elicit).toHaveBeenCalledTimes(1); + const request = elicit.mock.calls[0][0] as any; + expect(request.message).toBe( + "BrowserStack AI (Test Management) needs your approval to continue:\n\n" + + "Create folder \"Regression\".", + ); + // NOTHING IS REQUESTED. A `confirm` boolean here made the approve button unable to + // approve — the client rendered a form field and submitted its unset value, so an + // approval came back as a refusal. The action is the whole answer now, and this + // assertion is what stops the field creeping back. + expect(request.requestedSchema).toEqual({ type: "object", properties: {} }); + expect(request.requestedSchema.required).toBeUndefined(); + expect(Object.keys(request.requestedSchema.properties)).toEqual([]); + // The inner rung of the timeout ladder, shorter than Atlas's 300s gate. + expect((elicit.mock.calls[0][1] as any).timeout).toBe(270_000); + + // 3. the answer on the wire, echoing Atlas's own id + expect(stub.decisions[0]) + // 204: v2 §3.3, the decision endpoint has nothing to return. + .toEqual({ status: 204, body: { perm_id: PERM_A, decision: "allow", reason: "" } }); + + // 4. the result + expect(payload.ok).toBe(true); + expect(payload.answer).toBe("Created folder 12."); + // Atlas's trail wins, carrying the applied bit; ours is kept beside it. + expect(payload.approvals_source).toBe("atlas"); + expect(payload.approvals).toEqual([{ + description: "Create folder \"Regression\".", decision: "allow", reason: "", + applied: true, outcome: "approved, and the change went through", + }]); + expect(payload.elicitations).toEqual([{ + description: "Create folder \"Regression\".", decision: "allow", reason: "", + outcome: "approved; whether the change went through was not reported", + }]); + expect(payload.applied_before_stop).toBe(false); + expect(payload.permission_relay.used).toBe(true); + }); + + it("APPROVES when the client accepts without sending a confirm field", async () => { + // Exactly what a real client sends for a schema with no required fields, and the + // shape the user hit on preprod when they pressed approve and were told they had + // refused. + const server = await buildServer(); + const elicit = fakeClient(server.getInstance(), { elicitation: {} }, [ + { action: "accept" }, + ]); + const stub = atlas({ + asks: [{ perm_id: PERM_A, description: 'Creating the root folder "askrelay-smoke-1".' }], + payload: () => ({ + ok: true, status: "ok", answer: "Created it.", steps: [], + approvals: [{ + description: 'Creating the root folder "askrelay-smoke-1".', + decision: "allow", reason: "", applied: true, + }], + applied_before_stop: false, + permission_relay: { used: true, reason: "" }, + }), + }); + + const { payload } = await call(server.getTools()); + + expect(elicit).toHaveBeenCalledTimes(1); + // On the wire to Atlas: an allow, not a denial. + expect(stub.decisions[0].body) + .toEqual({ perm_id: PERM_A, decision: "allow", reason: "" }); + // And the two trails agree that it was approved. + expect(payload.approvals[0]).toMatchObject({ decision: "allow", applied: true }); + expect(payload.elicitations[0]).toMatchObject({ decision: "allow", reason: "" }); + expect(payload.approvals[0].outcome).toBe("approved, and the change went through"); + expect(payload.status).toBe("ok"); + }); + + it("logs the SHAPE of the answer, never the description or a credential", async () => { + // The module's default export is a Proxy, so it is swapped wholesale rather than spied. + const { setLogger } = await import("../../src/logger.js"); + const lines: string[] = []; + const capture = (...args: unknown[]) => lines.push(args.map(String).join(" ")); + const previous = (await import("pino")).pino({ level: "silent" }); + setLogger({ info: capture, warn: capture, error: capture, debug: capture, flush: () => {} }); + + const server = await buildServer(); + fakeClient(server.getInstance(), { elicitation: {} }, [{ action: "accept" }]); + atlas({ + asks: [{ perm_id: PERM_A, description: 'Creating the root folder "askrelay-smoke-1".' }], + }); + + try { + await call(server.getTools()); + } finally { + setLogger(previous); + } + + const answered = lines.filter((line) => line.includes("elicitation answered")); + expect(answered).toHaveLength(1); // once per ask, never per retry + expect(answered[0]).toContain("action=accept"); + expect(answered[0]).toContain("content=absent"); + // The whole point: readable next time, and safe to leave switched on. + const everything = lines.join("\n"); + expect(everything).not.toContain("askrelay-smoke-1"); + expect(everything).not.toContain("SECRET"); + expect(everything).not.toContain(MINTED); + }); + + it("denies when the human confirms false, and never asks a second time", async () => { + const server = await buildServer(); + const elicit = fakeClient(server.getInstance(), { elicitation: {} }, [ + { action: "accept", content: { confirm: false } }, + ]); + const stub = atlas({ + asks: [{ perm_id: PERM_A, description: "Delete the sprint." }], + payload: () => ({ + status: "blocked", answer: "", needs_approval: ["Delete the sprint."], + approvals: [ + { description: "Delete the sprint.", decision: "deny", reason: "declined", applied: false }, + ], + applied_before_stop: false, + }), + }); + + const { payload } = await call(server.getTools()); + + expect(stub.decisions[0].body) + .toEqual({ perm_id: PERM_A, decision: "deny", reason: "declined" }); + expect(elicit).toHaveBeenCalledTimes(1); + expect(payload.status).toBe("blocked"); + expect(payload.approvals[0].reason).toBe("declined"); + expect(payload.approvals[0].outcome).toBe("refused: a human said no"); + // Nothing had applied, so a retry of the whole task is safe. + expect(payload.applied_before_stop).toBe(false); + }); + + it("denies a decline", async () => { + const server = await buildServer(); + fakeClient(server.getInstance(), { elicitation: {} }, [{ action: "decline" }]); + const stub = atlas({ asks: [{ perm_id: PERM_A, description: "Archive the plan." }] }); + + const { payload } = await call(server.getTools()); + expect(stub.decisions[0].body) + .toEqual({ perm_id: PERM_A, decision: "deny", reason: "declined" }); + expect(payload.approvals[0].decision).toBe("deny"); + }); + + it("denies a cancel as 'nobody was there', not as a refusal, and does not retry", async () => { + const server = await buildServer(); + const elicit = fakeClient(server.getInstance(), { elicitation: {} }, [{ action: "cancel" }]); + const stub = atlas({ asks: [{ perm_id: PERM_A, description: "Archive the plan." }] }); + + const { payload } = await call(server.getTools()); + + // This is the security property: an unattended run cannot self-approve, and a second + // prompt would only be an attempt to wear a human down. + expect(stub.decisions[0].body) + .toEqual({ perm_id: PERM_A, decision: "deny", reason: "cancelled" }); + expect(elicit).toHaveBeenCalledTimes(1); + expect(payload.approvals[0].reason).toBe("cancelled"); + }); + + it("denies on an elicitation timeout without killing the run", async () => { + const server = await buildServer(); + fakeClient(server.getInstance(), { elicitation: {} }, [ + new McpError(ErrorCode.RequestTimeout, "timed out"), + ]); + const stub = atlas({ asks: [{ perm_id: PERM_A, description: "Archive the plan." }] }); + + const { payload } = await call(server.getTools()); + expect(stub.decisions[0]) + .toEqual({ status: 204, body: { perm_id: PERM_A, decision: "deny", reason: "timeout" } }); + expect(payload.approvals[0].reason).toBe("timeout"); + }); + + it("routes each elicitation onto the tool call's own stream (relatedRequestId)", async () => { + // THE BUG THIS EXISTS FOR, found only against the hosted server. + // + // Streamable HTTP sends a server->client message on the stream of the request it + // relates to. With no `relatedRequestId` the SDK falls back to the standalone SSE + // stream — and a host answering GET /mcp with 405 has none, so the elicitation is + // DROPPED SILENTLY ("Stream is disconnected"). The tool then waits out its 270s and + // Atlas's gate expires into `reason: "timeout"`: the person is told they failed to + // answer a question they were never shown. + // + // Invisible on stdio, which has one pipe and nothing to route — which is why every + // local test passed while the hosted run timed out. Note the shared `call()` helper + // passes `{}` as `extra`, so it could never have caught this; this test supplies a + // request id the way the SDK does. + const server = await buildServer(); + const elicit = fakeClient(server.getInstance(), { elicitation: {} }, [ + { action: "accept" }, + ]); + atlas({ asks: [{ perm_id: PERM_A, description: "Archive the plan." }] }); + + await server.getTools().askBrowserstackAI.handler( + { product: "tm", query: "make a folder" }, + { requestId: 4242 } as never, + ); + + expect(elicit).toHaveBeenCalledTimes(1); + const options = elicit.mock.calls[0][1]; + expect(options).toMatchObject({ relatedRequestId: 4242 }); + }); + + it("fails closed with a non-200 when the relay breaks in an unexpected way", async () => { + const server = await buildServer(); + fakeClient(server.getInstance(), { elicitation: {} }, [new Error("client went away")]); + const stub = atlas({ asks: [{ perm_id: PERM_A, description: "Archive the plan." }] }); + + const { payload } = await call(server.getTools()); + // A1 inverts WHERE the break shows up. Under A2 the callback answered 500 and + // Atlas read that as a deny. Now the elicitation itself failed on our side, so we + // are the ones who must send an explicit deny — silence would leave Atlas waiting + // out its 300s gate. The invariant is the same and it is the one that matters: a + // broken channel never becomes an approval. + expect(stub.decisions[0].body).toEqual({ + perm_id: PERM_A, decision: "deny", reason: "error", + }); + expect(payload.approvals[0]).toEqual({ + description: "Archive the plan.", decision: "deny", reason: "error", + outcome: "refused: the approval channel broke before any answer arrived", + }); + }); + + it("does not retry or fail the run when a decision is refused", async () => { + // The A1 replacement for A2's "a stray local process cannot present the run's + // token". That hazard is GONE: nothing dials in, so there is no inbound + // connection to authenticate and no per-run bearer to steal. Atlas authorises the + // decision instead, on the attested JWT plus an unguessable run_id (v2 §3.1). + // + // What remains on this side is the opposite risk: a decision Atlas refuses (409 + // already-decided, 404 stale) must not be re-sent. A retry could land an approval + // on a step the run has already moved past. Losing it is safe — Atlas's gate + // denies on its own expiry — so we log and carry on. + const server = await buildServer(); + const elicit = fakeClient(server.getInstance(), { elicitation: {} }, [ + { action: "accept" }, + ]); + const stub = atlas({ + asks: [{ perm_id: PERM_A, description: "Create folder." }], + decisionStatus: 409, + }); + + const { payload } = await call(server.getTools()); + expect(elicit).toHaveBeenCalledTimes(1); + expect(stub.decisions).toHaveLength(1); // sent once, never re-sent + expect(payload.status).toBe("ok"); // and the run still completed + }); + + it("says some steps applied before it stopped", async () => { + const server = await buildServer(); + fakeClient(server.getInstance(), { elicitation: {} }, [ + { action: "accept", content: { confirm: true } }, + { action: "decline" }, + ]); + atlas({ + asks: [ + { perm_id: PERM_A, description: "Create folder \"Regression\"." }, + { perm_id: PERM_B, description: "Move 40 test cases into it." }, + ], + payload: () => ({ + status: "blocked", answer: "Created the folder; the move was refused.", + needs_approval: ["Move 40 test cases into it."], + approvals: [ + { description: "Create folder \"Regression\".", decision: "allow", reason: "", applied: true }, + { description: "Move 40 test cases into it.", decision: "deny", reason: "declined", applied: false }, + ], + // Atlas's own verdict: it stopped on a refusal AND something had already changed. + applied_before_stop: true, + }), + }); + + const { payload } = await call(server.getTools()); + // The whole point of the field: a caller must not retry this task from scratch. It is + // READ from Atlas, which is the only side that knows the folder creation landed. + expect(payload.applied_before_stop).toBe(true); + expect(payload.approvals_source).toBe("atlas"); + expect(payload.approvals.map((a: any) => a.decision)).toEqual(["allow", "deny"]); + expect(payload.approvals.map((a: any) => a.applied)).toEqual([true, false]); + expect(payload.approvals[0].outcome).toBe("approved, and the change went through"); + expect(payload.approvals[1].outcome).toBe("refused: a human said no"); + expect(payload.needs_approval).toEqual(["Move 40 test cases into it."]); + }); + + it("reports a server-side disabled relay as a configuration fact, not a refusal", async () => { + const server = await buildServer(); + const elicit = fakeClient(server.getInstance(), { elicitation: {} }, []); + // v1.1 §D: the block was supplied, `delegation.permission_relay` is "off", so Atlas + // ignored it and ran read-only. Nobody was ever asked. + const stub = atlas({ + payload: () => ({ + ok: true, status: "blocked", answer: "I could not create the folder.", steps: [], + needs_approval: ["Create folder \"Regression\"."], + permission_relay: { used: false, reason: "disabled" }, + }), + }); + + const { result, payload } = await call(server.getTools()); + + expect(stub.calls[0].body.permission_relay).toBeDefined(); // we DID offer it + expect(elicit).not.toHaveBeenCalled(); // Atlas never called back + expect(payload.permission_relay).toEqual({ + used: false, + reason: "disabled", + detail: expect.stringContaining("NOBODY DECLINED THIS"), + }); + expect(payload.approvals).toEqual([]); + // No gate ran, so Atlas omits the field and we must not invent a measured `false`. + expect(payload.applied_before_stop).toBeNull(); + // A refusal is not a tool failure, and rendering it as one invites the retry loop + // these distinct reasons exist to prevent. + expect(result.isError).toBeUndefined(); + }); + + it("treats an absent needs_approval as empty, end to end", async () => { + const server = await buildServer(); + fakeClient(server.getInstance(), { elicitation: {} }, [ + { action: "accept", content: { confirm: true } }, + ]); + atlas({ + asks: [{ perm_id: PERM_A, description: "Create folder." }], + // Exactly what public() emits when nothing needed approval: the key is absent. + payload: () => ({ + ok: true, status: "ok", answer: "Created folder 12.", steps: [], + permission_relay: { used: true, reason: "" }, + }), + }); + + const { result, payload } = await call(server.getTools()); + expect(payload.needs_approval).toEqual([]); + expect(payload.status).toBe("ok"); + expect(payload.permission_relay).toEqual({ + used: true, reason: "", detail: expect.stringContaining("asked before each change"), + }); + expect(result.isError).toBeUndefined(); + }); + + it("marks a genuine failure as an error but a refusal as a result", async () => { + const server = await buildServer(); + fakeClient(server.getInstance(), { elicitation: {} }, [{ action: "decline" }]); + atlas({ + asks: [{ perm_id: PERM_A, description: "Delete the sprint." }], + payload: () => ({ + ok: true, status: "rate_limited", answer: "", error: "too many runs", steps: [], + }), + }); + + const { result, payload } = await call(server.getTools()); + expect(payload.status).toBe("rate_limited"); + expect(payload.error).toBe("too many runs"); + expect(result.isError).toBe(true); + }); + + it("keeps the two trails apart when Atlas denies without us prompting (D4)", async () => { + // A2's version of this was a stray local process probing the loopback port with the + // wrong bearer: 401, zero prompts, and Atlas recording a denial we never saw. That + // hazard is GONE under A1 — there is no port to probe and no bearer to get wrong. + // + // The INVARIANT it protected is not gone, and is what this now covers: Atlas's + // trail and ours are separate records, and ours being empty is the only evidence + // that no human was ever prompted. Atlas can refuse a step on its own — an expired + // gate, a policy refusal — and when it does, the two trails must not be merged. + const server = await buildServer(); + const elicit = fakeClient(server.getInstance(), { elicitation: {} }, []); + const stub = atlas({ + // No asks: Atlas refused the step without ever putting one on the stream. + payload: () => ({ + status: "blocked", answer: "", needs_approval: ["Create folder."], + approvals: [ + { description: "Create folder.", decision: "deny", reason: "error", applied: false }, + ], + applied_before_stop: false, + permission_relay: { used: true, reason: "" }, + }), + }); + + const { payload } = await call(server.getTools()); + + expect(stub.decisions).toEqual([]); // we answered nothing + expect(elicit).not.toHaveBeenCalled(); // and nobody was prompted + // Atlas's is authoritative... + expect(payload.approvals_source).toBe("atlas"); + expect(payload.approvals[0].decision).toBe("deny"); + // ...and ours is empty, which is the ONLY record that no prompt ever appeared. + expect(payload.elicitations).toEqual([]); + expect(payload.applied_before_stop).toBe(false); + }); + + it("degrades cleanly when an older Atlas sends a trail with no applied bit", async () => { + const server = await buildServer(); + fakeClient(server.getInstance(), { elicitation: {} }, [ + { action: "accept", content: { confirm: true } }, + ]); + atlas({ + asks: [{ perm_id: PERM_A, description: "Create folder." }], + payload: () => ({ + status: "ok", answer: "done", + approvals: [{ description: "Create folder.", decision: "allow", reason: "" }], + // and no applied_before_stop either + }), + }); + + const { payload } = await call(server.getTools()); + expect("applied" in payload.approvals[0]).toBe(false); + // Not rendered as a failure: nobody measured it. + expect(payload.approvals[0].outcome).toBe( + "approved; whether the change went through was not reported", + ); + expect(payload.applied_before_stop).toBeNull(); + }); + + it("does not claim nobody was asked when the run fails after an approval (N1)", async () => { + // A2's version used a 502 whose BODY carried a real result. Under A1 that shape + // cannot occur: an ask requires an open 200 stream, so a 502 can never have + // carried one. The failure now arrives where it belongs — in the `result` event of + // a stream that did prompt and was approved. + // + // The invariant is unchanged and is the one N1 fixed: a run that asked and got a + // yes must NOT be reported as "nothing was asked". Reading the status alone got + // this wrong; the body is the signal. + const server = await buildServer(); + const elicit = fakeClient(server.getInstance(), { elicitation: {} }, [ + { action: "accept" }, + ]); + const stub = atlas({ + asks: [{ perm_id: PERM_A, description: 'Creating the "Regression" folder.' }], + payload: () => ({ + ok: false, status: "error", answer: "The folder was not created.", steps: [], + approvals: [{ + description: 'Creating the "Regression" folder.', + decision: "allow", reason: "", applied: false, + }], + applied_before_stop: false, + permission_relay: { used: true, reason: "" }, + }), + }); + + const { payload } = await call(server.getTools()); + + expect(elicit).toHaveBeenCalledTimes(1); + expect(stub.decisions[0].body.decision).toBe("allow"); + // The run failed, but a human WAS asked and did approve — so the relay verdict + // must not read as "not_reached", and the trail must survive. + expect(payload.permission_relay.used).toBe(true); + expect(payload.permission_relay.reason).toBe(""); + expect(payload.approvals[0].decision).toBe("allow"); + expect(payload.applied_before_stop).toBe(false); + }); + + // REMOVED WITH A2: "tears the listener down once the call ends". Its subject was the + // ephemeral loopback port the callback transport opened per tool call, and A1 opens + // none — there is nothing left to leak. It had also gone vacuous before it was + // deleted: it read `permission_relay.callback_url` off a body that now carries only + // `{mode}`, so it was probing `undefined` and passing on the resulting throw. The + // half of it that still means something (a failed call reports the relay as + // `not_reached` and prompts nobody) is asserted below, against the same stub. + }); + + describe("the client CANNOT elicit — the opencode/goose path", () => { + it("omits permission_relay entirely and explains the read-only run", async () => { + const server = await buildServer(); + const elicit = fakeClient(server.getInstance(), { roots: {} }, []); + const stub = atlas({ + payload: () => ({ + status: "blocked", answer: "I could not create the folder.", + needs_approval: ["Create folder \"Regression\"."], + }), + }); + + const { payload } = await call(server.getTools()); + + // Absence is what selects Atlas's read-only HeadlessGate — not an empty object. + expect("permission_relay" in stub.calls[0].body).toBe(false); + expect(Object.keys(stub.calls[0].body).sort()) + .toEqual(["product", "task", "user_id"]); + expect(elicit).not.toHaveBeenCalled(); + + expect(payload.status).toBe("blocked"); + expect(payload.approvals).toEqual([]); + expect(payload.approvals_source).toBe("mcp"); + expect(payload.applied_before_stop).toBeNull(); + expect(payload.needs_approval).toEqual(["Create folder \"Regression\"."]); + expect(payload.permission_relay).toEqual({ + used: false, + reason: "no_human", + detail: expect.stringMatching(/does not support MCP elicitation/), + }); + }); + + it("does not depend on sampling, which Claude Code does not declare", async () => { + const server = await buildServer(); + // Exactly what Claude Code sends: roots and elicitation, no sampling. + fakeClient(server.getInstance(), { roots: { listChanged: true }, elicitation: {} }, [ + { action: "accept", content: { confirm: true } }, + ]); + const stub = atlas({ asks: [{ perm_id: PERM_A, description: "Create folder." }] }); + + const { payload } = await call(server.getTools()); + expect(stub.calls[0].body.permission_relay).toBeDefined(); + expect(payload.approvals[0].decision).toBe("allow"); + }); + + it("treats a client with no capabilities at all as unable to be asked", async () => { + const server = await buildServer(); + fakeClient(server.getInstance(), undefined, []); + const stub = atlas({}); + + const { payload } = await call(server.getTools()); + expect(stub.calls[0].body.permission_relay).toBeUndefined(); + expect(payload.permission_relay.reason).toBe("no_human"); + }); + }); + + describe("POST /agent authentication — CONTRACT v1.2", () => { + it("omits user_id entirely, never as \"\", when no username is available", async () => { + // Driven through the seam: with central auth a blank username cannot sign in at all, + // so this branch is no longer reachable from the factory — but the wire rule still + // holds and must stay covered. + const mcp = new McpServer({ name: "t", version: "0" }); + const streamed = vi.fn(() => ({ + async *[Symbol.asyncIterator]() { + yield { event: "result", data: { status: "ok", answer: "" } }; + }, + })); + const tools = addAskBrowserstackAITool(mcp, { + agentUrl: () => "https://atlas.example/agent", + mintToken: async () => MINTED, + credentialsFor: () => ({ username: "", accessKey: "" }), + streamTransport: streamed as never, + }); + + await call(tools); + const body = (streamed.mock.calls[0] as never as Record[])[2]; + expect("user_id" in body).toBe(false); + expect(Object.keys(body).sort()).toEqual(["product", "task"]); + }); + + it("signs in against the built-in staging endpoint when nothing is configured", async () => { + // No refusal any more: the hosts ship with the tool, so an install needs no env var. + delete process.env.ASK_BROWSERSTACK_AUTH_TOKEN_URL; + delete process.env.ASK_BROWSERSTACK_ATLAS_URL; + const seen: string[] = []; + vi.stubGlobal("fetch", async (url: string) => { + seen.push(String(url)); + return { + status: 200, + headers: { get: () => "application/json" }, + json: async () => (String(url).includes("oauth2") + ? { access_token: MINTED, expires_in: 3600 } + : { ok: true, status: "ok", answer: "" }), + }; + }); + const server = await buildServer(); + fakeClient(server.getInstance(), { roots: {} }, []); + + const { payload } = await call(server.getTools()); + expect(payload.status).toBe("ok"); + expect(seen).toEqual([ + "https://auth.browserstack.com/oauth2/v2/token", + "https://workflows.browserstack.com/agent", + ]); + }); + + it("mints the token with the exact client_credentials grant", async () => { + const server = await buildServer(); + fakeClient(server.getInstance(), { elicitation: {} }, []); + const stub = atlas({}); + + await call(server.getTools()); + expect(stub.mints).toHaveLength(1); + expect(stub.mints[0]).toEqual({ + grant_type: "client_credentials", + username: "ing_Xx", + access_key: "SECRET", + // BOTH parts, exact string. `ai_agent_notify` is what Atlas matches on; + // `oauth_user_profile` is what makes the pair obtainable through this flow. + scope: "oauth_user_profile ai_agent_notify", + expires_in: "3600", + }); + }); + + it("does not re-mint on a second call inside the cache window", async () => { + const server = await buildServer(); + fakeClient(server.getInstance(), { elicitation: {} }, []); + const stub = atlas({}); + + await call(server.getTools()); + await call(server.getTools()); + await call(server.getTools()); + expect(stub.calls).toHaveLength(3); // three runs... + expect(stub.mints).toHaveLength(1); // ...one sign-in + }); + + it("never puts the access key or the minted token in the result", async () => { + const server = await buildServer(); + fakeClient(server.getInstance(), { elicitation: {} }, [ + { action: "accept", content: { confirm: true } }, + ]); + atlas({ asks: [{ perm_id: PERM_A, description: "Create folder." }] }); + + const { result } = await call(server.getTools()); + // The whole serialised result, not just the fields we happen to check. + expect(result.content[0].text).not.toContain("SECRET"); + expect(result.content[0].text).not.toContain(MINTED); + }); + + it("reports a refused scope as provisioning, not as a bad password", async () => { + const server = await buildServer(); + const elicit = fakeClient(server.getInstance(), { elicitation: {} }, []); + const stub = atlas({ authStatus: 400, authError: "invalid_scope" }); + + const { result, payload } = await call(server.getTools()); + + expect(payload.error).toContain("oauth_user_profile ai_agent_notify"); + expect(payload.error).toMatch(/provisioning problem/); + expect(payload.error).not.toMatch(/Check BROWSERSTACK_USERNAME/); + // Only one sign-in attempt, and no retry with a different scope. + expect(stub.mints).toHaveLength(1); + expect(stub.mints[0].scope).toBe("oauth_user_profile ai_agent_notify"); + // The body is read to classify but never surfaced. + expect(result.content[0].text).not.toContain("SECRET"); + expect(result.content[0].text).not.toContain("only valid for"); + expect(stub.calls).toHaveLength(0); + expect(elicit).not.toHaveBeenCalled(); + expect(payload.permission_relay.reason).toBe("not_reached"); + expect(payload.applied_before_stop).toBeNull(); + }); + + it("says the credentials were rejected, and NEVER echoes the auth body back", async () => { + const server = await buildServer(); + const elicit = fakeClient(server.getInstance(), { elicitation: {} }, []); + // The endpoint's own error body echoes the access key straight back. + const stub = atlas({ authStatus: 401 }); + + const { result, payload } = await call(server.getTools()); + + expect(payload.error).toMatch(/credentials were rejected by BrowserStack auth \(HTTP 401\)/); + expect(payload.error).toMatch(/BROWSERSTACK_ACCESS_KEY/); + // Only the status crosses. Not the body, not the key it contained. + expect(result.content[0].text).not.toContain("SECRET"); + expect(result.content[0].text).not.toContain("invalid_client"); + // Never got as far as Atlas, let alone a prompt. + expect(stub.calls).toHaveLength(0); + expect(elicit).not.toHaveBeenCalled(); + expect(payload.permission_relay.reason).toBe("not_reached"); + expect(payload.applied_before_stop).toBeNull(); + }); + + it("says auth was unreachable, distinctly from a rejected credential", async () => { + const server = await buildServer(); + const elicit = fakeClient(server.getInstance(), { elicitation: {} }, []); + vi.stubGlobal("fetch", async () => { + throw new Error("ECONNREFUSED"); + }); + + const { payload } = await call(server.getTools()); + expect(payload.error).toMatch(/Could not reach BrowserStack auth/); + expect(payload.error).not.toMatch(/credentials were rejected/); + expect(elicit).not.toHaveBeenCalled(); + expect(payload.permission_relay.reason).toBe("not_reached"); + }); + + it("refuses before any network call when a credential is missing", async () => { + const { BrowserStackMcpServer } = await import("../../src/server-factory.js"); + const server = new BrowserStackMcpServer({ + "browserstack-username": "ing_Xx", + "browserstack-access-key": "", + } as any); + fakeClient(server.getInstance(), { elicitation: {} }, []); + const fetchSpy = vi.fn(); + vi.stubGlobal("fetch", fetchSpy); + + const { payload } = await call(server.getTools()); + expect(payload.error).toMatch(/BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY/); + // Our missing configuration, not the user's password being wrong. + expect(payload.error).not.toMatch(/rejected/); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it("reads a 401 as rejected credentials, not as anyone declining", async () => { + const server = await buildServer(); + const elicit = fakeClient(server.getInstance(), { elicitation: {} }, []); + vi.stubGlobal("fetch", withAuth(async () => ({ + status: 401, + headers: { get: () => "application/json" }, + // Exactly what Atlas answers a bad bearer with: no `error` string of its own. + json: async () => ({ detail: "unauthorized" }), + }))); + + const { result, payload } = await call(server.getTools()); + + expect(payload.status).toBe("error"); + expect(result.isError).toBe(true); + // The THIRD failure mode: we signed in fine, Atlas refused the token. It must not + // read as "your password is wrong" — it is a server misconfiguration. + expect(payload.error).toMatch(/Signing in with your BrowserStack credentials SUCCEEDED/); + expect(payload.error).toMatch(/YOUR CREDENTIALS ARE NOT THE PROBLEM/); + expect(payload.error).toMatch(/delegation\.required_scope/); + expect(payload.error).toMatch(/NOBODY DECLINED ANYTHING/); + // Nothing was ever asked, so nothing can look like a refusal. + expect(elicit).not.toHaveBeenCalled(); + expect(payload.approvals).toEqual([]); + expect(payload.applied_before_stop).toBeNull(); + // ...and the relay verdict must agree with `error` rather than contradict it. + expect(payload.permission_relay).toEqual({ + used: false, + reason: "not_reached", + detail: expect.stringContaining("NOTHING WAS ASKED AND NOTHING WAS REFUSED"), + }); + }); + + it("does not claim the channel was used when Atlas is unreachable", async () => { + const server = await buildServer(); + const elicit = fakeClient(server.getInstance(), { elicitation: {} }, []); + const stub = atlas({ throws: true }); + + const { payload } = await call(server.getTools()); + + // We offered the channel and it was never exercised — zero prompts appeared. + expect(stub.calls[0].body.permission_relay).toBeDefined(); + expect(elicit).not.toHaveBeenCalled(); + expect(payload.status).toBe("error"); + expect(payload.error).toMatch(/could not be reached/); + expect(payload.permission_relay.used).toBe(false); + expect(payload.permission_relay.reason).toBe("not_reached"); + expect(payload.approvals).toEqual([]); + }); + + it("reads a 403 as 'AI is not enabled for your account', naming the product", async () => { + const server = await buildServer(); + const elicit = fakeClient(server.getInstance(), { elicitation: {} }, []); + vi.stubGlobal("fetch", withAuth(async () => ({ + status: 403, + headers: { get: () => "application/json" }, + json: async () => ({ detail: "agent is not enabled for this account" }), + }))); + + const { result, payload } = await call(server.getTools(), { + product: "a11y", query: "scan the site", + }); + + expect(payload.error).toContain( + "BrowserStack AI is not enabled for `a11y` on your account. Please contact your admin.", + ); + expect(payload.error).toMatch(/YOUR CREDENTIALS ARE FINE/); + expect(payload.permission_relay).toEqual({ + used: false, + reason: "not_entitled", + detail: expect.stringContaining("NOBODY DECLINED THIS AND NOTHING RAN"), + }); + // Nothing ran, nobody was asked, and no gate reported anything. + expect(elicit).not.toHaveBeenCalled(); + expect(payload.approvals).toEqual([]); + expect(payload.applied_before_stop).toBeNull(); + expect(payload.status).toBe("error"); + expect(result.isError).toBe(true); + }); + + it("classifies a 403 the same way when Atlas rewords the body", async () => { + // Keyed on the status, never the prose. + const server = await buildServer(); + fakeClient(server.getInstance(), { elicitation: {} }, []); + vi.stubGlobal("fetch", withAuth(async () => ({ + status: 403, + headers: { get: () => "application/json" }, + json: async () => ({ detail: "some entirely new sentence", code: "whatever" }), + }))); + + const { payload } = await call(server.getTools()); + expect(payload.permission_relay.reason).toBe("not_entitled"); + expect(payload.error).toMatch(/Please contact your admin/); + }); + + it.each([ + [400, "task is required"], + [503, "delegation is not enabled"], + ])("reads a %i pre-run refusal as nothing-asked, and says why", async (status, detail) => { + const server = await buildServer(); + const elicit = fakeClient(server.getInstance(), { elicitation: {} }, []); + // Exactly what Atlas answers with before the delegation layer: a bare detail. + vi.stubGlobal("fetch", withAuth(async () => ({ + status, + headers: { get: () => "application/json" }, + json: async () => ({ detail }), + }))); + + const { result, payload } = await call(server.getTools()); + + expect(payload.status).toBe("error"); + expect(result.isError).toBe(true); + expect(payload.error).toContain(`HTTP ${status}`); + expect(payload.error).toContain(detail); + expect(payload.permission_relay.reason).toBe("not_reached"); + expect(elicit).not.toHaveBeenCalled(); + expect(payload.approvals).toEqual([]); + }); + + it("keeps a 401 and a permission denial from reading alike", async () => { + const server = await buildServer(); + fakeClient(server.getInstance(), { elicitation: {} }, [{ action: "decline" }]); + atlas({ + asks: [{ perm_id: PERM_A, description: "Delete the sprint." }], + payload: () => ({ + ok: true, status: "blocked", answer: "", steps: [], + needs_approval: ["Delete the sprint."], + permission_relay: { used: true, reason: "" }, + }), + }); + + const { result, payload } = await call(server.getTools()); + // A denial: not an error, and the trail says who said no. + expect(payload.status).toBe("blocked"); + expect(result.isError).toBeUndefined(); + expect(payload.error).toBeUndefined(); + expect(payload.approvals[0]).toMatchObject({ decision: "deny", reason: "declined" }); + }); + + it("an environment selector no longer changes anything", async () => { + // The map and ASK_BROWSERSTACK_ENV are gone; a leftover selector must be inert rather + // than quietly repointing the tool. + process.env.ASK_BROWSERSTACK_ENV = "prod"; + process.env.ASK_BROWSERSTACK_ATLAS_URL_PROD = "https://should-be-ignored.example"; + try { + const server = await buildServer(); + fakeClient(server.getInstance(), { roots: {} }, []); + const stub = atlas({}); + + await call(server.getTools()); + // Still the explicit override this suite sets, never the selector's host. + expect(stub.calls[0].url).toBe("https://atlas.example/agent"); + expect(stub.calls[0].headers.Authorization).toBe(`Bearer ${MINTED}`); + } finally { + delete process.env.ASK_BROWSERSTACK_ENV; + delete process.env.ASK_BROWSERSTACK_ATLAS_URL_PROD; + } + }); + }); + + describe("REMOTE_MCP — the hosted deployment must not attempt the relay", () => { + /** + * `appConfig` reads `process.env.REMOTE_MCP` once at module load, so the whole graph is + * re-imported with the env in place — the same trick `tests/lib/tm-base-url.test.ts` uses. + */ + async function buildRemoteServer() { + vi.resetModules(); + process.env.REMOTE_MCP = "true"; + const { BrowserStackMcpServer } = await import("../../src/server-factory.js"); + return new BrowserStackMcpServer(CONFIG); + } + + afterEach(() => { + delete process.env.REMOTE_MCP; + vi.resetModules(); + }); + + it("never binds a listener, omits permission_relay, and says why", async () => { + const server = await buildRemoteServer(); + // A client that CAN be prompted — this is the case where remote_mode has to beat + // no_human, because switching clients would not help. + const elicit = fakeClient(server.getInstance(), { roots: {}, elicitation: {} }, []); + const stub = atlas({ + payload: () => ({ + ok: true, status: "blocked", answer: "I could not create the folder.", steps: [], + needs_approval: ["Create folder \"Regression\"."], + }), + }); + + const { result, payload } = await call(server.getTools()); + + // 1. nothing was offered to Atlas + expect("permission_relay" in stub.calls[0].body).toBe(false); + expect(Object.keys(stub.calls[0].body).sort()) + .toEqual(["product", "task", "user_id"]); + // 2. nothing was ever asked + expect(elicit).not.toHaveBeenCalled(); + // 3. the result blames the deployment, not the human and not the client + expect(payload.permission_relay).toEqual({ + used: false, + reason: "remote_mode", + detail: expect.stringContaining("hosted, multi-tenant mode"), + }); + expect(payload.permission_relay.detail) + .not.toMatch(/does not support MCP elicitation/); + // 4. a read-only run is not a tool failure + expect(result.isError).toBeUndefined(); + expect(payload.needs_approval).toEqual(["Create folder \"Regression\"."]); + }); + + it("offers no relay at all — `permission_relay` is never put on the body", async () => { + // Not "offered and left to fail on an ask nobody can be shown": never offered. The + // ask channel A1 uses needs a server-initiated elicitation, which the stateless + // hosted `/mcp` cannot do across replicas (v2 §5). + // + // Asserted through the injected seam rather than a module spy, so a negative result + // means the code did not send it — not that the spy failed to attach. The positive + // control below is what makes this assertion mean anything. + vi.resetModules(); + process.env.REMOTE_MCP = "true"; + const { addAskBrowserstackAITool } = await import( + "../../src/tools/ask-browserstack/register.js" + ); + const { McpServer: RemoteMcpServer } = await import( + "@modelcontextprotocol/sdk/server/mcp.js" + ); + + const streamed = vi.fn(() => ({ + async *[Symbol.asyncIterator]() { + yield { event: "result", data: { status: "ok", answer: "" } }; + }, + })); + + const remote = new RemoteMcpServer({ name: "t", version: "0" }); + vi.spyOn(remote.server, "getClientCapabilities") + .mockReturnValue({ elicitation: {} } as never); + const tools = addAskBrowserstackAITool(remote, { + agentUrl: () => "https://atlas.example/agent", + mintToken: async () => MINTED, + credentialsFor: () => ({ username: "ing_Xx", accessKey: "SECRET" }), + streamTransport: streamed as never, + }); + + const { payload } = await call(tools); + // A1 binds nothing anywhere, so "never bound a port" is no longer the property to + // assert — it is now true by construction. What still matters, and is what this + // guarded all along, is that the hosted deployment OFFERS no relay: the ask + // channel it would get cannot survive being spread across replicas (v2 §5). + expect("permission_relay" in (streamed.mock.calls[0] as never as unknown[])[2]!).toBe(false); + expect(payload.permission_relay.reason).toBe("remote_mode"); + }); + + it("DOES offer the relay in remote mode once the operator opts in", async () => { + // The refusal above is about the HOST, not this tool: a stateless host cannot + // deliver an elicitation answer to the instance waiting for it. Once the host keeps + // one server per session (verified against the hosted Streamable HTTP server, + // browserstack/remote-mcp-server#96) the refusal is wrong, so it is opt-in rather + // than absolute. + vi.resetModules(); + process.env.REMOTE_MCP = "true"; + process.env.ASK_BROWSERSTACK_ALLOW_REMOTE_RELAY = "true"; + try { + const { addAskBrowserstackAITool } = await import( + "../../src/tools/ask-browserstack/register.js" + ); + const { McpServer: RemoteMcpServer } = await import( + "@modelcontextprotocol/sdk/server/mcp.js" + ); + const streamed = vi.fn(() => ({ + async *[Symbol.asyncIterator]() { + yield { event: "result", data: { status: "ok", answer: "" } }; + }, + })); + const remote = new RemoteMcpServer({ name: "t", version: "0" }); + vi.spyOn(remote.server, "getClientCapabilities") + .mockReturnValue({ elicitation: {} } as never); + const tools = addAskBrowserstackAITool(remote, { + agentUrl: () => "https://atlas.example/agent", + mintToken: async () => MINTED, + credentialsFor: () => ({ username: "ing_Xx", accessKey: "SECRET" }), + streamTransport: streamed as never, + }); + + const { payload } = await call(tools); + expect((streamed.mock.calls[0] as never as unknown[])[2]) + .toMatchObject({ permission_relay: { mode: "stream" } }); + expect(payload.permission_relay.reason).not.toBe("remote_mode"); + } finally { + delete process.env.ASK_BROWSERSTACK_ALLOW_REMOTE_RELAY; + } + }); + + it("the opt-in does NOT force the relay onto a client that cannot be asked", async () => { + // The flag only lifts the blanket refusal. Whether a human can actually be reached + // is still per-client, and a client that never declared `elicitation` must still get + // a read-only run — otherwise the hosted server would stream asks nobody can see. + vi.resetModules(); + process.env.REMOTE_MCP = "true"; + process.env.ASK_BROWSERSTACK_ALLOW_REMOTE_RELAY = "true"; + try { + const { addAskBrowserstackAITool } = await import( + "../../src/tools/ask-browserstack/register.js" + ); + const { McpServer: RemoteMcpServer } = await import( + "@modelcontextprotocol/sdk/server/mcp.js" + ); + const streamed = vi.fn(() => ({ + async *[Symbol.asyncIterator]() { + yield { event: "result", data: { status: "ok", answer: "" } }; + }, + })); + const remote = new RemoteMcpServer({ name: "t", version: "0" }); + vi.spyOn(remote.server, "getClientCapabilities") + .mockReturnValue({ roots: {} } as never); // no elicitation + const tools = addAskBrowserstackAITool(remote, { + agentUrl: () => "https://atlas.example/agent", + mintToken: async () => MINTED, + credentialsFor: () => ({ username: "ing_Xx", accessKey: "SECRET" }), + streamTransport: streamed as never, + }); + + const { payload } = await call(tools); + expect("permission_relay" in (streamed.mock.calls[0] as never as unknown[])[2]!) + .toBe(false); + expect(payload.permission_relay.reason).toBe("no_human"); + } finally { + delete process.env.ASK_BROWSERSTACK_ALLOW_REMOTE_RELAY; + } + }); + + it("positive control: the same seam IS called when not in remote mode", async () => { + // Without this, the assertion above would pass just as happily if the seam were + // broken and nothing ever called it. + vi.resetModules(); + delete process.env.REMOTE_MCP; + const { addAskBrowserstackAITool } = await import( + "../../src/tools/ask-browserstack/register.js" + ); + const { McpServer: StdioMcpServer } = await import( + "@modelcontextprotocol/sdk/server/mcp.js" + ); + + const streamed = vi.fn(() => ({ + async *[Symbol.asyncIterator]() { + yield { event: "result", data: { status: "ok", answer: "" } }; + }, + })); + + const stdio = new StdioMcpServer({ name: "t", version: "0" }); + vi.spyOn(stdio.server, "getClientCapabilities") + .mockReturnValue({ elicitation: {} } as never); + const tools = addAskBrowserstackAITool(stdio, { + agentUrl: () => "https://atlas.example/agent", + mintToken: async () => MINTED, + credentialsFor: () => ({ username: "ing_Xx", accessKey: "SECRET" }), + streamTransport: streamed as never, + }); + + await call(tools); + // Without this the assertion above would pass just as happily if the relay were + // never offered to anyone. + expect(streamed).toHaveBeenCalledTimes(1); + expect((streamed.mock.calls[0] as never as unknown[])[2]) + .toMatchObject({ permission_relay: { mode: "stream" } }); + }); + + it("stdio does not regress: the relay is still offered and still works", async () => { + // REMOTE_MCP unset — byte-identical to every other test in this file. + const server = await buildServer(); + fakeClient(server.getInstance(), { elicitation: {} }, [ + { action: "accept" }, + ]); + const stub = atlas({ asks: [{ perm_id: PERM_A, description: "Create folder." }] }); + + const { payload } = await call(server.getTools()); + expect(stub.calls[0].body.permission_relay).toEqual({ mode: "stream" }); + expect(payload.permission_relay.used).toBe(true); + expect(payload.permission_relay.reason).toBe(""); + }); + + it("REMOTE_MCP=\"false\" is stdio, not remote", async () => { + vi.resetModules(); + process.env.REMOTE_MCP = "false"; + const { BrowserStackMcpServer } = await import("../../src/server-factory.js"); + const server = new BrowserStackMcpServer(CONFIG); + fakeClient(server.getInstance(), { elicitation: {} }, [{ action: "accept" }]); + const stub = atlas({ asks: [{ perm_id: PERM_A, description: "Create folder." }] }); + + const { payload } = await call(server.getTools()); + expect(stub.calls[0].body.permission_relay).toBeDefined(); + expect(payload.permission_relay.used).toBe(true); + }); + }); + + it("falls back to the built-in staging host when no override is set", async () => { + delete process.env.ASK_BROWSERSTACK_ATLAS_URL; + const seen: string[] = []; + vi.stubGlobal("fetch", async (url: string) => { + seen.push(String(url)); + return { + status: 200, + headers: { get: () => "application/json" }, + json: async () => (String(url).includes("oauth2") + ? { access_token: MINTED, expires_in: 3600 } + : { ok: true, status: "ok", answer: "" }), + }; + }); + const server = await buildServer(); + fakeClient(server.getInstance(), { roots: {} }, []); + + const { result } = await call(server.getTools()); + expect(result.isError).toBeUndefined(); + // DEFAULT-PROD-HOSTS: asserted literally so repointing must be deliberate. + expect(seen).toContain("https://workflows.browserstack.com/agent"); + }); +}); + +describe("askBrowserstackAI, against the injected seam", () => { + afterEach(() => vi.restoreAllMocks()); + + it("refuses rather than calling Atlas unauthenticated, and never names the token", async () => { + const mcp = new McpServer({ name: "t", version: "0" }); + const streamed = vi.fn(); + const tools = addAskBrowserstackAITool(mcp, { + agentUrl: () => "https://atlas.example/agent", + mintToken: async () => { + throw new AskError( + "BrowserStack AI is not authenticated: BROWSERSTACK_USERNAME and " + + "BROWSERSTACK_ACCESS_KEY are required to sign in", + ); + }, + credentialsFor: () => ({ username: "u", accessKey: "k" }), + streamTransport: streamed as never, + }); + + const { payload } = await call(tools); + expect(payload.ok).toBe(false); + expect(payload.error).toMatch(/BROWSERSTACK_ACCESS_KEY/); + // The point of the test: no token, so the stream is never opened at all — the + // refusal happens before anything reaches the network. + expect(streamed).not.toHaveBeenCalled(); + }); + + it("does not need the user's access key to reach /agent", async () => { + // It is not sent on this route, so its absence must not refuse the call the way the + // product-API path would. + const mcp = new McpServer({ name: "t", version: "0" }); + const streamed = vi.fn(() => ({ + async *[Symbol.asyncIterator]() { + yield { event: "result", data: { status: "ok", answer: "" } }; + }, + })); + const tools = addAskBrowserstackAITool(mcp, { + agentUrl: () => "https://atlas.example/agent", + mintToken: async () => MINTED, + credentialsFor: () => ({ username: "ing_Xx", accessKey: "" }), + streamTransport: streamed as never, + }); + + const { payload } = await call(tools); + expect(payload.ok).toBe(true); + expect(streamed).toHaveBeenCalledTimes(1); + expect((streamed.mock.calls[0] as never as unknown[])[2]) + .toMatchObject({ user_id: "ing_Xx" }); + }); + + it("reports a transport failure with nothing left to clean up", async () => { + // Under A2 this test existed because a thrown transport could strand a bound port, + // and the assertion was that the listener still closed. A1 binds NOTHING — no port, + // no listener, no per-run bearer — so the leak this guarded against cannot happen. + // What is left worth pinning is that the failure still surfaces as a clean result + // rather than an exception escaping the tool. + const mcp = new McpServer({ name: "t", version: "0" }); + vi.spyOn(mcp.server, "getClientCapabilities").mockReturnValue({ elicitation: {} } as never); + const tools = addAskBrowserstackAITool(mcp, { + agentUrl: () => "https://atlas.example/agent", + mintToken: async () => MINTED, + credentialsFor: () => ({ username: "u", accessKey: "k" }), + streamTransport: (() => { + throw new Error("boom"); + }) as never, + }); + + const { payload } = await call(tools); + expect(payload.error).toBe("boom"); + expect(payload.ok).toBe(false); + }); +}); diff --git a/tests/tools/askBrowserstackStream.test.ts b/tests/tools/askBrowserstackStream.test.ts new file mode 100644 index 0000000..a1d36f0 --- /dev/null +++ b/tests/tools/askBrowserstackStream.test.ts @@ -0,0 +1,364 @@ +/** + * CONTRACT v2 (A1) — the stream transport. + * + * These test the PIPE, not the judgement. Whether a human approved, how an elicitation + * outcome maps to allow/deny, what the result looks like: all of that is `relay.ts`, + * which A1 does not touch and which `askBrowserstack.test.ts` already covers. Mixing + * the two here would imply the transport can influence a decision, which is the exact + * property the design prevents. + * + * Chunk boundaries get most of the attention on purpose. A dropped frame is an ask that + * never reaches the human — a write that silently never gets approved and never says + * why — and it is the failure a hand-rolled SSE parser actually produces. + */ + +import { describe, expect, it, vi } from "vitest"; + +import { + EVENT_PERMISSION, + EVENT_RESULT, + EVENT_RUN, + WHOLE_RUN_TIMEOUT_MS, + decisionUrl, + fetchAgentStreamTransport, + fetchDecisionTransport, + parseFrame, + splitFrames, +} from "../../src/tools/ask-browserstack/stream.js"; + +describe("splitFrames", () => { + it("returns complete frames and keeps the remainder", () => { + const { frames, rest } = splitFrames("a\n\nb\n\npartial"); + expect(frames).toEqual(["a", "b"]); + expect(rest).toBe("partial"); + }); + + it("holds a frame that has not terminated yet", () => { + // The single most likely real failure: an ask arrives split across two reads. If + // this returned it early the JSON would be truncated and the ask lost. + const { frames, rest } = splitFrames("event: permission\ndata: {\"perm"); + expect(frames).toEqual([]); + expect(rest).toBe('event: permission\ndata: {"perm'); + }); + + it("handles several frames arriving in one read", () => { + const { frames, rest } = splitFrames("one\n\ntwo\n\nthree\n\n"); + expect(frames).toEqual(["one", "two", "three"]); + expect(rest).toBe(""); + }); +}); + +describe("parseFrame", () => { + it("parses an event and its JSON payload", () => { + expect(parseFrame('event: run\ndata: {"run_id":"run-abc"}')).toEqual({ + event: "run", + data: { run_id: "run-abc" }, + }); + }); + + it("ignores the heartbeat", () => { + // A bare comment. Its only job is to be a read on the socket so the ingress does + // not close the connection while a human is thinking, so it must not surface as + // an event the caller has to know about. + expect(parseFrame(": keepalive")).toBeNull(); + }); + + it("ignores a frame with no event name", () => { + expect(parseFrame('data: {"stray":true}')).toBeNull(); + }); + + it("drops a frame whose JSON will not parse rather than guessing", () => { + // Safe to drop: the only consequence is that an ask goes unanswered and Atlas's + // gate denies on its own expiry. Never that something is approved. + expect(parseFrame("event: permission\ndata: {not json")).toBeNull(); + }); + + it("keeps a description containing JSON-escaped punctuation", () => { + const frame = + 'event: permission\ndata: {"perm_id":"p","description":"Mark run \\"1043\\" done"}'; + expect((parseFrame(frame)?.data as { description: string }).description).toBe( + 'Mark run "1043" done', + ); + }); +}); + +/** A Response whose body streams the given chunks, as Node's fetch would. */ +function streamingResponse(chunks: string[], status = 200): Response { + const encoder = new TextEncoder(); + const body = new ReadableStream({ + start(controller) { + for (const c of chunks) controller.enqueue(encoder.encode(c)); + controller.close(); + }, + }); + return new Response(body, { + status, + headers: { "content-type": "text/event-stream" }, + }); +} + +describe("fetchAgentStreamTransport", () => { + it("yields run, permission and result in order across chunk boundaries", async () => { + // The frames are deliberately split mid-JSON and mid-frame, which is what a real + // socket does and what a naive parser gets wrong. + const fetchMock = vi.fn().mockResolvedValue( + streamingResponse([ + 'event: run\ndata: {"run_id":"run-1"}\n', + '\nevent: permission\ndata: {"perm_id":"perm-1","product":"tm",', + '"mode":"ask-once","description":"Mark run 1043 complete"}\n\n', + ": keepalive\n\n", + 'event: result\ndata: {"ok":true,"status":"ok"}\n\n', + ]), + ); + vi.stubGlobal("fetch", fetchMock); + + const seen: Array<{ event: string; data: unknown }> = []; + for await (const ev of fetchAgentStreamTransport()( + "https://atlas.test/agent", + { Authorization: "Bearer t" }, + { task: "t", product: "tm" } as never, + )) { + seen.push(ev); + } + + expect(seen.map((e) => e.event)).toEqual([ + EVENT_RUN, + EVENT_PERMISSION, + EVENT_RESULT, + ]); + expect((seen[1].data as { description: string }).description).toBe( + "Mark run 1043 complete", + ); + // Asks for a stream explicitly, and keeps the caller's auth header. + const init = fetchMock.mock.calls[0][1]; + expect(init.headers.Accept).toBe("text/event-stream"); + expect(init.headers.Authorization).toBe("Bearer t"); + expect(init.redirect).toBe("manual"); + vi.unstubAllGlobals(); + }); + + it("throws rather than iterating empty when the response is not a stream", async () => { + // An empty iteration is indistinguishable from "the run finished and said + // nothing", so a refusal has to be loud. + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue(new Response("nope", { status: 403 })), + ); + const iterate = async () => { + for await (const _ of fetchAgentStreamTransport()( + "https://atlas.test/agent", + {}, + {} as never, + )) { + void _; + } + }; + await expect(iterate()).rejects.toThrow(/HTTP 403/); + vi.unstubAllGlobals(); + }); + + it("yields a trailing frame that never got its blank line", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue( + streamingResponse(['event: result\ndata: {"ok":true}']), + ), + ); + const seen = []; + for await (const ev of fetchAgentStreamTransport()( + "https://atlas.test/agent", + {}, + {} as never, + )) { + seen.push(ev); + } + expect(seen).toHaveLength(1); + expect(seen[0].event).toBe(EVENT_RESULT); + vi.unstubAllGlobals(); + }); + + it("bounds the whole run, not one request", () => { + // v2 §4: the old 330s outer rung meant nothing once the stream spans a run that + // may hold several 300s approvals in series. + expect(WHOLE_RUN_TIMEOUT_MS).toBe(1_800_000); + }); +}); + +describe("fetchDecisionTransport", () => { + it("returns the status so the caller can tell 204 from 409", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue(new Response(null, { status: 204 })), + ); + const status = await fetchDecisionTransport()( + "https://atlas.test/agent/run-1/permission", + {}, + { perm_id: "perm-1", decision: "allow", reason: "" }, + ); + expect(status).toBe(204); + vi.unstubAllGlobals(); + }); + + it("reports 0 when the decision never left, rather than implying a refusal", async () => { + // The gate on the far side is still waiting and denies on its own expiry, so a + // lost decision is safe. But it must not be reported as "the human said no". + vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new Error("socket closed"))); + const status = await fetchDecisionTransport()( + "https://atlas.test/agent/run-1/permission", + {}, + { perm_id: "perm-1", decision: "allow", reason: "" }, + ); + expect(status).toBe(0); + vi.unstubAllGlobals(); + }); +}); + +describe("decisionUrl", () => { + it("builds the v2 §3 path from the agent url", () => { + expect(decisionUrl("https://atlas.test/agent", "run-abc")).toBe( + "https://atlas.test/agent/run-abc/permission", + ); + }); + + it("tolerates a trailing slash and escapes the id", () => { + expect(decisionUrl("https://atlas.test/agent/", "run-a/b")).toBe( + "https://atlas.test/agent/run-a%2Fb/permission", + ); + }); +}); + +describe("the relay never claims a person answered when none did", () => { + it("says nobody was present, instead of 'the answers are in approvals'", async () => { + // Observed live against a real Atlas: a headless client returned `cancel`, and the + // result carried BOTH "BrowserStack asked before each change and the answers are in + // `approvals`" AND an approvals entry reading "refused: nobody was there to be + // asked". Two sentences in one payload contradicting each other. + const { buildResult, RELAY_ON_NO_ANSWER_DETAIL, nobodyAnswered } = await import( + "../../src/tools/ask-browserstack/relay.js" + ); + + expect( + nobodyAnswered([ + { description: "Create folder.", decision: "deny", reason: "cancelled" }, + ]), + ).toBe(true); + + const result = buildResult( + { + status: 200, + body: { + status: "blocked", + answer: null, + approvals: [ + { description: "Create folder.", decision: "deny", reason: "cancelled", + applied: false }, + ], + applied_before_stop: false, + permission_relay: { used: true, reason: "" }, + }, + }, + [{ description: "Create folder.", decision: "deny", reason: "cancelled" }], + "offered", + "tm", + ); + + expect(result.permission_relay.used).toBe(true); + expect(result.permission_relay.detail).toBe(RELAY_ON_NO_ANSWER_DETAIL); + // And the two halves of the payload now agree with each other. + expect(result.approvals[0].outcome).toMatch(/nobody was there/); + expect(result.permission_relay.detail).toMatch(/without a person/); + }); + + it("still credits a real human answer", async () => { + // The correction must not fire when somebody actually declined — "a person said no" + // and "no person was there" call for different things from the reader. + const { buildResult, RELAY_ON_DETAIL, nobodyAnswered } = await import( + "../../src/tools/ask-browserstack/relay.js" + ); + expect( + nobodyAnswered([ + { description: "x", decision: "deny", reason: "declined" }, + ]), + ).toBe(false); + + const result = buildResult( + { + status: 200, + body: { + status: "blocked", + approvals: [{ description: "x", decision: "deny", reason: "declined", + applied: false }], + permission_relay: { used: true, reason: "" }, + }, + }, + [{ description: "x", decision: "deny", reason: "declined" }], + "offered", + "tm", + ); + expect(result.permission_relay.detail).toBe(RELAY_ON_DETAIL); + }); + + it("does not fire on an empty trail", async () => { + // Nothing asked at all is a different state, already described correctly. + const { nobodyAnswered } = await import("../../src/tools/ask-browserstack/relay.js"); + expect(nobodyAnswered([])).toBe(false); + }); + + it("does not fire when one ask was allowed", async () => { + const { nobodyAnswered } = await import("../../src/tools/ask-browserstack/relay.js"); + expect( + nobodyAnswered([ + { description: "a", decision: "allow", reason: "" }, + { description: "b", decision: "deny", reason: "cancelled" }, + ]), + ).toBe(false); + }); +}); + +describe("an auth outage is not a credential problem", () => { + it("a 5xx says the service is down, and never names your credentials", async () => { + // Observed live: preprod auth returned 503 during an outage and the tool answered + // "Your BrowserStack credentials were rejected … Check BROWSERSTACK_USERNAME and + // BROWSERSTACK_ACCESS_KEY" — sending someone to audit env vars that had worked + // minutes earlier. The reader asked "is preprod down?", which was right, and the + // message argued against it. + const { AUTH_SERVER_ERROR_DETAIL, mintCentralToken } = await import( + "../../src/tools/ask-browserstack/central-oauth.js" + ); + + for (const status of [500, 502, 503, 504]) { + const transport = vi.fn(async () => ({ status, body: null })); + await expect( + mintCentralToken( + "https://auth.example/token", + { username: "u", accessKey: "k" }, + transport as never, + ), + ).rejects.toThrow(/auth is unavailable/); + } + + const text = AUTH_SERVER_ERROR_DETAIL(503); + expect(text).toMatch(/CREDENTIALS ARE NOT THE PROBLEM/); + expect(text).not.toMatch(/BROWSERSTACK_USERNAME/); + expect(text).not.toMatch(/rejected/); + }); + + it("still blames the credentials on a 401", async () => { + // The split must not make a genuine rejection sound like an outage: those need + // opposite actions from the reader. + const { mintCentralToken } = await import( + "../../src/tools/ask-browserstack/central-oauth.js" + ); + const transport = vi.fn(async () => ({ + status: 401, + body: { error: "invalid_client" }, + })); + await expect( + mintCentralToken( + "https://auth.example/token", + { username: "u", accessKey: "k" }, + transport as never, + ), + ).rejects.toThrow(/credentials were rejected/); + }); +});